diff --git a/Cargo.lock b/Cargo.lock index 1dd9fb7a..454f5d48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -261,6 +261,7 @@ name = "arca-7z" version = "0.10.0" dependencies = [ "arca-core", + "arca-xz", "crc32fast", "getrandom 0.4.3", "sevenz-rust2", @@ -3062,13 +3063,12 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rars" version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0665518097392be04a322dec9eba41e88028f6aca85f5d61f8abb068691f2266" dependencies = [ "aes", "aho-corasick", "getrandom 0.4.3", "hmac", + "rayon", "sha1", "sha2", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index dce37126..a126d5ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ ] # Solo compila en Windows: se queda fuera del workspace para que # `cargo build` siga funcionando en Linux y macOS. -exclude = ["windows/arca-shell", "arca-7z/vendor/sevenz-rust2"] +exclude = ["windows/arca-shell", "arca-7z/vendor/sevenz-rust2", "arca-rar/vendor/rars"] [workspace.package] version = "0.10.0" @@ -80,3 +80,4 @@ inherits = "release" [patch.crates-io] sevenz-rust2 = { path = "arca-7z/vendor/sevenz-rust2" } +rars = { path = "arca-rar/vendor/rars" } diff --git a/RAR-NOTICES.txt b/RAR-NOTICES.txt index 96212748..d19659e7 100644 --- a/RAR-NOTICES.txt +++ b/RAR-NOTICES.txt @@ -113,6 +113,24 @@ Source: registry+https://github.com/rust-lang/crates.io-index LICENSE-APACHE -> SHA-256 c6596eb7be8581c18be736c846fb9173b69eccf6ef94c5135893ec56bd92ba08 LICENSE-MIT -> SHA-256 61d383b05b87d78f94d2937e2580cce47226d17823c0430fbcad09596537efcf +crossbeam-deque 0.8.8 +Declared license: MIT OR Apache-2.0 +Source: registry+https://github.com/rust-lang/crates.io-index + LICENSE-APACHE -> SHA-256 a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2 + LICENSE-MIT -> SHA-256 5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab + +crossbeam-epoch 0.9.21 +Declared license: MIT OR Apache-2.0 +Source: registry+https://github.com/rust-lang/crates.io-index + LICENSE-APACHE -> SHA-256 a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2 + LICENSE-MIT -> SHA-256 5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab + +crossbeam-utils 0.8.23 +Declared license: MIT OR Apache-2.0 +Source: registry+https://github.com/rust-lang/crates.io-index + LICENSE-APACHE -> SHA-256 a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2 + LICENSE-MIT -> SHA-256 5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab + crypto-common 0.2.2 Declared license: MIT OR Apache-2.0 Source: registry+https://github.com/rust-lang/crates.io-index @@ -131,6 +149,12 @@ Source: registry+https://github.com/rust-lang/crates.io-index LICENSE-APACHE -> SHA-256 a9040321c3712d8fd0b09cf52b17445de04a23a10165049ae187cd39e5c86be5 LICENSE-MIT -> SHA-256 af59cea35d7f5e2777a713b8d155d65efa2c339eb43f3c14e868c6ac8506edad +either 1.18.0 +Declared license: MIT OR Apache-2.0 +Source: registry+https://github.com/rust-lang/crates.io-index + LICENSE-APACHE -> SHA-256 a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2 + LICENSE-MIT -> SHA-256 7576269ea71f767b99297934c0b2367532690f8c4badc695edf8e04ab6a1e545 + errno 0.3.14 Declared license: MIT OR Apache-2.0 Source: registry+https://github.com/rust-lang/crates.io-index @@ -275,9 +299,21 @@ Source: registry+https://github.com/rust-lang/crates.io-index rars 0.10.2 Declared license: Apache-2.0 -Source: registry+https://github.com/rust-lang/crates.io-index +Source: vendored in arca-rar/vendor from registry+https://github.com/rust-lang/crates.io-index COPYING -> SHA-256 cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 +rayon 1.12.0 +Declared license: MIT OR Apache-2.0 +Source: registry+https://github.com/rust-lang/crates.io-index + LICENSE-APACHE -> SHA-256 a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2 + LICENSE-MIT -> SHA-256 0621878e61f0d0fda054bcbe02df75192c28bde1ecc8289cbd86aeba2dd72720 + +rayon-core 1.13.0 +Declared license: MIT OR Apache-2.0 +Source: registry+https://github.com/rust-lang/crates.io-index + LICENSE-APACHE -> SHA-256 a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2 + LICENSE-MIT -> SHA-256 0621878e61f0d0fda054bcbe02df75192c28bde1ecc8289cbd86aeba2dd72720 + rustix 1.1.4 Declared license: Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT Source: registry+https://github.com/rust-lang/crates.io-index @@ -415,6 +451,34 @@ This project is dual-licensed under the Unlicense and MIT licenses. You may use this code under the terms of either license. +===== SHA-256 0621878e61f0d0fda054bcbe02df75192c28bde1ecc8289cbd86aeba2dd72720 ===== +Copyright (c) 2010 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + ===== SHA-256 0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f ===== The MIT License (MIT) @@ -1111,6 +1175,36 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +===== SHA-256 5734ed989dfca1f625b40281ee9f4530f91b2411ec01cb748223e7eb87e201ab ===== +The MIT License (MIT) + +Copyright (c) 2019 The Crossbeam Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + ===== SHA-256 5b3d8a674979e158328c57f5c4cc12150df79d0bdf12c86eafc2e9c6006a2d4e ===== Short version for non-lawyers: @@ -1462,6 +1556,34 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +===== SHA-256 7576269ea71f767b99297934c0b2367532690f8c4badc695edf8e04ab6a1e545 ===== +Copyright (c) 2015 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + ===== SHA-256 831e0f43ad0bf014c1c4fec5767aae470434c1d66d6e671be2d823e729063e25 ===== Copyright (c) 2016-2026 The RustCrypto Project Developers Copyright (c) 2016 Artyom Pavlov diff --git a/arca-7z/Cargo.toml b/arca-7z/Cargo.toml index a679bb07..65a26779 100644 --- a/arca-7z/Cargo.toml +++ b/arca-7z/Cargo.toml @@ -9,6 +9,7 @@ description = "Bounded, safe Rust 7z reading and writing for Arca" [dependencies] arca-core.workspace = true +arca-xz.workspace = true sevenz-rust2 = { version = "=0.23.0", default-features = false, features = ["aes256", "compress", "deflate"] } crc32fast.workspace = true tempfile = "3" diff --git a/arca-7z/src/lib.rs b/arca-7z/src/lib.rs index 00e92eb9..508f2352 100644 --- a/arca-7z/src/lib.rs +++ b/arca-7z/src/lib.rs @@ -4,7 +4,7 @@ mod read; mod write; pub use read::{Phase, Progress, SevenZArchive}; -pub use write::{create_7z, CreateOptions, Source}; +pub use write::{create_7z, workers, CreateOptions, Source}; use arca_core::{Error, Result}; diff --git a/arca-7z/src/read.rs b/arca-7z/src/read.rs index 8dd71f32..5a9766ca 100644 --- a/arca-7z/src/read.rs +++ b/arca-7z/src/read.rs @@ -300,6 +300,10 @@ impl SevenZArchive { } Ok(()) }; + // One thread: the multithreaded LZMA2 reader decodes a whole run between + // dictionary resets into memory, and a stream with no resets is a whole + // block, which an archive can make as large as it likes. + let threads = 1; for (block, &needed) in blocks.iter().enumerate() { if !needed { continue; @@ -316,15 +320,16 @@ impl SevenZArchive { } else { &no_password }; - let result = BlockDecoder::new(1, block, &self.archive, password, &mut self.source) - .for_each_entries(&mut |_, reader| { - if let Err(error) = visit(index, reader) { - callback_error = Some(error); - return Ok(false); - } - index += 1; - Ok(true) - }); + let result = + BlockDecoder::new(threads, block, &self.archive, password, &mut self.source) + .for_each_entries(&mut |_, reader| { + if let Err(error) = visit(index, reader) { + callback_error = Some(error); + return Ok(false); + } + index += 1; + Ok(true) + }); if let Some(error) = callback_error { return Err(error); } diff --git a/arca-7z/src/write.rs b/arca-7z/src/write.rs index 6000c613..dfdbcdc2 100644 --- a/arca-7z/src/write.rs +++ b/arca-7z/src/write.rs @@ -7,7 +7,7 @@ use std::{ use arca_core::{Error, Level, Result}; use sevenz_rust2::{ encoder_options::{AesEncoderOptions, Lzma2Options}, - ArchiveEntry, ArchiveWriter, EncoderConfiguration, EncoderMethod, Password, + ArchiveEntry, ArchiveWriter, EncoderConfiguration, EncoderMethod, Password, SourceReader, }; use crate::{entry_path, upstream, Phase, Progress}; @@ -98,9 +98,8 @@ pub fn create_7z( let mut writer = ArchiveWriter::new(io::BufWriter::new(temporary.as_file_mut())) .map_err(|e| upstream(e, false))?; writer.set_encrypt_header(options.hide_names); - let mut bytes = 0; - for (index, (source, meta)) in sources.iter().zip(&metadata).enumerate() { - writer.set_content_methods(methods(options)?); + let entry_of = |index: usize| { + let (source, meta) = (&sources[index], &metadata[index]); let mut entry = if meta.is_dir() { ArchiveEntry::new_directory(&source.name) } else { @@ -113,45 +112,69 @@ pub fn create_7z( entry.has_last_modified_date = true; entry.last_modified_date = time; } - let progress = Progress { + entry + }; + // Folders and empty files have no data and go in first, outside any block. + let (empty, full): (Vec, Vec) = + (0..sources.len()).partition(|&i| metadata[i].is_dir() || metadata[i].len() == 0); + for &index in &empty { + if !notify(Progress { phase: Phase::Create, entries_done: index, entries_total: sources.len(), - bytes_done: bytes, - name: &source.name, - }; - if !notify(progress) { + bytes_done: 0, + name: &sources[index].name, + }) { return Err(Error::Cancelled); } - if meta.is_dir() || meta.len() == 0 { - entry.has_stream = false; - writer - .push_archive_entry::(entry, None) - .map_err(|e| upstream(e, false))?; - } else { - let file = fs::File::open(&source.path)?; - let mut input = Input { - file, - notify, - progress, - cancelled: false, - bytes: 0, - }; - let result = writer.push_archive_entry(entry, Some(&mut input)); - if input.cancelled { - return Err(Error::Cancelled); - } - result.map_err(|e| upstream(e, false))?; - if input.bytes != meta.len() { - return Err(Error::Io(io::Error::other( - "7z source changed size during creation", - ))); - } - bytes += input.bytes; + let mut entry = entry_of(index); + entry.has_stream = false; + writer + .push_archive_entry::(entry, None) + .map_err(|e| upstream(e, false))?; + } + let tracker = std::cell::RefCell::new(Tracker { + notify, + total: sources.len(), + done: empty.len(), + bytes: 0, + cancelled: false, + changed: false, + }); + let workers = workers(options.level); + for block in solid_blocks(sources, &metadata, &full) { + writer.set_content_methods(methods(options, block.executables, workers)?); + let entries = block.files.iter().map(|&i| entry_of(i)).collect(); + let readers = block + .files + .iter() + .map(|&i| { + SourceReader::new(Lazy { + path: &sources[i].path, + name: &sources[i].name, + expected: metadata[i].len(), + file: None, + read: 0, + tracker: &tracker, + }) + }) + .collect(); + let result = writer.push_archive_entries(entries, readers); + let state = tracker.borrow(); + if state.cancelled { + return Err(Error::Cancelled); } + if state.changed { + return Err(Error::Io(io::Error::other( + "7z source changed size during creation", + ))); + } + drop(state); + result.map_err(|e| upstream(e, false))?; } + let Tracker { notify, bytes, .. } = tracker.into_inner(); // A header must never reuse the last file's AES salt and IV. - writer.set_content_methods(methods(options)?); + writer.set_content_methods(methods(options, false, 1)?); writer.finish()?.flush()?; if !notify(Progress { phase: Phase::Create, @@ -167,7 +190,73 @@ pub fn create_7z( Ok(()) } -fn methods(options: &CreateOptions<'_>) -> Result> { +/// The most a solid block holds. Reading one entry decodes everything before +/// it in its block, so a block of the whole archive would make opening one +/// file of a large archive as slow as extracting all of it. +const SOLID_BYTES: u64 = 256 << 20; + +/// Extensions of x86 and x64 machine code, which the BCJ filter makes much +/// easier to compress by turning relative call targets into absolute ones. +/// 7-Zip picks them the same way, by name. +const EXECUTABLE: [&str; 9] = [ + "exe", "dll", "sys", "ocx", "cpl", "scr", "drv", "efi", "mui", +]; + +struct Block { + files: Vec, + executables: bool, +} + +/// Groups the files with data into solid blocks, executables apart so that +/// only they go through the filter, each group sorted by extension and then +/// by name so that similar files sit next to each other in the stream. +fn solid_blocks(sources: &[Source], metadata: &[fs::Metadata], files: &[usize]) -> Vec { + let extension = |i: usize| { + Path::new(&sources[i].name) + .extension() + .map(|e| e.to_string_lossy().to_ascii_lowercase()) + .unwrap_or_default() + }; + let (mut executables, mut rest): (Vec, Vec) = files + .iter() + .partition(|&&i| EXECUTABLE.contains(&extension(i).as_str())); + let mut blocks = Vec::new(); + for (group, filtered) in [(&mut rest, false), (&mut executables, true)] { + group.sort_by_cached_key(|&i| (extension(i), sources[i].name.to_lowercase())); + let mut block = Vec::new(); + let mut size = 0u64; + for &i in group.iter() { + if !block.is_empty() && size + metadata[i].len() > SOLID_BYTES { + blocks.push(Block { + files: std::mem::take(&mut block), + executables: filtered, + }); + size = 0; + } + size += metadata[i].len(); + block.push(i); + } + if !block.is_empty() { + blocks.push(Block { + files: block, + executables: filtered, + }); + } + } + blocks +} + +/// LZMA2 threads for `level`, as many cores as fit in the same memory budget +/// the .xz encoder keeps to. +pub fn workers(level: Level) -> usize { + arca_xz::workers(level, 0) +} + +fn methods( + options: &CreateOptions<'_>, + executables: bool, + workers: usize, +) -> Result> { let mut methods = Vec::new(); if let Some(password) = options.password { let mut iv = [0; 16]; @@ -185,31 +274,80 @@ fn methods(options: &CreateOptions<'_>) -> Result> { .into(), ); } - methods.push(match options.level.get() { - 0 => EncoderMethod::COPY.into(), - n => Lzma2Options::from_level(u32::from(n)).into(), + let level = u32::from(options.level.get()); + if level == 0 { + methods.push(EncoderMethod::COPY.into()); + return Ok(methods); + } + let preset = arca_xz::preset(options.level); + methods.push(if workers > 1 { + Lzma2Options::from_level_mt(level, workers as u32, arca_xz::block_size(preset)).into() + } else { + Lzma2Options::from_level(level).into() }); + // The last method is the first applied to the data. + if executables { + methods.push(EncoderMethod::BCJ_X86_FILTER.into()); + } Ok(methods) } -struct Input<'a> { - file: fs::File, - notify: &'a mut dyn FnMut(Progress<'_>) -> bool, - progress: Progress<'a>, - cancelled: bool, +/// What every file of a solid block shares while the block is written: the +/// caller's progress, and whether reading stopped and why. +struct Tracker<'n> { + notify: &'n mut dyn FnMut(Progress<'_>) -> bool, + total: usize, + done: usize, bytes: u64, + cancelled: bool, + changed: bool, } -impl Read for Input<'_> { +/// A file of a solid block, opened when the block first reads from it so that +/// a block of thousands of files never holds thousands of handles. +struct Lazy<'a, 'n> { + path: &'a Path, + name: &'a str, + expected: u64, + file: Option, + read: u64, + tracker: &'a std::cell::RefCell>, +} + +impl Read for Lazy<'_, '_> { fn read(&mut self, buffer: &mut [u8]) -> io::Result { - let mut progress = self.progress; - progress.bytes_done += self.bytes; - if !(self.notify)(progress) { - self.cancelled = true; + let mut tracker = self.tracker.borrow_mut(); + let progress = Progress { + phase: Phase::Create, + entries_done: tracker.done, + entries_total: tracker.total, + bytes_done: tracker.bytes, + name: self.name, + }; + if !(tracker.notify)(progress) { + tracker.cancelled = true; return Err(io::Error::other("cancelled")); } - let size = self.file.read(buffer)?; - self.bytes += size as u64; + if self.file.is_none() { + self.file = Some(fs::File::open(self.path)?); + } + let size = match self.file.as_mut() { + Some(file) => file.read(buffer)?, + None => 0, + }; + self.read += size as u64; + tracker.bytes += size as u64; + if size == 0 { + if self.read != self.expected { + tracker.changed = true; + return Err(io::Error::other("7z source changed size during creation")); + } + self.file = None; + tracker.done += 1; + } else if self.read > self.expected { + tracker.changed = true; + return Err(io::Error::other("7z source changed size during creation")); + } Ok(size) } } diff --git a/arca-7z/tests/archive.rs b/arca-7z/tests/archive.rs index abe5792a..04856914 100644 --- a/arca-7z/tests/archive.rs +++ b/arca-7z/tests/archive.rs @@ -215,12 +215,21 @@ fn empty_and_unicode_roundtrip_all_levels_and_password_modes() { let mut archive = SevenZArchive::open(fs::File::open(path).unwrap(), password).unwrap(); assert_eq!(archive.has_encrypted_header(), hide_names); assert_eq!(archive.has_encrypted(), password.is_some()); - assert_eq!(archive.entries()[1].name, sources[1].name); - assert!(archive.entries()[0].is_dir); - assert!(!archive.entries()[2].is_dir); - assert_eq!(archive.read_entry(2, 0, &mut |_| true).unwrap(), b""); + // Entries are grouped by kind for solid compression, as 7-Zip does, + // so they are found by name rather than by the order they were given. + let at = |name: &str| { + archive + .entries() + .iter() + .position(|e| e.name == name) + .unwrap() + }; + let (folder, data, empty) = (at("folder"), at(&sources[1].name), at("empty")); + assert!(archive.entries()[folder].is_dir); + assert!(!archive.entries()[empty].is_dir); + assert_eq!(archive.read_entry(empty, 0, &mut |_| true).unwrap(), b""); assert_eq!( - archive.read_entry(1, 100, &mut |_| true).unwrap(), + archive.read_entry(data, 100, &mut |_| true).unwrap(), b"hello compressed world" ); archive.test(&mut |_| true).unwrap(); @@ -671,3 +680,127 @@ fn malformed_header_mutations_never_panic() { } } } + +/// Bytes that look like x86 code and compress like a program: the same few +/// call instructions with targets that drift, which BCJ is there to smooth. +fn machine_code(len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let mut target = 0x1000u32; + while out.len() < len { + out.push(0xE8); + out.extend_from_slice(&target.wrapping_sub(out.len() as u32).to_le_bytes()); + out.extend_from_slice(b"\x48\x89\xc7\x90"); + target = target.wrapping_add(16); + } + out.truncate(len); + out +} + +/// Text that varies enough to need several multithreaded LZMA2 chunks. +fn prose(len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + let mut n = 0u64; + while out.len() < len { + n = n.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + out.extend_from_slice(format!("line {} of the archive, {}\n", n % 977, n >> 50).as_bytes()); + } + out.truncate(len); + out +} + +// Solid blocks, executables in a block of their own behind the BCJ filter, +// more files than a block could hold open at once, and data large enough to +// be cut into chunks by every LZMA2 thread: all of it reads back, with and +// without a password. +#[test] +fn solid_blocks_with_executables_and_many_files_round_trip() { + let temp = tempfile::tempdir().unwrap(); + let mut sources = Vec::new(); + let mut expected = Vec::new(); + let mut add = |name: String, data: Vec| { + let path = temp.path().join(format!("f{}", sources.len())); + fs::write(&path, &data).unwrap(); + sources.push(Source { + path, + name: name.clone(), + }); + expected.push((name, data)); + }; + add("app/program.exe".into(), machine_code(300_000)); + add("app/library.DLL".into(), machine_code(120_000)); + add("big.txt".into(), prose(7 << 20)); + for i in 0..1500 { + add( + format!("many/{i:04}.txt"), + format!("file number {i}\n").repeat(i % 7 + 1).into_bytes(), + ); + } + for (level, password) in [ + (Level::FAST, None), + (Level::FAST, Some("secret")), + (Level::STORE, None), + ] { + let path = temp.path().join("out.7z"); + create_7z( + &path, + &sources, + &CreateOptions { + level, + password, + hide_names: false, + }, + &mut |_| true, + ) + .unwrap(); + let mut archive = SevenZArchive::open(fs::File::open(&path).unwrap(), password).unwrap(); + assert_eq!(archive.entries().len(), expected.len()); + archive.test(&mut |_| true).unwrap(); + let checked = [ + "app/program.exe", + "app/library.DLL", + "big.txt", + "many/0000.txt", + "many/1499.txt", + ]; + for (name, data) in expected + .iter() + .filter(|(n, _)| checked.contains(&n.as_str())) + { + let index = archive + .entries() + .iter() + .position(|e| &e.name == name) + .unwrap(); + let read = archive.read_entry(index, 32 << 20, &mut |_| true).unwrap(); + assert!(&read == data, "{name} at {level:?}"); + } + } +} + +// A source that grows while a solid block reads it fails the whole archive +// instead of writing an entry whose recorded size is wrong. +#[test] +fn a_source_that_changes_size_in_a_solid_block_fails() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("a.txt"); + fs::write(&path, b"abc").unwrap(); + let sources = vec![Source { + path: path.clone(), + name: "a.txt".into(), + }]; + let mut grown = false; + let result = create_7z( + &temp.path().join("out.7z"), + &sources, + &CreateOptions::default(), + &mut |p| { + if !grown && p.name == "a.txt" { + fs::write(&path, b"abcdef").unwrap(); + grown = true; + } + true + }, + ); + assert!(result.is_err()); + assert!(!temp.path().join("out.7z").exists()); +} diff --git a/arca-7z/vendor/PATCHES.md b/arca-7z/vendor/PATCHES.md index f726e324..bf5bf493 100644 --- a/arca-7z/vendor/PATCHES.md +++ b/arca-7z/vendor/PATCHES.md @@ -30,6 +30,9 @@ Local changes: encrypted headers without a verifiable nonempty stream. - Populate the format's default substream mapping when its optional section is absent; reject unused pack streams and inconsistent file/substream counts. +- Cap the threads of a multithreaded LZMA2 decoder so that one dictionary per + thread stays within four times the single-thread memory limit, since the + dictionary size comes from the archive. No new compression or cryptography is implemented. Upstream's existing bounded AES work factor (max power 24) and single-entry KDF cache are unchanged. Arca diff --git a/arca-7z/vendor/sevenz-rust2/src/decoder.rs b/arca-7z/vendor/sevenz-rust2/src/decoder.rs index 9f7217ce..3b5cf90f 100644 --- a/arca-7z/vendor/sevenz-rust2/src/decoder.rs +++ b/arca-7z/vendor/sevenz-rust2/src/decoder.rs @@ -120,6 +120,10 @@ pub fn add_decoder( }); } + // Each thread holds a dictionary of its own: an archive that names a + // large one gets fewer threads, so all of them together stay within + // four times the single-thread limit. + let threads = threads.min((4 * max_mem_limit_kb / mem_size.max(1)) as u32); let lz = if threads < 2 { Decoder::Lzma2(Box::new(Lzma2Reader::new(input, dic_size, None))) } else { diff --git a/arca-cli/src/main.rs b/arca-cli/src/main.rs index a3930ffc..2288b581 100644 --- a/arca-cli/src/main.rs +++ b/arca-cli/src/main.rs @@ -45,7 +45,7 @@ enum Cmd { short = 'j', long, default_value_t = 0, - help = "Threads to use. 0 means every core. .7z and .rar creation is sequential; .xz and .tar.xz use as many as fit in 2 GiB" + help = "Threads to use. 0 means every core. .xz and .tar.xz use as many as fit in 2 GiB; .7z and .rar always use every core their memory budget allows" )] threads: usize, #[arg( @@ -174,7 +174,7 @@ fn tar_source(archive: &Path, format_kind: Format) -> Result> { _ => { let f = BufReader::with_capacity(BUF, File::open(archive)?); if format_kind == Format::TarGz { - Box::new(flate2::read::GzDecoder::new(f)) + Box::new(flate2::read::MultiGzDecoder::new(f)) } else { Box::new(f) } @@ -192,10 +192,14 @@ fn conflict(policy: OnConflict) -> Conflict { // Built under a temporary name beside `out` and renamed only once complete, so // a failed or interrupted run never leaves a half-written archive behind. -fn staged(out: &Path, write: impl FnOnce(BufWriter) -> Result) -> Result { +fn staged_path(out: &Path) -> PathBuf { let mut name = out.file_name().unwrap_or_default().to_os_string(); name.push(".arca-new"); - let temp = out.with_file_name(name); + out.with_file_name(name) +} + +fn staged(out: &Path, write: impl FnOnce(BufWriter) -> Result) -> Result { + let temp = staged_path(out); let result = File::create(&temp) .map_err(Error::from) .and_then(|f| write(BufWriter::with_capacity(BUF, f))); @@ -378,7 +382,9 @@ fn create( } let codec = resolve_codec(if xz { CodecArg::Auto } else { codec_arg }, format_kind)?; let mut threads = resolve_threads(requested_threads); - let raw_list = collect_files(inputs)?; + let own = arca_core::extraction::OwnOutput::new(out); + let mut raw_list = collect_files(inputs)?; + raw_list.retain(|(path, _)| !own.is_own(path)); if raw_list.is_empty() { return Err(Error::Format("there is nothing to add".into())); } @@ -404,9 +410,15 @@ fn create( // Nothing to report while it runs -- the summary is printed at the end // -- so the answer to "carry on?" is always yes. Container::Rar | Container::Iso => return Err(format_kind.read_only()), - Container::Zip => arca_zip::create_zip(out, &files, threads, password, &|_, _, _| true)?, - Container::Tar | Container::TarGz => { - let f = BufWriter::with_capacity(BUF, File::create(out)?); + Container::Zip => { + let temp = staged_path(out); + arca_zip::create_zip(&temp, &files, threads, password, &|_, _, _| true)?; + if let Err(e) = fs::rename(&temp, out) { + let _ = fs::remove_file(&temp); + return Err(e.into()); + } + } + Container::Tar | Container::TarGz => staged(out, |f| { let dest: Box = if format_kind == Format::TarGz { Box::new(flate2::write::GzEncoder::new( f, @@ -417,20 +429,24 @@ fn create( }; let mut w = TarWriter::new(dest); for s in &files { - let entrada = BufReader::with_capacity(BUF, File::open(&s.path)?); - w.add(&s.name, s.size, s.mtime, 0o644, entrada)?; + let file = File::open(&s.path)?; + let mode = arca_tar::mode_of(&file.metadata()?); + let entrada = BufReader::with_capacity(BUF, file); + w.add(&s.name, s.size, s.mtime, mode, entrada)?; } - let mut d = w.finish()?; - d.flush()?; - } + w.finish()?.flush()?; + Ok(()) + })?, Container::TarXz => { threads = staged(out, |f| { let e = arca_xz::encoder(f, level, requested_threads)?; let used = e.workers(); let mut w = TarWriter::new(e); for s in &files { - let entrada = BufReader::with_capacity(BUF, File::open(&s.path)?); - w.add(&s.name, s.size, s.mtime, 0o644, entrada)?; + let file = File::open(&s.path)?; + let mode = arca_tar::mode_of(&file.metadata()?); + let entrada = BufReader::with_capacity(BUF, file); + w.add(&s.name, s.size, s.mtime, mode, entrada)?; } w.finish()?.finish()?.flush()?; Ok(used) @@ -463,7 +479,7 @@ fn create( 0.0 }; println!( - "{}: {} files, {} -> {} ({:.1}% smaller) in {:.3} s · {:.0} MB/s · {} threads", + "{}: {} files, {} -> {} ({:.1}% smaller) in {:.3} s · {:.0} MiB/s · {} threads", out.display(), files.len(), human(total), @@ -598,7 +614,7 @@ fn free_name(path: &Path, claimed: &HashSet) -> PathBuf { .unwrap_or_default(); for n in 1..10_000u32 { let candidate = dir.join(format!("{stem} ({n}){ext}")); - if !candidate.exists() && !claimed.contains(&candidate) { + if !candidate.exists() && !claimed.contains(&arca_core::extraction::claim_key(&candidate)) { return candidate; } } @@ -635,8 +651,8 @@ fn resolve_conflict( policy: OnConflict, claimed: &mut HashSet, ) -> Option { - let taken = - claimed.contains(&path) || (!matches!(policy, OnConflict::Overwrite) && path.exists()); + let taken = claimed.contains(&arca_core::extraction::claim_key(&path)) + || (!matches!(policy, OnConflict::Overwrite) && path.exists()); let chosen = if !taken { path } else { @@ -646,7 +662,7 @@ fn resolve_conflict( OnConflict::Rename => free_name(&path, claimed), } }; - claimed.insert(chosen.clone()); + claimed.insert(arca_core::extraction::claim_key(&chosen)); Some(chosen) } @@ -730,6 +746,19 @@ fn extract( }; jobs.push((e.clone(), path)); } + // Two entries with one name, overwriting: the later one is what + // extracting in order would leave, and writing both from two + // threads at once would leave a mix of the two. + if matches!(policy, OnConflict::Overwrite) { + let mut seen: HashSet = HashSet::new(); + let mut kept: Vec<(arca_core::Entry, PathBuf)> = jobs + .into_iter() + .rev() + .filter(|(_, path)| seen.insert(arca_core::extraction::claim_key(path))) + .collect(); + kept.reverse(); + jobs = kept; + } drop(a); let pool = rayon::ThreadPoolBuilder::new() @@ -744,8 +773,11 @@ fn extract( ExtractionReader::with_capacity(BUF, File::open(archive)?)?; let mut staging = Vec::new(); let mut total = 0u64; + // Nothing already on disk is touched until the entry has + // decoded: a small one is decoded into memory first, a + // large one or one replacing a file goes through a + // temporary that is renamed over it at the end. for (e, path) in batch { - let mut f = File::create(path)?; if e.size <= BUF as u64 { staging.clear(); let mut bounded = Bounded { @@ -758,14 +790,30 @@ fn extract( &mut bounded, password, )?; - f.write_all(&staging)?; - arca_core::extraction::set_mtime(&f, e.mtime); + let put = |f: &mut File| -> Result { + f.write_all(&staging)?; + arca_core::extraction::set_mtime(f, e.mtime); + Ok(0) + }; + if path.exists() { + arca_core::extraction::write_staged(path, put)?; + } else { + put(&mut File::create(path)?)?; + } } else { - let mut f = BufWriter::with_capacity(BUF, f); - total += - arca_zip::extract_entry_with(&mut source, e, &mut f, password)?; - f.flush()?; - arca_core::extraction::set_mtime(f.get_ref(), e.mtime); + total += arca_core::extraction::write_staged(path, |f| { + let mut w = BufWriter::with_capacity(BUF, &mut *f); + let n = arca_zip::extract_entry_with( + &mut source, + e, + &mut w, + password, + )?; + w.flush()?; + drop(w); + arca_core::extraction::set_mtime(f, e.mtime); + Ok(n) + })?; } } Ok(total) @@ -793,11 +841,18 @@ fn extract( let mut r = TarReader::new(tar_source(archive, format_kind)?); let mut destination = Destination::new(dest); while let Some(e) = r.next_entry()? { + // Links and devices have no data to write, and a file in their + // place would be an empty one where a link belongs. + if !e.is_plain() { + r.skip_data(&e)?; + continue; + } let written = r.with_data(&e, |data| { - destination.write_entry( + destination.write_file( &e.entry.name, e.entry.is_dir, e.entry.mtime, + Some(e.mode), data, &mut |_| conflict(policy), ) @@ -1013,7 +1068,7 @@ fn pass_fail(ok: bool) -> &'static str { } fn human(b: u64) -> String { - const U: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; + const U: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; let mut v = b as f64; let mut i = 0; while v >= 1024.0 && i < U.len() - 1 { diff --git a/arca-cli/src/rar.rs b/arca-cli/src/rar.rs index 6031d433..940fa926 100644 --- a/arca-cli/src/rar.rs +++ b/arca-cli/src/rar.rs @@ -57,20 +57,36 @@ pub(super) fn create( if cfg!(not(feature = "rar")) { return create_rar(out, &[], &CreateOptions::default(), &|_, _, _| true); } - let options = options(level, codec, requested_threads, password, hide_names)?; + let options = options(level, codec, password, hide_names)?; let cancelled = interruption()?; let walk = collect(out, inputs, Bounds::WRITER, &cancelled)?; let t0 = Instant::now(); - create_rar(out, &walk.sources, &options, &|_, _, _| { - !cancelled.load(Ordering::SeqCst) - })?; + let run = || { + create_rar(out, &walk.sources, &options, &|_, _, _| { + !cancelled.load(Ordering::SeqCst) + }) + }; + // The writer works on whatever rayon pool it runs in, so -j is a pool of + // that many threads; without it, every core. + let threads = if requested_threads > 0 { + rayon::ThreadPoolBuilder::new() + .num_threads(requested_threads) + .build() + .map_err(|e| Error::Format(format!("cannot start the thread pool: {e}")))? + .install(run)?; + requested_threads + } else { + run()?; + rayon::current_num_threads() + }; println!( - "{}: {} files, {} -> {} in {:.3} s (sequential)", + "{}: {} files, {} -> {} in {:.3} s \u{b7} {} threads", out.display(), walk.files, human(walk.bytes), human(fs::metadata(out)?.len()), - t0.elapsed().as_secs_f64() + t0.elapsed().as_secs_f64(), + threads ); Ok(()) } @@ -80,7 +96,6 @@ pub(super) fn create( fn options( level: Level, codec: CodecArg, - requested_threads: usize, password: Option<&str>, hide_names: bool, ) -> Result { @@ -94,11 +109,6 @@ fn options( ".rar creation does not support encryption; use .zip or .7z for a password".into(), )); } - if requested_threads > 1 { - return Err(Error::Unsupported( - ".rar creation is sequential; leave --threads at 0 or 1".into(), - )); - } let level = match codec { CodecArg::Auto => level, CodecArg::Store => Level::STORE, diff --git a/arca-cli/src/sevenz.rs b/arca-cli/src/sevenz.rs index 3fbc0ae9..bfca8a21 100644 --- a/arca-cli/src/sevenz.rs +++ b/arca-cli/src/sevenz.rs @@ -34,6 +34,7 @@ pub(super) fn create( password, hide_names, }; + let own = arca_core::extraction::OwnOutput::new(out); let mut sources = Vec::new(); let mut bytes = 0; let mut files = 0; @@ -41,6 +42,7 @@ pub(super) fn create( collect( input, input.parent().unwrap_or(Path::new("")), + &own, &mut sources, &mut bytes, &mut files, @@ -49,11 +51,16 @@ pub(super) fn create( let t0 = Instant::now(); create_7z(out, &sources, &options, &mut |_| true)?; println!( - "{}: {files} files, {} -> {} in {:.3} s (sequential)", + "{}: {files} files, {} -> {} in {:.3} s \u{b7} {} threads", out.display(), human(bytes), human(fs::metadata(out)?.len()), - t0.elapsed().as_secs_f64() + t0.elapsed().as_secs_f64(), + if options.level == Level::STORE { + 1 + } else { + arca_7z::workers(options.level) + } ); Ok(()) } @@ -61,11 +68,15 @@ pub(super) fn create( fn collect( path: &Path, base: &Path, + own: &arca_core::extraction::OwnOutput, sources: &mut Vec, bytes: &mut u64, files: &mut u64, ) -> Result<()> { let meta = fs::symlink_metadata(path)?; + if meta.is_file() && own.is_own(path) { + return Ok(()); + } if is_link(&meta) || !meta.is_file() && !meta.is_dir() { return Err(Error::Unsupported(format!( "7z source is not a regular file or directory: '{}'", @@ -93,7 +104,7 @@ fn collect( let mut children = fs::read_dir(path)?.collect::>>()?; children.sort_by_key(|entry| entry.file_name()); for child in children { - collect(&child.path(), base, sources, bytes, files)?; + collect(&child.path(), base, own, sources, bytes, files)?; } } Ok(()) diff --git a/arca-cli/tests/mtime.rs b/arca-cli/tests/mtime.rs index b71010f6..05beb5ca 100644 --- a/arca-cli/tests/mtime.rs +++ b/arca-cli/tests/mtime.rs @@ -55,3 +55,21 @@ fn extraction_restores_the_recorded_modification_time() { } } } + +#[test] +fn an_archive_made_inside_its_own_folder_leaves_itself_out() { + let room = tempfile::tempdir().unwrap(); + let dir = room.path(); + fs::create_dir_all(dir.join("src")).unwrap(); + fs::write(dir.join("src/a.txt"), b"contents").unwrap(); + for archive in ["pack.zip", "pack.tar", "pack.tar.gz", "pack.7z"] { + let inside = format!("src/{archive}"); + fs::write(dir.join(&inside), b"an older archive").unwrap(); + ok(dir, &["create", &inside, "src"]); + let out = dir.join(format!("out-{archive}")); + ok(dir, &["extract", &inside, "-o", out.to_str().unwrap()]); + assert!(out.join("src/a.txt").is_file(), "{archive}"); + assert!(!out.join(&inside).exists(), "{archive} took itself in"); + fs::remove_file(dir.join(&inside)).unwrap(); + } +} diff --git a/arca-cli/tests/rar.rs b/arca-cli/tests/rar.rs index 2a9d89ca..2a811c91 100644 --- a/arca-cli/tests/rar.rs +++ b/arca-cli/tests/rar.rs @@ -92,7 +92,7 @@ fn help_mentions_rar_creation() { let help = stdout(&result); assert!(help.contains(".rar"), "{help}"); assert!( - help.contains(".7z and .rar creation is sequential"), + help.contains(".7z and .rar always use every core"), "{help}" ); } @@ -243,8 +243,6 @@ fn unsupported_options_are_refused_before_any_output() { (vec!["-c", "deflate"], "--codec"), (vec!["--codec", "zstd"], "--codec"), (vec!["--codec", "lzma2"], "--codec"), - (vec!["-j", "2"], "--threads"), - (vec!["--threads", "8", "-l", "best"], "--threads"), ] { let mut full: Vec<&std::ffi::OsStr> = vec!["create".as_ref(), out.as_os_str(), input.as_os_str()]; diff --git a/arca-cli/tests/sevenz.rs b/arca-cli/tests/sevenz.rs index 8efaf5d4..a8aa5dfb 100644 --- a/arca-cli/tests/sevenz.rs +++ b/arca-cli/tests/sevenz.rs @@ -133,14 +133,14 @@ fn create_list_test_extract_all_levels_and_password_modes() { args.push("--hide-names"); } let summary = temp.ok(&args); - assert!(summary.contains("sequential")); + assert!(summary.contains("threads")); let archive = SevenZArchive::open( File::open(temp.0.join("archive.7z")).unwrap(), Some(password), ) .unwrap(); assert_eq!(archive.has_encrypted_header(), mode == "hidden"); - assert!(!archive.is_solid()); + assert!(archive.is_solid()); let entry = archive .entries() .iter() diff --git a/arca-cli/tests/zip_extract.rs b/arca-cli/tests/zip_extract.rs index 2ff616c2..0699d809 100644 --- a/arca-cli/tests/zip_extract.rs +++ b/arca-cli/tests/zip_extract.rs @@ -210,7 +210,7 @@ fn mixed_sizes_and_duplicate_names_share_one_reader_per_chunk() { assert_eq!(fs::read(output.path().join(name)).unwrap(), *body); } let dup = fs::read(output.path().join("dup.txt")).unwrap(); - assert!(dup == b"first" || dup == b"second"); + assert_eq!(dup, b"second"); let output = tempfile::tempdir().unwrap(); success(extract( &archive, @@ -247,3 +247,35 @@ fn an_entry_that_understates_its_size_fails_without_panicking() { assert!(!result.status.success()); assert!(!String::from_utf8_lossy(&result.stderr).contains("panicked")); } + +#[test] +fn a_failed_overwrite_keeps_the_files_it_would_have_replaced() { + let dir = tempfile::tempdir().unwrap(); + let archive = dir.path().join("locked.zip"); + let mut writer = ZipWriter::new(File::create(&archive).unwrap()); + let big = vec![b'x'; 2 << 20]; + for (name, body) in [("small.txt", &b"tiny"[..]), ("big.bin", &big[..])] { + writer + .add_with_password(name, body, Codec::Deflate, Level::FAST, None, Some("right")) + .unwrap(); + } + writer.finish().unwrap(); + let output = tempfile::tempdir().unwrap(); + for name in ["small.txt", "big.bin"] { + fs::write(output.path().join(name), b"mine").unwrap(); + } + let result = extract( + &archive, + output.path(), + &["--on-conflict", "overwrite", "-p", "wrong"], + ); + assert!(!result.status.success()); + for name in ["small.txt", "big.bin"] { + assert_eq!( + fs::read(output.path().join(name)).unwrap(), + b"mine", + "{name}" + ); + } + assert_eq!(fs::read_dir(output.path()).unwrap().count(), 2); +} diff --git a/arca-core/src/extraction.rs b/arca-core/src/extraction.rs index 9f645b91..69618d54 100644 --- a/arca-core/src/extraction.rs +++ b/arca-core/src/extraction.rs @@ -33,6 +33,8 @@ impl Destination { fn root(&mut self) -> Result<&Dir> { if self.directories.is_empty() { let absolute = std::path::absolute(&self.path)?; + #[cfg(unix)] + let absolute = as_the_disk_has_it(&absolute); let mut components = absolute.components().peekable(); let mut anchor = PathBuf::new(); while let Some(Component::Prefix(_) | Component::RootDir) = components.peek() { @@ -65,6 +67,20 @@ impl Destination { mtime: Option, reader: &mut dyn Read, ask: &mut dyn FnMut(&Path) -> Conflict, + ) -> Result> { + self.write_file(name, is_dir, mtime, None, reader, ask) + } + + /// `write_entry` for a format that records Unix permissions: the file + /// also gets the execute bits `mode` has, see [`set_executable`]. + pub fn write_file( + &mut self, + name: &str, + is_dir: bool, + mtime: Option, + mode: Option, + reader: &mut dyn Read, + ask: &mut dyn FnMut(&Path) -> Conflict, ) -> Result> { let relative = safe_name(name)?; let directory = if is_dir { @@ -87,7 +103,7 @@ impl Destination { let parent = directories.last().unwrap(); let mut name = relative.file_name().unwrap().to_os_string(); let exists = exists_without_link(parent, Path::new(&name))?; - if exists || self.claimed.contains(&relative) { + if exists || self.claimed.contains(&claim_key(&relative)) { match ask(&self.path.join(&relative)) { Conflict::Overwrite => {} Conflict::Skip => return Ok(None), @@ -98,13 +114,14 @@ impl Destination { } } let chosen = relative.with_file_name(&name); - self.claimed.insert(chosen); + self.claimed.insert(claim_key(&chosen)); let (mut staged, file) = StagedFile::new(parent, Path::new(&name))?; let mut writer = BufWriter::with_capacity(256 * 1024, file); let bytes = io::copy(reader, &mut writer)?; writer.flush()?; - let file = writer.into_inner().map_err(|e| e.into_error())?; - set_mtime(&file.into_std(), mtime); + let file = writer.into_inner().map_err(|e| e.into_error())?.into_std(); + set_mtime(&file, mtime); + set_executable(&file, mode); exists_without_link(parent, Path::new(&name))?; // Rename replaces the directory entry without following it, even if it changed. parent.rename(&staged.name, parent, &name)?; @@ -113,6 +130,86 @@ impl Destination { } } +/// The folder somebody asked to extract into, with the links on the way to it +/// followed: on macOS `/tmp` and `/var` are links, and on any Unix a home or +/// a mount point may be. The folder itself is not resolved, so one that is a +/// link is still refused when it is opened, and everything under it is still +/// opened without following links. +#[cfg(unix)] +fn as_the_disk_has_it(path: &Path) -> PathBuf { + let Some(mut existing) = path.parent() else { + return path.to_path_buf(); + }; + loop { + if let Ok(real) = std::fs::canonicalize(existing) { + return real.join(path.strip_prefix(existing).unwrap_or(Path::new(""))); + } + match existing.parent() { + Some(parent) => existing = parent, + None => return path.to_path_buf(), + } + } +} + +/// The form two paths share when the file system would take them for one +/// file. Windows and macOS compare names without regard to case by default, so +/// an archive holding `A.txt` and `a.txt` names one file there, and a run that +/// keeps track of the names it handed out has to compare them this way. +pub fn claim_key(path: &Path) -> PathBuf { + if cfg!(any(windows, target_os = "macos")) { + PathBuf::from(path.to_string_lossy().to_lowercase()) + } else { + path.to_path_buf() + } +} + +/// Writes `target` through a temporary file beside it, renamed over `target` +/// only once `fill` has succeeded. A wrong password, a corrupt entry or a stop +/// halfway leaves whatever was at `target` before exactly as it was, and no +/// temporary behind. +pub fn write_staged( + target: &Path, + fill: impl FnOnce(&mut std::fs::File) -> Result, +) -> Result { + static NEXT: AtomicU64 = AtomicU64::new(0); + let dir = target.parent().unwrap_or(Path::new(".")); + let mut opened = None; + for _ in 0..10_000 { + let temp = dir.join(format!( + ".arca-{}-{}.tmp", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp) + { + Ok(f) => { + opened = Some((temp, f)); + break; + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.into()), + } + } + let Some((temp, mut file)) = opened else { + return Err(Error::Limit( + "cannot allocate an extraction temporary file".into(), + )); + }; + let result = fill(&mut file); + drop(file); + let result = result.and_then(|n| { + std::fs::rename(&temp, target)?; + Ok(n) + }); + if result.is_err() { + let _ = std::fs::remove_file(&temp); + } + result +} + /// Gives an extracted file the modification time its archive recorded. /// /// Best effort: a file system that refuses the time still has every byte, and @@ -126,6 +223,28 @@ pub fn set_mtime(file: &std::fs::File, mtime: Option) { } } +/// Lets an extracted program run where the archive says it could: the execute +/// bits of `mode`, the Unix permissions the archive recorded, are added to +/// the file's. Nothing is ever taken away and nothing else is added, so the +/// most an archive can do is mark one of its own files executable. A no-op +/// where there are no such bits, and best effort like `set_mtime`. +pub fn set_executable(file: &std::fs::File, mode: Option) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let Some(execute) = mode.map(|m| m & 0o111).filter(|m| *m != 0) else { + return; + }; + if let Ok(meta) = file.metadata() { + let mut permissions = meta.permissions(); + permissions.set_mode(permissions.mode() | execute); + let _ = file.set_permissions(permissions); + } + } + #[cfg(not(unix))] + let _ = (file, mode); +} + fn open_child(parent: &Dir, name: &Path) -> Result { let dir = match parent.open_dir_nofollow(name) { Ok(dir) => dir, @@ -180,7 +299,9 @@ fn free_name(parent: &Dir, relative: &Path, claimed: &HashSet) -> Resul for n in 1..10_000 { let name = OsString::from(format!("{stem} ({n}){ext}")); let candidate = relative.with_file_name(&name); - if !claimed.contains(&candidate) && !exists_without_link(parent, Path::new(&name))? { + if !claimed.contains(&claim_key(&candidate)) + && !exists_without_link(parent, Path::new(&name))? + { return Ok(name); } } @@ -234,15 +355,97 @@ impl Drop for StagedFile<'_> { } } +/// The files a new archive must not take in: itself, when it is written into +/// a folder being archived, and the temporary it is built under. Without this +/// an archive replacing an older one swallows the older one, or reads itself +/// while it grows. +pub struct OwnOutput { + names: Vec, + dir: Option, +} + +impl OwnOutput { + pub fn new(out: &Path) -> Self { + let name = out.file_name().unwrap_or_default().to_os_string(); + let mut staged = name.clone(); + staged.push(".arca-new"); + let parent = match out.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + }; + Self { + names: vec![name, staged], + dir: std::fs::canonicalize(parent).ok(), + } + } + + pub fn is_own(&self, path: &Path) -> bool { + let Some(name) = path.file_name() else { + return false; + }; + let key = claim_key(Path::new(name)); + if !self.names.iter().any(|n| claim_key(Path::new(n)) == key) { + return false; + } + let parent = match path.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + }; + match (&self.dir, std::fs::canonicalize(parent)) { + (Some(dir), Ok(found)) => *dir == found, + _ => false, + } + } +} + #[cfg(test)] mod tests { use super::*; + + #[test] + fn an_archive_written_into_its_own_input_does_not_take_itself_in() { + let room = room(); + let out = room.path().join("sub").join("pack.zip"); + fs::create_dir_all(out.parent().unwrap()).unwrap(); + let own = OwnOutput::new(&out); + assert!(own.is_own(&room.path().join("sub/./pack.zip"))); + assert!(own.is_own(&room.path().join("sub/pack.zip.arca-new"))); + assert!(!own.is_own(&room.path().join("pack.zip"))); + assert!(!own.is_own(&room.path().join("sub/other.zip"))); + } use std::fs; fn room() -> tempfile::TempDir { tempfile::tempdir_in(fs::canonicalize(std::env::temp_dir()).unwrap()).unwrap() } + #[test] + fn a_staged_write_replaces_the_target_only_when_it_succeeds() { + let room = room(); + let target = room.path().join("keep.txt"); + fs::write(&target, b"original").unwrap(); + let failed = write_staged(&target, |f| { + f.write_all(b"half")?; + Err(Error::Format("wrong password".into())) + }); + assert!(failed.is_err()); + assert_eq!(fs::read(&target).unwrap(), b"original"); + let written = write_staged(&target, |f| { + f.write_all(b"new")?; + Ok(3) + }); + assert_eq!(written.unwrap(), 3); + assert_eq!(fs::read(&target).unwrap(), b"new"); + let left: Vec<_> = fs::read_dir(room.path()).unwrap().collect(); + assert_eq!(left.len(), 1, "no temporary may be left behind"); + } + + #[test] + fn names_that_differ_only_in_case_are_one_claim_where_the_disk_says_so() { + let same = claim_key(Path::new("Dir/A.txt")) == claim_key(Path::new("dir/a.txt")); + assert_eq!(same, cfg!(any(windows, target_os = "macos"))); + } + #[test] fn unused_and_invalid_destinations_do_not_write() { let room = room(); @@ -333,6 +536,37 @@ mod tests { assert!(modified("undated") > when as u64); } + // A program in a tarball has to come out runnable, and a plain file must + // not come out runnable just for having passed through an archive. + #[cfg(unix)] + #[test] + fn programs_keep_their_execute_bits_and_nothing_more() { + use std::os::unix::fs::PermissionsExt; + let room = room(); + let mut dest = Destination::new(room.path()); + for (name, mode) in [ + ("program", Some(0o755)), + ("plain", Some(0o644)), + ("setuid", Some(0o4755)), + ("unknown", None), + ] { + dest.write_file(name, false, None, mode, &mut &b"x"[..], &mut |_| { + Conflict::Overwrite + }) + .unwrap(); + } + let mode = |name: &str| { + fs::metadata(room.path().join(name)) + .unwrap() + .permissions() + .mode() + }; + assert_eq!(mode("program") & 0o111, 0o111); + assert_eq!(mode("plain") & 0o111, 0); + assert_eq!(mode("unknown") & 0o111, 0); + assert_eq!(mode("setuid") & 0o7000, 0); + } + #[test] fn pinned_root_cannot_be_redirected_before_directory_creation() { let room = room(); @@ -485,4 +719,47 @@ mod tests { ); } } + + // The folder asked for may be reached through a link, as /tmp is on + // macOS: that link is followed, and the files land where it leads. + #[cfg(unix)] + #[test] + fn a_destination_reached_through_a_link_is_used() { + let room = room(); + let real = room.path().join("real"); + fs::create_dir(&real).unwrap(); + std::os::unix::fs::symlink(&real, room.path().join("link")).unwrap(); + let mut dest = Destination::new(&room.path().join("link").join("out")); + dest.write_entry( + "a.txt", + false, + None, + &mut &b"contents"[..], + &mut |_| unreachable!(), + ) + .unwrap(); + assert_eq!(fs::read(real.join("out/a.txt")).unwrap(), b"contents"); + } + + // Only the folders above the one asked for are followed: that one, if it + // is a link, is refused like any link under it. + #[cfg(unix)] + #[test] + fn a_destination_that_is_itself_a_link_is_refused() { + let room = room(); + let real = room.path().join("real"); + fs::create_dir(&real).unwrap(); + std::os::unix::fs::symlink(&real, room.path().join("link")).unwrap(); + let mut dest = Destination::new(&room.path().join("link")); + assert!(dest + .write_entry( + "a.txt", + false, + None, + &mut &b"contents"[..], + &mut |_| unreachable!(), + ) + .is_err()); + assert_eq!(fs::read_dir(&real).unwrap().count(), 0); + } } diff --git a/arca-core/src/lib.rs b/arca-core/src/lib.rs index 80a56967..bb2c6e8e 100644 --- a/arca-core/src/lib.rs +++ b/arca-core/src/lib.rs @@ -435,7 +435,15 @@ pub fn safe_name(name: &str) -> Result { "entry '{name}' carries a drive letter" ))); } - out.push(p); + // On NTFS `file:stream` writes into a hidden stream of `file`, + // which may be a file outside anything the archive created. + // The colon becomes `_`, as other Windows tools do, so a name + // like `10:30.log` from elsewhere still comes out. + if cfg!(windows) && p.contains(':') { + out.push(p.replace(':', "_")); + } else { + out.push(p); + } } } } @@ -469,6 +477,15 @@ mod tests { assert!(safe_name("..\\..\\windows\\system32").is_err()); assert!(safe_name("C:/tmp/x").is_err()); assert!(safe_name("a/b/c.txt").is_ok()); + assert!(safe_name("x:y/file").is_err()); + let stream = safe_name("dir/notes.txt:hidden").unwrap(); + let wanted = if cfg!(windows) { + "dir/notes.txt_hidden" + } else { + "dir/notes.txt:hidden" + }; + assert_eq!(stream, std::path::PathBuf::from(wanted)); + assert!(safe_name("logs/2024-01-01T10:30.log").is_ok()); assert_eq!( safe_name("./a//b.txt").unwrap(), std::path::PathBuf::from("a/b.txt") diff --git a/arca-gui/src/archive_ops/io.rs b/arca-gui/src/archive_ops/io.rs index f9047ff8..f740e845 100644 --- a/arca-gui/src/archive_ops/io.rs +++ b/arca-gui/src/archive_ops/io.rs @@ -50,38 +50,45 @@ impl Write for Watched<'_, W> { // Writes one entry to `path` while keeping `notify` in the loop, so a pause // holds and a stop lands inside the file rather than after it. A stop, like -// any other failure, takes the half-written file away with it. +// any other failure, takes the half-written file away with it, and a file that +// was already there and answered Replace stays untouched until the entry has +// come out whole. `stamp` is the time and, where the format keeps them, the +// Unix permissions to give it. fn unpack( path: &Path, - mtime: Option, + (mtime, mode): (Option, Option), notify: &(dyn Fn(usize, usize, &str) -> bool + Sync), progress: (usize, usize, &str), fill: impl FnOnce(&mut dyn Write) -> arca_core::Result, ) -> arca_core::Result { - let mut out = Watched { - inner: BufWriter::with_capacity(BUF, File::create(path)?), - notify, - progress, - since: 0, - cancelled: false, - }; - let result = fill(&mut out); - let Watched { - mut inner, - cancelled, - .. - } = out; - let result = if cancelled { - Err(arca_core::Error::Cancelled) - } else { - result.and_then(|n| { - inner.flush()?; - arca_core::extraction::set_mtime(inner.get_ref(), mtime); - Ok(n) - }) + let write = |file: &mut File| -> arca_core::Result { + let mut out = Watched { + inner: BufWriter::with_capacity(BUF, &mut *file), + notify, + progress, + since: 0, + cancelled: false, + }; + let result = fill(&mut out); + let Watched { + inner, cancelled, .. + } = out; + if cancelled { + return Err(arca_core::Error::Cancelled); + } + let n = result?; + let file = inner.into_inner().map_err(|e| e.into_error())?; + arca_core::extraction::set_mtime(file, mtime); + arca_core::extraction::set_executable(file, mode); + Ok(n) }; + if path.exists() { + return arca_core::extraction::write_staged(path, write); + } + let mut file = File::create(path)?; + let result = write(&mut file); if result.is_err() { - drop(inner); + drop(file); let _ = fs::remove_file(path); } result @@ -93,7 +100,7 @@ pub(crate) fn open_source(archive: &Path, format: Format) -> arca_core::Result Box::new(flate2::read::GzDecoder::new(f)), + Format::TarGz => Box::new(flate2::read::MultiGzDecoder::new(f)), _ => Box::new(f), }) } @@ -226,7 +233,7 @@ pub(crate) fn check_access( let a = arca_rar::RarArchive::open_with_progress(archive, password, notify)?; if a.entries().iter().any(|e| e.encrypted) { let password = password.ok_or(arca_core::Error::PasswordRequired)?; - a.test(Some(password), notify) + a.check_password(password, notify) } else { Ok(()) } @@ -448,9 +455,13 @@ pub(crate) fn extract_one_into( fs::create_dir_all(parent)?; } let mut source = BufReader::with_capacity(BUF, File::open(archive)?); - unpack(&path, entry.mtime, notify, (0, 1, &entry.name), |out| { - arca_zip::extract_entry_with(&mut source, entry, out, password) - })?; + unpack( + &path, + (entry.mtime, None), + notify, + (0, 1, &entry.name), + |out| arca_zip::extract_entry_with(&mut source, entry, out, password), + )?; Ok(path) } Container::Tar | Container::TarGz | Container::TarXz => { @@ -462,10 +473,10 @@ pub(crate) fn extract_one_into( // the ones before it. let mut r = TarReader::new(open_source(archive, format)?); while let Some(e) = r.next_entry()? { - if e.entry.name == entry.name && !e.entry.is_dir { + if e.entry.name == entry.name && !e.entry.is_dir && e.is_plain() { unpack( &path, - entry.mtime, + (entry.mtime, Some(e.mode)), notify, (0, 1, &entry.name), |mut out| r.copy_data(&e, &mut out), @@ -569,7 +580,7 @@ pub(crate) fn free_name(path: &Path, claimed: &HashSet) -> PathBuf { .unwrap_or_default(); for n in 1..10_000u32 { let candidate = dir.join(format!("{stem} ({n}){ext}")); - if !candidate.exists() && !claimed.contains(&candidate) { + if !candidate.exists() && !claimed.contains(&arca_core::extraction::claim_key(&candidate)) { return candidate; } } @@ -592,8 +603,8 @@ pub(crate) fn dest_path( if let Some(p) = path.parent() { fs::create_dir_all(p)?; } - if !path.exists() && !claimed.contains(&path) { - claimed.insert(path.clone()); + if !path.exists() && !claimed.contains(&arca_core::extraction::claim_key(&path)) { + claimed.insert(arca_core::extraction::claim_key(&path)); return Ok(Some(path)); } let chosen = match ask(&path) { @@ -610,7 +621,7 @@ pub(crate) fn dest_path( } Answer::Cancel => return Err(arca_core::Error::Cancelled), }; - claimed.insert(chosen.clone()); + claimed.insert(arca_core::extraction::claim_key(&chosen)); Ok(Some(chosen)) } @@ -763,6 +774,15 @@ pub(crate) fn extract( } } drop(a); + // Two entries of one name both answered Replace: the later one is + // what extracting in order leaves, and writing both from two + // threads at once would leave a mix of the two. + let mut seen = HashSet::new(); + jobs.reverse(); + jobs.retain(|(e, path)| { + e.is_dir || seen.insert(arca_core::extraction::claim_key(path)) + }); + jobs.reverse(); let total = jobs.len(); let done = AtomicUsize::new(0); @@ -771,7 +791,7 @@ pub(crate) fn extract( .map(|(e, path)| { let mut source = BufReader::with_capacity(BUF, File::open(archive)?); let at = (done.load(Ordering::Relaxed), total, e.name.as_str()); - let w = unpack(path, e.mtime, notify, at, |out| { + let w = unpack(path, (e.mtime, None), notify, at, |out| { arca_zip::extract_entry_with(&mut source, e, out, password) })?; if !notify(done.fetch_add(1, Ordering::Relaxed) + 1, total, &e.name) { @@ -804,7 +824,7 @@ pub(crate) fn extract( if !notify(i, total, &e.entry.name) { return Err(arca_core::Error::Cancelled); } - if !wanted.is_empty() && !wanted.get(i).copied().unwrap_or(true) { + if !e.is_plain() || !wanted.is_empty() && !wanted.get(i).copied().unwrap_or(true) { r.skip_data(&e)?; i += 1; continue; @@ -812,7 +832,7 @@ pub(crate) fn extract( match dest_path(dest, &e.entry.name, e.entry.is_dir, ask, &mut claimed)? { Some(path) => { let at = (i, total, e.entry.name.as_str()); - bytes += unpack(&path, e.entry.mtime, notify, at, |out| { + bytes += unpack(&path, (e.entry.mtime, Some(e.mode)), notify, at, |out| { r.with_data(&e, |data| Ok(std::io::copy(data, out)?)) })?; } @@ -1183,7 +1203,9 @@ pub(crate) fn compress( let final_size = fs::metadata(out).map(|m| m.len()).unwrap_or(0); return Ok((source_bytes, final_size)); } - let files = collect_sources(inputs, format == Format::SevenZ)?; + let own = arca_core::extraction::OwnOutput::new(out); + let mut files = collect_sources(inputs, format == Format::SevenZ)?; + files.retain(|(path, name)| name.ends_with('/') || !own.is_own(path)); if format == Format::Xz && !(inputs.len() == 1 && files.len() == 1 && inputs[0].is_file()) { return Err(arca_core::Error::Unsupported( "an .xz holds exactly one file; make a .tar.xz to keep several".into(), @@ -1231,29 +1253,52 @@ pub(crate) fn compress( level, }); } - arca_zip::create_zip(out, &sources, 0, password, notify)?; + let temp = staged_path(out); + arca_zip::create_zip(&temp, &sources, 0, password, notify)?; + if let Err(e) = fs::rename(&temp, out) { + let _ = fs::remove_file(&temp); + return Err(e.into()); + } } Container::Tar | Container::TarGz => { - let raw = BufWriter::with_capacity(BUF, File::create(out)?); - let sink: Box = if format == Format::TarGz { - Box::new(flate2::write::GzEncoder::new( - raw, - flate2::Compression::new(level.to_flate2()), - )) - } else { - Box::new(raw) - }; - let mut w = TarWriter::new(sink); - for (i, (path, name)) in files.iter().enumerate() { - if !notify(i, total, name) { - return Err(arca_core::Error::Cancelled); + fn tar_into( + sink: W, + files: &[(PathBuf, String)], + notify: &(dyn Fn(usize, usize, &str) -> bool + Sync), + bytes: &mut u64, + ) -> arca_core::Result { + let mut w = TarWriter::new(sink); + for (i, (path, name)) in files.iter().enumerate() { + if !notify(i, files.len(), name) { + return Err(arca_core::Error::Cancelled); + } + let meta = fs::metadata(path)?; + let f = BufReader::with_capacity(BUF, File::open(path)?); + w.add( + name, + meta.len(), + mtime_of(&meta), + arca_tar::mode_of(&meta), + f, + )?; + *bytes += meta.len(); } - let meta = fs::metadata(path)?; - let f = BufReader::with_capacity(BUF, File::open(path)?); - w.add(name, meta.len(), 0, 0o644, f)?; - source_bytes += meta.len(); + w.finish() } - w.finish()?; + staged(out, |raw| { + if format == Format::TarGz { + let gz = flate2::write::GzEncoder::new( + raw, + flate2::Compression::new(level.to_flate2()), + ); + tar_into(gz, &files, notify, &mut source_bytes)? + .finish()? + .flush()?; + } else { + tar_into(raw, &files, notify, &mut source_bytes)?.flush()?; + } + Ok(()) + })?; } Container::TarXz => staged(out, |f| { let mut w = TarWriter::new(arca_xz::encoder(f, level, 0)?); @@ -1267,7 +1312,13 @@ pub(crate) fn compress( name, notify, ); - let result = w.add(name, meta.len(), mtime_of(&meta), 0o644, &mut f); + let result = w.add( + name, + meta.len(), + mtime_of(&meta), + arca_tar::mode_of(&meta), + &mut f, + ); f.check(result)?; source_bytes += meta.len(); } diff --git a/arca-gui/src/archive_ops/sevenz_tests.rs b/arca-gui/src/archive_ops/sevenz_tests.rs index bb6e1b8f..587780fe 100644 --- a/arca-gui/src/archive_ops/sevenz_tests.rs +++ b/arca-gui/src/archive_ops/sevenz_tests.rs @@ -502,7 +502,7 @@ fn exhausted_extraction_rename_does_not_fall_back_to_overwrite() { let original = room.path("a.txt"); fs::write(&original, b"original").unwrap(); let mut claimed = (1..10_000) - .map(|n| room.path(&format!("a ({n}).txt"))) + .map(|n| arca_core::extraction::claim_key(&room.path(&format!("a ({n}).txt")))) .collect(); assert!(matches!( dest_path(&room.0, "a.txt", false, &|_| Answer::Rename, &mut claimed), diff --git a/arca-gui/src/archive_ops/xz_tests.rs b/arca-gui/src/archive_ops/xz_tests.rs index 54eb8c10..9c15b03e 100644 --- a/arca-gui/src/archive_ops/xz_tests.rs +++ b/arca-gui/src/archive_ops/xz_tests.rs @@ -255,3 +255,120 @@ fn a_stop_during_a_small_xz_job_publishes_nothing() { )); assert!(leftovers(&room.0).is_empty()); } + +// Two ZIP entries of one name, both replaced: the later one wins, as it would +// extracting in order, and never a mix of the two written at once. +#[test] +fn the_later_of_two_same_named_zip_entries_is_the_one_extracted() { + let room = Room::new(); + let archive = room.path("dup.zip"); + let mut zip = arca_zip::ZipWriter::new(File::create(&archive).unwrap()); + for body in [&b"first"[..], &b"second, which is longer"[..]] { + zip.add("dup.txt", body, Codec::Store, Level::STORE, None) + .unwrap(); + } + zip.finish().unwrap(); + for run in 0..20 { + let dest = room.path(&format!("out{run}")); + extract(&archive, &dest, &[], GO, &replace, None).unwrap(); + assert_eq!( + fs::read(dest.join("dup.txt")).unwrap(), + b"second, which is longer" + ); + } +} + +// Replacing a file with an entry that then fails to come out, here for a wrong +// password, leaves the file that was there as it was. +#[test] +fn a_failed_replace_keeps_the_file_it_would_have_replaced() { + let room = Room::new(); + let archive = room.path("locked.zip"); + let mut zip = arca_zip::ZipWriter::new(File::create(&archive).unwrap()); + zip.add_with_password( + "keep.txt", + &b"from the archive"[..], + Codec::Deflate, + Level::FAST, + None, + Some("right"), + ) + .unwrap(); + zip.finish().unwrap(); + let dest = room.path("out"); + fs::create_dir_all(&dest).unwrap(); + fs::write(dest.join("keep.txt"), b"mine").unwrap(); + assert!(extract(&archive, &dest, &[], GO, &replace, Some("wrong")).is_err()); + assert_eq!(fs::read(dest.join("keep.txt")).unwrap(), b"mine"); + assert_eq!(fs::read_dir(&dest).unwrap().count(), 1); +} + +// Two names that differ only in case are one file on Windows and macOS, and +// must not be written by two threads at once there. +#[test] +fn zip_entries_that_differ_only_in_case_are_one_file_where_the_disk_says_so() { + let room = Room::new(); + let archive = room.path("case.zip"); + let mut zip = arca_zip::ZipWriter::new(File::create(&archive).unwrap()); + for (name, body) in [ + ("Readme.txt", &b"upper"[..]), + ("readme.txt", &b"lower, later"[..]), + ] { + zip.add(name, body, Codec::Store, Level::STORE, None) + .unwrap(); + } + zip.finish().unwrap(); + let dest = room.path("out"); + extract(&archive, &dest, &[], GO, &replace, None).unwrap(); + if cfg!(any(windows, target_os = "macos")) { + assert_eq!(fs::read_dir(&dest).unwrap().count(), 1); + assert_eq!(fs::read(dest.join("readme.txt")).unwrap(), b"lower, later"); + } else { + assert_eq!(fs::read_dir(&dest).unwrap().count(), 2); + } +} + +// An archive written into the folder it is made from, over an older copy of +// itself, leaves both the older copy and itself out. +#[test] +fn an_archive_made_inside_its_own_folder_leaves_itself_out() { + let room = Room::new(); + let source = sample(&room); + for format in [Format::Zip, Format::Tar, Format::TarGz, Format::SevenZ] { + let out = source.join(format!("pack.{}", format.extension())); + fs::write(&out, b"an older archive").unwrap(); + make(&out, std::slice::from_ref(&source), format).unwrap(); + let (entries, _) = list_entries(&out, None, GO).unwrap(); + assert!( + entries.iter().all(|e| !e.name.contains("pack.")), + "{format:?}: {:?}", + entries.iter().map(|e| &e.name).collect::>() + ); + fs::remove_file(&out).unwrap(); + } +} + +// Stopping halfway through an archive that replaces another leaves the other +// one where it was, for the formats that used to write in place. +#[test] +fn a_stopped_replacement_keeps_the_archive_it_was_replacing() { + let room = Room::new(); + let source = sample(&room); + let stop: &(dyn Fn(usize, usize, &str) -> bool + Sync) = &|_, _, _| false; + for format in [Format::Zip, Format::Tar, Format::TarGz] { + let out = room.path(&format!("old.{}", format.extension())); + fs::write(&out, b"keep me").unwrap(); + let r = compress( + &out, + std::slice::from_ref(&source), + format, + Codec::Deflate, + Level::FAST, + stop, + (None, false), + ); + assert!(r.is_err(), "{format:?}"); + assert_eq!(fs::read(&out).unwrap(), b"keep me", "{format:?}"); + assert!(leftovers(&room.0).is_empty(), "{format:?}"); + } +} diff --git a/arca-gui/src/clipboard.rs b/arca-gui/src/clipboard.rs index 57c960f5..32ab09de 100644 --- a/arca-gui/src/clipboard.rs +++ b/arca-gui/src/clipboard.rs @@ -132,6 +132,18 @@ pub fn clear() { } } +/// A path out of a text/uri-list without the carriage return that ends its +/// line. The list is CRLF by its RFC, and Nautilus and Dolphin write it so, +/// but arboard splits it at the line feed alone and leaves the return on the +/// path, which then names a file that does not exist. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn without_line_end(path: PathBuf) -> PathBuf { + match path.to_str() { + Some(text) if text.ends_with('\r') => PathBuf::from(text.trim_end_matches('\r')), + _ => path, + } +} + #[cfg(target_os = "linux")] mod linux { use super::PathBuf; @@ -172,7 +184,11 @@ mod linux { } pub fn files() -> Vec { - with(|clipboard| clipboard.get().file_list().unwrap_or_default()).unwrap_or_default() + with(|clipboard| clipboard.get().file_list().unwrap_or_default()) + .unwrap_or_default() + .into_iter() + .map(super::without_line_end) + .collect() } pub fn taken() -> (Vec, bool) { @@ -247,7 +263,7 @@ pub fn clear() {} #[cfg(test)] mod tests { - use super::shell_path; + use super::{shell_path, without_line_end}; use std::path::PathBuf; #[test] @@ -264,4 +280,16 @@ mod tests { assert_eq!(plain(r"C:\Temp\a.txt"), PathBuf::from(r"C:\Temp\a.txt")); assert_eq!(plain("/tmp/a.txt"), PathBuf::from("/tmp/a.txt")); } + + #[test] + fn a_uri_list_written_with_crlf_still_names_the_files() { + assert_eq!( + without_line_end(PathBuf::from("/home/me/a b.txt\r")), + PathBuf::from("/home/me/a b.txt") + ); + assert_eq!( + without_line_end(PathBuf::from("/home/me/plain.txt")), + PathBuf::from("/home/me/plain.txt") + ); + } } diff --git a/arca-gui/src/controller/disk.rs b/arca-gui/src/controller/disk.rs index 1e831540..6039e031 100644 --- a/arca-gui/src/controller/disk.rs +++ b/arca-gui/src/controller/disk.rs @@ -160,7 +160,7 @@ fn transfer_trees( } } } - claimed.insert(to.clone()); + claimed.insert(arca_core::extraction::claim_key(&to)); if copy { copy_tree(from, &to).map_err(|(path, e)| could_not(s, true, &path, &e))?; } else if fs::rename(from, &to).is_err() { @@ -458,6 +458,47 @@ fn mounted_volumes() -> Vec { /// The outcome a device job in the background leaves for the window. pub(crate) type DeviceJob = Arc>>>; +/// What the sidebar shows of the machine, read on a thread of its own: a +/// drive letter that names a network share nobody is answering for blocks +/// the call that looks at it for as long as the network takes to give up, +/// and that call has no business on the thread that draws the window. +#[derive(Clone, Debug, Default)] +pub(crate) struct Survey { + pub(crate) home: Option, + pub(crate) folders: Vec<(PlaceKind, PathBuf)>, + pub(crate) drives: Vec, +} + +pub(crate) type SurveyJob = Arc)>>>; + +pub(crate) fn survey() -> Survey { + let mut out = Survey { + home: home_dir(), + ..Survey::default() + }; + if let Some(home) = &out.home { + let known = user_dirs(home); + for (kind, name) in [ + (PlaceKind::Desktop, "Desktop"), + (PlaceKind::Documents, "Documents"), + (PlaceKind::Downloads, "Downloads"), + (PlaceKind::Pictures, "Pictures"), + (PlaceKind::Music, "Music"), + (PlaceKind::Videos, "Videos"), + ] { + let path = known.get(name).cloned().unwrap_or_else(|| home.join(name)); + if path.is_dir() && path != *home { + out.folders.push((kind, path)); + } + } + } + #[cfg(windows)] + { + out.drives = drives(); + } + out +} + /// A storage volume the sidebar can offer: a partition with a file system /// that is not part of the running system. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -757,7 +798,10 @@ impl AppController { pub(crate) fn rename_on_disk(&mut self, from: &str, to: &str) { let s = self.s(); let (a, b) = (disk_path(from), disk_path(to)); - if b.exists() { + // On a disk that ignores case, `b` exists when only the case of the + // name is changing, and that one it is: renaming it is the point. + let same = arca_core::extraction::claim_key(&a) == arca_core::extraction::claim_key(&b); + if b.exists() && !same { let name = b .file_name() .map(|n| n.to_string_lossy().into_owned()) @@ -1057,7 +1101,14 @@ impl AppController { pub(crate) fn folder_total(&self) -> usize { if self.on_disk() { - children_of(&self.state.entries, &self.state.active_pane().directory).len() + self.rows_for( + &self.state.active_pane().directory, + "", + (SortColumn::Name, true), + false, + false, + ) + .len() } else { self.state.entries.len() } @@ -1107,6 +1158,7 @@ impl AppController { self.state.archive = None; self.state.origin = None; self.state.entries.clear(); + self.entries_changed(); self.state.active_pane_mut().checked.clear(); self.state.active_pane_mut().cursor = None; self.state.folders = tree::folders_of(&[]); @@ -1175,7 +1227,10 @@ impl AppController { continue; } match list_directory(&ancestor, show_hidden) { - Ok(entries) => self.state.entries.extend(entries), + Ok(entries) => { + self.state.entries.extend(entries); + self.entries_changed(); + } Err(error) => { let name = ancestor .trim_end_matches('/') @@ -1222,6 +1277,7 @@ impl AppController { }) .collect(); self.state.entries.clear(); + self.entries_changed(); self.state.active_pane_mut().checked.clear(); self.state.disk = Some(DiskState::default()); self.load_disk_ancestors(&dir); @@ -1344,30 +1400,27 @@ impl AppController { pub(crate) fn places(&self) -> Vec { let s = self.s(); let mut out = Vec::new(); - if let Some(home) = home_dir() { + if let Some(home) = &self.survey.home { out.push(Place { kind: PlaceKind::Home, label: s.home_word.to_string(), path: home.clone(), }); - let known = user_dirs(&home); - for (kind, name, label) in [ - (PlaceKind::Desktop, "Desktop", s.desktop_word), - (PlaceKind::Documents, "Documents", s.documents_word), - (PlaceKind::Downloads, "Downloads", s.downloads_word), - (PlaceKind::Pictures, "Pictures", s.pictures_word), - (PlaceKind::Music, "Music", s.music_word), - (PlaceKind::Videos, "Videos", s.videos_word), - ] { - let path = known.get(name).cloned().unwrap_or_else(|| home.join(name)); - if path.is_dir() && path != home { - out.push(Place { - kind, - label: label.to_string(), - path, - }); - } - } + } + for (kind, path) in &self.survey.folders { + let label = match kind { + PlaceKind::Desktop => s.desktop_word, + PlaceKind::Documents => s.documents_word, + PlaceKind::Downloads => s.downloads_word, + PlaceKind::Pictures => s.pictures_word, + PlaceKind::Music => s.music_word, + _ => s.videos_word, + }; + out.push(Place { + kind: *kind, + label: label.to_string(), + path: path.clone(), + }); } out } @@ -1392,15 +1445,33 @@ impl AppController { .collect() } - /// Re-reads the volumes every few seconds and collects what a format - /// job in the background has to say. - pub(crate) fn refresh_volumes(&mut self) { + /// Re-reads places, drives and volumes every few seconds on a thread, and + /// collects what a format job in the background has to say. Answers + /// whether a reading is still on its way, so the window knows to look + /// again soon. + pub(crate) fn refresh_volumes(&mut self) -> bool { + let read = self + .survey_job + .as_ref() + .and_then(|job| job.lock().ok().and_then(|mut slot| slot.take())); + if let Some((survey, volumes)) = read { + self.survey_job = None; + self.survey = survey; + self.volumes = volumes; + } let due = self .volumes_at .is_none_or(|at| at.elapsed() >= std::time::Duration::from_secs(3)); - if due { - self.volumes = volumes(); + if due && self.survey_job.is_none() { + let slot: SurveyJob = Arc::new(Mutex::new(None)); + self.survey_job = Some(slot.clone()); self.volumes_at = Some(std::time::Instant::now()); + std::thread::spawn(move || { + let read = (survey(), volumes()); + if let Ok(mut slot) = slot.lock() { + *slot = Some(read); + } + }); } let finished = self .volume_job @@ -1411,6 +1482,7 @@ impl AppController { self.volumes_at = None; self.report_device(outcome); } + self.survey_job.is_some() } pub(crate) fn volume_at(&self, path: &Path) -> Option<&Volume> { @@ -1503,12 +1575,11 @@ impl AppController { label: s.computer_word.to_string(), path: PathBuf::new(), }); - #[cfg(windows)] - for root in drives() { + for root in &self.survey.drives { out.push(Place { kind: PlaceKind::Drive, label: root.to_string_lossy().trim_end_matches('\\').to_string(), - path: root, + path: root.clone(), }); } } else { @@ -1594,7 +1665,18 @@ pub(crate) fn duplicate_target(path: &Path) -> Option { /// Moves a file or folder to the Trash the way the desktop does it. #[cfg(all(unix, not(target_os = "macos")))] pub(crate) fn trash_item(path: &Path) -> Result<(), String> { - let path = fs::canonicalize(path).map_err(|e| e.to_string())?; + // The folder is made absolute, the item itself is not: a link is trashed + // as the link, never as whatever it points at. + let leaf_name = path + .file_name() + .ok_or_else(|| "nothing to trash".to_string())?; + let parent = match path.parent() { + Some(p) if !p.as_os_str().is_empty() => p, + _ => Path::new("."), + }; + let path = fs::canonicalize(parent) + .map_err(|e| e.to_string())? + .join(leaf_name); let data = std::env::var_os("XDG_DATA_HOME") .map(PathBuf::from) .filter(|p| p.is_absolute()) diff --git a/arca-gui/src/controller/disk_tests.rs b/arca-gui/src/controller/disk_tests.rs index d533a92f..dfd926a1 100644 --- a/arca-gui/src/controller/disk_tests.rs +++ b/arca-gui/src/controller/disk_tests.rs @@ -504,3 +504,50 @@ fn dragging_out_hands_over_each_dragged_row_as_itself() { std::fs::canonicalize(room.path("nested")).unwrap() ); } + +// The sidebar reads the machine on a thread: asking for places never waits on +// the disk, and the answer arrives a little later. +#[test] +fn places_and_drives_are_read_off_the_drawing_thread() { + let mut app = AppController::new(Settings::default()); + assert!(app.places().is_empty()); + let started = std::time::Instant::now(); + assert!( + app.refresh_volumes(), + "the first reading starts on its own thread" + ); + assert!(started.elapsed() < std::time::Duration::from_millis(50)); + while app.refresh_volumes() { + assert!(started.elapsed() < std::time::Duration::from_secs(30)); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + if super::disk::home_dir().is_some() { + assert_eq!(app.places()[0].kind, super::disk::PlaceKind::Home); + } + assert!( + !app.refresh_volumes(), + "nothing is read again before it is due" + ); +} + +// Changing only the case of a name is a rename like any other, though on a +// disk that ignores case the new name already "exists". +#[test] +fn a_name_can_change_only_its_case() { + let room = Room::new(); + std::fs::write(room.path("note.txt"), b"x").unwrap(); + let mut app = AppController::new(Settings::default()); + let dir = super::disk::disk_dir(&room.path("")); + let dir = if dir.ends_with('/') { + dir + } else { + format!("{dir}/") + }; + app.rename_on_disk(&format!("{dir}note.txt"), &format!("{dir}Note.txt")); + assert!(!app.state.error, "{}", app.state.notice); + let names: Vec = std::fs::read_dir(room.path("")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!(names, ["Note.txt"]); +} diff --git a/arca-gui/src/controller/jobs.rs b/arca-gui/src/controller/jobs.rs index 1f1953d2..159f2530 100644 --- a/arca-gui/src/controller/jobs.rs +++ b/arca-gui/src/controller/jobs.rs @@ -61,6 +61,9 @@ pub(crate) struct Task { pub(crate) reread_after: Option<(PathBuf, Option, String)>, pub(crate) close_when_done: bool, pub(crate) quiet: bool, + // The archive this task rebuilds and the word for the change, offered to + // be taken back once the task has finished, and only if it worked. + pub(crate) undo: Option<(PathBuf, &'static str)>, work: Option, } @@ -93,6 +96,7 @@ impl Task { reread_after: None, close_when_done: false, quiet: false, + undo: None, work: Some(work), } } @@ -322,6 +326,14 @@ impl AppController { }; task.stop.store(true, Ordering::Relaxed); task.hold.store(false, Ordering::Relaxed); + // A task waiting on an answer about a conflict is not looking at the + // stop flag; the answer it is waiting for is the way to stop it. + if task.conflict.take().is_some() { + task.conflict_from = None; + if let Some(replies) = &task.replies { + let _ = replies.send(Answer::Cancel); + } + } if task.status == TaskStatus::Queued { task.status = TaskStatus::Stopped; task.result = stopped; @@ -476,6 +488,10 @@ impl AppController { close_window = true; } } + let landed = task.status == TaskStatus::Done; + if let Some(undo) = task.undo.take() { + self.keep_undo(undo, landed); + } } } if let Some((armed_by, _)) = &self.state.cut_armed { diff --git a/arca-gui/src/controller/jobs_tests.rs b/arca-gui/src/controller/jobs_tests.rs index 3f2c29f7..d9020e19 100644 --- a/arca-gui/src/controller/jobs_tests.rs +++ b/arca-gui/src/controller/jobs_tests.rs @@ -555,6 +555,68 @@ fn leaving_for_the_disk_forgets_a_cut_out_of_the_archive() { assert!(app.state.cut_names.is_empty()); } +// The window wrote every .tar and .tar.gz entry dated 1970. +#[test] +fn a_tar_made_in_the_window_keeps_the_dates_of_its_files() { + let room = Room::new(); + let source = room.path("dated"); + std::fs::create_dir_all(&source).unwrap(); + let file = source.join("a.txt"); + std::fs::write(&file, b"a").unwrap(); + let when = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_600_000_000); + std::fs::File::options() + .write(true) + .open(&file) + .unwrap() + .set_modified(when) + .unwrap(); + for format in [Format::Tar, Format::TarGz, Format::TarXz] { + let archive = room.path(&format!("dated.{}", format.extension())); + crate::archive_ops::compress( + &archive, + std::slice::from_ref(&source), + format, + arca_core::Codec::Deflate, + arca_core::Level::NORMAL, + &|_, _, _| true, + (None, false), + ) + .unwrap(); + let (entries, _) = + crate::archive_ops::list_entries(&archive, None, &|_, _, _| true).unwrap(); + let entry = entries.iter().find(|e| e.name.ends_with("a.txt")).unwrap(); + assert_eq!(entry.mtime, Some(1_600_000_000), "{format:?}"); + } +} + +// A tar.xz compresses the whole stream, so the saving is the file against +// what it holds; summing the entries said 0% for every one of them. +#[test] +fn a_compressed_tar_says_how_much_its_file_saved() { + let room = Room::new(); + let source = room.path("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("a.txt"), "the same line again\n".repeat(5000)).unwrap(); + let archive = room.path("saved.tar.xz"); + crate::archive_ops::compress( + &archive, + &[source], + Format::TarXz, + arca_core::Codec::Deflate, + arca_core::Level::NORMAL, + &|_, _, _| true, + (None, false), + ) + .unwrap(); + let mut app = AppController::new(Settings::default()); + app.open_with_password(archive.clone(), None); + settle(&mut app); + let bytes = std::fs::metadata(&archive).unwrap().len(); + let summary = app.summary(); + assert!(summary.contains(&human(bytes)), "{summary}"); + assert!(!summary.contains(" 0.0%"), "{summary}"); +} + #[test] fn a_rewrite_that_failed_behind_keeps_the_reread_of_the_one_before() { let room = Room::new(); @@ -767,3 +829,104 @@ fn a_second_edit_keeps_one_step_to_undo() { assert_eq!(app.state.entries.len(), after_first); assert!(!crate::archive_ops::undo_path(&archive).exists()); } + +#[test] +fn an_edit_that_failed_offers_nothing_to_undo() { + let room = Room::new(); + let source = room.path("source"); + fs::create_dir_all(&source).unwrap(); + fs::write(source.join("a.txt"), b"first contents").unwrap(); + let archive = room.path("broken.zip"); + crate::archive_ops::compress( + &archive, + &[source], + crate::Format::Zip, + Codec::Deflate, + Level::NORMAL, + &|_, _, _| true, + (None, false), + ) + .unwrap(); + let mut app = AppController::new(Settings::default()); + app.open_with_password(archive.clone(), None); + settle(&mut app); + let stale = crate::archive_ops::undo_path(&archive); + fs::write(&stale, b"left over from another session").unwrap(); + fs::write(&archive, b"no longer a zip").unwrap(); + app.run_job(Job::Delete { + archive: archive.clone(), + names: vec!["source/a.txt".into()], + password: None, + }); + settle(&mut app); + assert!(app.state.error); + assert!(app.state.undo.is_none()); + app.undo_last(); + assert_eq!(fs::read(&archive).unwrap(), b"no longer a zip"); +} + +#[test] +fn undoing_puts_back_an_archive_a_rewrite_left_missing() { + let room = Room::new(); + let (mut app, archive) = edited_archive(&room); + let after = app.state.entries.len(); + fs::remove_file(&archive).unwrap(); + app.undo_last(); + settle(&mut app); + assert!(archive.exists()); + assert!(!app.state.error, "{}", app.state.notice); + assert_eq!(app.state.entries.len(), after + 1); +} + +#[test] +fn opening_an_archive_can_be_given_up_on() { + let room = Room::new(); + let source = room.path("source"); + fs::create_dir_all(&source).unwrap(); + fs::write(source.join("a.txt"), b"contents").unwrap(); + let archive = room.path("slow.zip"); + crate::archive_ops::compress( + &archive, + &[source], + crate::Format::Zip, + Codec::Deflate, + Level::NORMAL, + &|_, _, _| true, + (None, false), + ) + .unwrap(); + let mut app = AppController::new(Settings::default()); + app.browse_disk(&room.path("")); + settle(&mut app); + app.open_with_password(archive.clone(), None); + assert!(app.state.listing); + app.cancel_opening(); + assert!(!app.state.listing && !app.state.busy); + assert!(app.state.channel.is_none()); + std::thread::sleep(Duration::from_millis(100)); + app.receive(); + assert!(app.on_disk(), "the window went back to the folder"); + assert_ne!(app.state.archive.as_ref(), Some(&archive)); +} + +#[test] +fn stopping_a_task_that_waits_on_a_question_lets_it_go() { + let mut app = AppController::new(Settings::default()); + let (answered_tx, answered_rx) = channel(); + let asker: Work = Box::new(move |tx, replies, _, _| { + let _ = tx.send(Message::Conflict("there.txt".into())); + let answer = replies.recv().unwrap_or(Answer::Cancel); + let _ = answered_tx.send(answer); + let _ = tx.send(Message::Failed(arca_core::Error::Cancelled.to_string())); + }); + let asking = app.enqueue(Task::new("Extract", "a", asker)); + settle_until(&mut app, |app| app.conflict().is_some()); + app.cancel_task(asking); + assert!(app.conflict().is_none()); + assert_eq!( + answered_rx.recv_timeout(Duration::from_secs(10)).unwrap(), + Answer::Cancel + ); + settle(&mut app); + assert_eq!(status(&app, asking), TaskStatus::Stopped); +} diff --git a/arca-gui/src/controller/mod.rs b/arca-gui/src/controller/mod.rs index c063fde6..ba7deb6e 100644 --- a/arca-gui/src/controller/mod.rs +++ b/arca-gui/src/controller/mod.rs @@ -52,8 +52,28 @@ pub(crate) struct AppController { pub(crate) volumes: Vec, volumes_at: Option, volume_job: Option, + survey: disk::Survey, + survey_job: Option, + // Bumped by every change to `state.entries`; the rows built from them + // are kept until it moves. See `entries_changed`. + entries_gen: u64, + rows: std::cell::RefCell, } +/// Rows already built, so that drawing a frame does not build them again. A +/// list of 100,000 entries costs that many strings to sort into folders, and +/// the list is asked for several times a frame. +#[derive(Default)] +struct RowCache { + stamp: (u64, usize, usize), + slots: Vec<(RowKey, std::rc::Rc>)>, + by_name: Option>>, +} + +type RowKey = (String, String, (SortColumn, bool), bool, bool); + +const ROW_SLOTS: usize = 8; + fn rename_destination(path: &str, name: &str) -> String { let path = path.trim_end_matches('/'); match path.rsplit_once('/') { @@ -698,8 +718,8 @@ impl AppController { } pub(crate) fn select_all_visible(&mut self) { self.cancel_preview(); - for row in self.visible_rows() { - self.set_checked_projection(&row, true); + for row in self.visible_rows().iter() { + self.set_checked_projection(row, true); } } pub(crate) fn invert_visible(&mut self) { @@ -759,6 +779,10 @@ impl AppController { volumes: Vec::new(), volumes_at: None, volume_job: None, + survey: Default::default(), + survey_job: None, + entries_gen: 0, + rows: Default::default(), state: AppState { view: View::Browse, settings, @@ -797,6 +821,7 @@ impl AppController { add_password: String::new(), hide_names: false, archive_password: None, + archive_bytes: None, reread_dir: None, reread_queued: None, history: vec![String::new()], @@ -833,7 +858,13 @@ impl AppController { pub(crate) fn summary(&self) -> String { let s = self.s(); if self.on_disk() { - let rows = children_of(&self.state.entries, &self.state.active_pane().directory); + let rows = self.rows_for( + &self.state.active_pane().directory, + "", + (SortColumn::Name, true), + false, + false, + ); let raw: u64 = rows .iter() .filter(|row| !row.is_dir) @@ -853,11 +884,16 @@ impl AppController { } let n = self.state.entries.iter().filter(|e| !e.is_dir).count(); let raw: u64 = self.state.entries.iter().map(|e| e.size).sum(); - let packed: u64 = self.state.entries.iter().map(|e| e.compressed_size).sum(); + let packed: u64 = match self.state.archive_bytes { + Some(bytes) if whole_stream(self.state.format) => bytes, + _ => self.state.entries.iter().map(|e| e.compressed_size).sum(), + }; + // A plain .tar is a little larger than what it holds, and "-2% saved" + // says nothing anybody wants to know. let ratio = if raw == 0 { 0.0 } else { - (1.0 - packed as f64 / raw as f64) * 100.0 + ((1.0 - packed as f64 / raw as f64) * 100.0).max(0.0) }; format!( "{n} {} \u{b7} {} {} \u{b7} {} {} \u{b7} {ratio:.1}% {}", @@ -994,6 +1030,24 @@ impl AppController { // stands for all of them, so a copy hands the clipboard one folder instead // of the fifteen hundred files inside it, and the Explorer pastes a folder // rather than a heap of loose files. + /// Gives up on an archive still being read and goes back to where the + /// window was. The reading thread is told to stop and whatever it still + /// sends is not listened to. + pub(crate) fn cancel_opening(&mut self) { + if !self.state.listing { + return; + } + self.state + .stop + .store(true, std::sync::atomic::Ordering::Relaxed); + self.state.channel = None; + self.state.busy = false; + self.state.listing = false; + self.state.overlay = false; + self.state.view = View::Browse; + self.close_archive(); + } + pub(crate) fn cancel_password(&mut self) { let was_job = matches!(self.state.waiting_on_password, Some(Pending::Extract(_))); let was_opening = matches!(self.state.waiting_on_password, Some(Pending::OpenArchive)); @@ -1030,7 +1084,7 @@ impl AppController { if self.blocked() { return; } - let Some((archive, _)) = self.state.undo.take() else { + let Some((archive, word)) = self.state.undo.take() else { return; }; let keep = undo_path(&archive); @@ -1038,14 +1092,35 @@ impl AppController { return; } let pw = self.state.archive_password.clone(); - if let Err(e) = fs::remove_file(&archive).and_then(|_| fs::rename(&keep, &archive)) { + // One rename, which replaces the archive where there is one: removing + // it first would lose it if the rename then failed, and would fail + // outright when a rewrite left no archive behind. + if let Err(e) = fs::rename(&keep, &archive) { self.state.notice = e.to_string(); self.state.error = true; + self.state.undo = Some((archive, word)); return; } self.open_with_password(archive, pw); } + // Offers to take a change back once it has landed. One that failed left + // the archive as it was, and whatever was on offer before still holds; + // except when it failed after the archive stepped aside, which leaves the + // kept version as the only copy and putting it back as the way out. + pub(crate) fn keep_undo(&mut self, undo: (PathBuf, &'static str), landed: bool) { + let stranded = !undo.0.exists() && undo_path(&undo.0).exists(); + if !landed && !stranded { + return; + } + // A change to the same archive replaced the kept version itself when + // it stepped the archive aside; anything else would leave it orphaned. + if self.state.undo.as_ref().map(|(archive, _)| archive) != Some(&undo.0) { + self.drop_undo(); + } + self.state.undo = Some(undo); + } + // Reads the names in the archive again under another code page. // // Only the person looking at it can know which one an unflagged zip was @@ -1061,6 +1136,7 @@ impl AppController { e.name = arca_zip::pages::decode(&e.raw_name, page); e.is_dir = e.name.ends_with('/') || e.name.ends_with('\\'); } + self.entries_changed(); self.state.folders = tree::folders_of(&self.state.entries); self.clear_picked(); self.state.active_pane_mut().cursor = None; @@ -1858,9 +1934,11 @@ impl AppController { self.state.active_pane_mut().checked = vec![false; v.len()]; self.state.folders = tree::folders_of(&v); self.state.entries = v; + self.entries_changed(); if let Some(f) = detect(&path) { self.state.format = f; } + self.state.archive_bytes = fs::metadata(&path).ok().map(|m| m.len()); // The name of what is open goes where every other // program puts it, which frees a whole row above the // list for nothing at all. @@ -1971,7 +2049,7 @@ impl AppController { } } - pub(crate) fn visible_rows(&self) -> Vec { + pub(crate) fn visible_rows(&self) -> std::rc::Rc> { self.rows_for( &self.state.active_pane().directory, &self.state.active_pane().filter, @@ -1981,6 +2059,28 @@ impl AppController { ) } + /// Every change to `state.entries` goes through here or is followed by + /// it, so that rows built from the old ones are not shown again. + pub(crate) fn entries_changed(&mut self) { + self.entries_gen = self.entries_gen.wrapping_add(1); + } + + fn row_cache(&self) -> std::cell::RefMut<'_, RowCache> { + let stamp = ( + self.entries_gen, + self.state.entries.as_ptr() as usize, + self.state.entries.len(), + ); + let mut cache = self.rows.borrow_mut(); + if cache.stamp != stamp { + *cache = RowCache { + stamp, + ..RowCache::default() + }; + } + cache + } + pub(crate) fn rows_for( &self, directory: &str, @@ -1988,8 +2088,48 @@ impl AppController { order: (SortColumn, bool), flat: bool, up: bool, - ) -> Vec { + ) -> std::rc::Rc> { let filter = filter.trim().to_lowercase(); + let key: RowKey = (directory.to_string(), filter, order, flat, up); + if let Some((_, rows)) = self.row_cache().slots.iter().find(|(k, _)| *k == key) { + return rows.clone(); + } + let rows = std::rc::Rc::new(self.build_rows(&key.0, &key.1, order, flat, up)); + let mut cache = self.row_cache(); + if cache.slots.len() >= ROW_SLOTS { + cache.slots.remove(0); + } + cache.slots.push((key, rows.clone())); + rows + } + + /// The entries sorted by their name with forward slashes, so that the ones + /// under a folder are one run that starts where its path would sort. + fn name_index(&self) -> std::rc::Rc> { + if let Some(index) = &self.row_cache().by_name { + return index.clone(); + } + let mut index: Vec<(String, usize)> = self + .state + .entries + .iter() + .enumerate() + .map(|(i, e)| (tree::normalized(e), i)) + .collect(); + index.sort_unstable(); + let index = std::rc::Rc::new(index); + self.row_cache().by_name = Some(index.clone()); + index + } + + fn build_rows( + &self, + directory: &str, + filter: &str, + order: (SortColumn, bool), + flat: bool, + up: bool, + ) -> Vec { // A search reaches down through the folders, so it starts where you // are: from the root it still walks the whole archive, and inside a // folder it stays under that folder. The flat view is the same list @@ -1998,7 +2138,7 @@ impl AppController { search_under( &self.state.entries, if flat { "" } else { directory }, - &filter, + filter, ) } else if flat { self.state @@ -2216,6 +2356,7 @@ impl AppController { self.state.password_wrong = false; self.state.reread_dir = None; self.state.entries.clear(); + self.entries_changed(); self.state.active_pane_mut().checked.clear(); self.state.folders = tree::Folder::default(); self.state.active_pane_mut().directory.clear(); @@ -2403,14 +2544,6 @@ impl AppController { Job::Move { archive, .. } => Some((archive.clone(), words.moving_word)), _ => None, }; - // A change to the same archive replaces the kept version itself when - // it steps the archive aside; anything else would leave it orphaned. - let kept = self.state.undo.as_ref().map(|(archive, _)| archive); - if kept != undo.as_ref().map(|(archive, _)| archive) { - self.drop_undo(); - } - self.state.undo = undo; - let (reads, writes, outputs) = touched_by(&job); let mut task = Task::new( verb, @@ -2436,7 +2569,7 @@ impl AppController { let created = if let Job::Compress { out, password, - format: format @ (Format::SevenZ | Format::Rar), + format, .. } = &mut job { @@ -2474,6 +2607,7 @@ impl AppController { task.in_bytes = in_bytes; task.stop_leaves_files = stop_leaves_files; task.reread_after = reread_after; + task.undo = undo; task.close_when_done = self.state.close_when_done; self.enqueue(task); } @@ -2496,8 +2630,13 @@ impl AppController { match row.entry { Some(i) => selected(i), None => { - let under = entries_under(&self.state.entries, &row.path); - !under.is_empty() && under.into_iter().all(selected) + let index = self.name_index(); + let start = index.partition_point(|(name, _)| name.as_str() < row.path.as_str()); + let mut under = index[start..] + .iter() + .take_while(|(name, _)| name.starts_with(&row.path)) + .peekable(); + under.peek().is_some() && under.all(|(_, i)| selected(*i)) } } } @@ -3067,7 +3206,8 @@ mod filter_tests { controller.state.active_pane_mut().filter = filter.to_string(); controller .visible_rows() - .into_iter() + .iter() + .cloned() .map(|r| r.label) .collect() } diff --git a/arca-gui/src/controller/panes.rs b/arca-gui/src/controller/panes.rs index db6353e7..878e0f97 100644 --- a/arca-gui/src/controller/panes.rs +++ b/arca-gui/src/controller/panes.rs @@ -68,9 +68,9 @@ impl AppController { self.cancel_preview(); } - pub(crate) fn pane_rows(&self, pane: usize) -> Vec { + pub(crate) fn pane_rows(&self, pane: usize) -> std::rc::Rc> { let Some(state) = self.state.browser.panes.get(pane) else { - return Vec::new(); + return Default::default(); }; if pane == self.state.browser.active { self.visible_rows() @@ -417,8 +417,9 @@ mod tests { } let folder = c .visible_rows() - .into_iter() + .iter() .find(|r| r.path == "a/") + .cloned() .unwrap(); c.state.active_pane_mut().checked[1] = true; assert!(!c.pane_is_checked(0, &folder)); @@ -492,8 +493,9 @@ mod tests { assert!(c.selected_names().is_empty()); let middle = c .visible_rows() - .into_iter() + .iter() .find(|r| r.path == "a/b/middle.txt") + .cloned() .unwrap(); c.set_checked(&middle, true); c.set_pane_filter(2, "middle".into()); @@ -582,8 +584,9 @@ mod tests { let mut c = controller(); let root = c .visible_rows() - .into_iter() + .iter() .find(|r| r.path == "root.txt") + .cloned() .unwrap(); c.set_checked(&root, true); c.set_pane_filter(0, "absent".into()); @@ -637,4 +640,131 @@ mod tests { assert_eq!(c.visible_rows()[2].path, "a/b/middle.txt"); assert_eq!(c.opened_descendant(usize::MAX), None); } + + #[test] + fn rows_are_built_once_until_something_they_show_changes() { + let mut c = controller(); + let first = c.visible_rows(); + assert!(std::rc::Rc::ptr_eq(&first, &c.visible_rows())); + + c.state.active_pane_mut().filter = "deep".into(); + let filtered = c.visible_rows(); + assert!(!std::rc::Rc::ptr_eq(&first, &filtered)); + c.state.active_pane_mut().filter = " DEEP ".into(); + assert!(std::rc::Rc::ptr_eq(&filtered, &c.visible_rows())); + c.state.active_pane_mut().filter.clear(); + assert!(std::rc::Rc::ptr_eq(&first, &c.visible_rows())); + + c.state.active_pane_mut().order = (SortColumn::Name, false); + assert!(!std::rc::Rc::ptr_eq(&first, &c.visible_rows())); + c.state.active_pane_mut().order = (SortColumn::Name, true); + + c.state.entries[0].name = "renamed.txt".into(); + c.entries_changed(); + let after = c.visible_rows(); + assert!(!std::rc::Rc::ptr_eq(&first, &after)); + assert!(after.iter().any(|r| r.path == "renamed.txt")); + } + + #[test] + fn reading_names_under_another_code_page_shows_the_new_names() { + let mut c = controller(); + c.state.entries[0].utf8 = false; + c.state.entries[0].raw_name = b"caf\x82.txt".to_vec(); + let before = c.visible_rows(); + assert!(before.iter().any(|r| r.path == "root.txt")); + c.reread_names(arca_zip::pages::Page::default()); + let after = c.visible_rows(); + assert!(!after.iter().any(|r| r.path == "root.txt")); + assert!(after.iter().any(|r| r.path.starts_with("caf"))); + } + + #[test] + fn a_folder_counts_as_picked_only_when_everything_under_it_is() { + let mut c = controller(); + let folder = c + .visible_rows() + .iter() + .find(|r| r.path == "a/") + .cloned() + .unwrap(); + assert!(!c.is_checked(&folder)); + for (i, e) in c.state.entries.clone().iter().enumerate() { + c.state.active_pane_mut().checked[i] = e.name.starts_with("a/"); + } + assert!(c.is_checked(&folder)); + c.state.active_pane_mut().checked[2] = false; + assert!(!c.is_checked(&folder)); + let empty = Row { + path: "nothing/".into(), + entry: None, + ..folder + }; + assert!(!c.is_checked(&empty)); + } + + // cargo test -p arca-gui --release rows_of_a_large_archive -- --ignored --nocapture + #[test] + #[ignore = "timing, run by hand"] + fn rows_of_a_large_archive() { + let mut c = AppController::new(Settings::default()); + c.state.entries = (0..100_000) + .map(|i| entry(&format!("dir{}/sub{}/file{i}.txt", i % 25, i % 7))) + .collect(); + c.state.active_pane_mut().checked = vec![false; c.state.entries.len()]; + c.go_to("dir3/".into()); + c.entries_changed(); + let started = std::time::Instant::now(); + let built = c.visible_rows(); + for row in built.iter() { + std::hint::black_box(c.is_checked(row)); + } + let first = started.elapsed(); + let started = std::time::Instant::now(); + for _ in 0..20 { + let rows = c.build_rows("dir3/", "", (SortColumn::Name, true), false, true); + for row in rows.iter().filter(|r| r.is_dir) { + std::hint::black_box(crate::tree::entries_under(&c.state.entries, &row.path)); + } + } + let old = started.elapsed() / 20; + let started = std::time::Instant::now(); + for _ in 0..1000 { + let rows = c.visible_rows(); + assert!(std::rc::Rc::ptr_eq(&rows, &built)); + for row in rows.iter() { + std::hint::black_box(c.is_checked(row)); + } + } + let again = started.elapsed() / 1000; + println!( + "{} rows: built once in {first:?}, then {again:?} a frame; before, {old:?} each time", + built.len() + ); + } + + // 7z names an empty folder without the slash ZIP gives it, and its row + // still has to be one that can be ticked. + #[test] + fn an_empty_folder_named_without_a_slash_can_be_picked() { + let mut c = controller(); + let mut empty = entry("empty"); + empty.is_dir = true; + c.state.entries.push(empty); + c.entries_changed(); + c.state.active_pane_mut().checked = vec![false; c.state.entries.len()]; + let row = c + .visible_rows() + .iter() + .find(|r| r.path == "empty/") + .cloned() + .expect("the empty folder has a row"); + assert!(!c.is_checked(&row)); + c.dispatch(AppAction::SetChecked { + row: row.clone(), + value: true, + }); + assert!(c.is_checked(&row)); + assert!(*c.state.active_pane().checked.last().unwrap()); + } } diff --git a/arca-gui/src/controller/preview.rs b/arca-gui/src/controller/preview.rs index f1017a00..a40f79c5 100644 --- a/arca-gui/src/controller/preview.rs +++ b/arca-gui/src/controller/preview.rs @@ -766,6 +766,7 @@ mod tests { let fixture = Fixture::zip(None); let mut c = fixture.controller(); c.state.entries[0].size = VIEW_LIMIT + 1; + c.entries_changed(); c.request_preview(0); assert_eq!( c.state.preview.status, @@ -773,6 +774,7 @@ mod tests { ); assert!(c.state.preview.running.is_none()); c.state.entries[0].size = 1; + c.entries_changed(); c.request_preview(0); settle(&mut c); assert!(matches!(c.state.preview.status, PreviewStatus::Error(_))); diff --git a/arca-gui/src/controller/state.rs b/arca-gui/src/controller/state.rs index 98601690..4003470c 100644 --- a/arca-gui/src/controller/state.rs +++ b/arca-gui/src/controller/state.rs @@ -69,6 +69,8 @@ pub(crate) struct AppState { // Held for the archive currently open in the window, so extracting from it // does not ask again for every button press. pub(crate) archive_password: Option, + /// The size of the open archive's file, read when its listing arrives. + pub(crate) archive_bytes: Option, // The folder to restore after rereading a rewritten archive. It is consumed // by the next listing, so opening an archive normally still starts at root. pub(crate) reread_dir: Option, diff --git a/arca-gui/src/main.rs b/arca-gui/src/main.rs index e67b8136..4b15ed71 100644 --- a/arca-gui/src/main.rs +++ b/arca-gui/src/main.rs @@ -318,6 +318,16 @@ c76ecf12e8e05b8f4730fb9933450ea121fe1ce3699ab9b7d20b058f872815f4 *arca-setup-0.6 assert_eq!(wheel_speed(900.0), wheel_speed(2000.0)); } + // Sizes are counted in powers of 1024, and say so: "17.4 MB" for 17.4 + // MiB is 1.3 MB off by the other reading. + #[test] + fn sizes_are_named_in_binary_units() { + assert_eq!(crate::model::human(1023), "1023 B"); + assert_eq!(crate::model::human(1024), "1.0 KiB"); + assert_eq!(crate::model::human(17_400 * 1024 * 1024 / 1000), "17.4 MiB"); + assert_eq!(crate::model::human(3 << 30), "3.0 GiB"); + } + // The only arithmetic in this file that can be wrong without anyone // noticing: a date is either right or plausible, and plausible is worse. #[test] diff --git a/arca-gui/src/model.rs b/arca-gui/src/model.rs index 20a21c9e..9ae209b5 100644 --- a/arca-gui/src/model.rs +++ b/arca-gui/src/model.rs @@ -45,6 +45,16 @@ pub(crate) fn read_only_suffix(format: Option) -> String { } } +/// Whether a format compresses the archive as one stream rather than entry by +/// entry, so that no entry has a compressed size of its own and only the +/// file's size says how much was saved. +pub(crate) fn whole_stream(format: Format) -> bool { + matches!( + format.container(), + arca_core::Container::Tar | arca_core::Container::TarGz | arca_core::Container::TarXz + ) +} + pub(crate) fn saved_of(r: &Row) -> f64 { if r.size == 0 { 0.0 @@ -86,7 +96,7 @@ pub(crate) fn civil(t: i64) -> String { } pub(crate) fn human(n: u64) -> String { - const U: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; + const U: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; let mut v = n as f64; let mut i = 0; while v >= 1024.0 && i < U.len() - 1 { diff --git a/arca-gui/src/tree.rs b/arca-gui/src/tree.rs index 11a2daab..362643a9 100644 --- a/arca-gui/src/tree.rs +++ b/arca-gui/src/tree.rs @@ -63,8 +63,15 @@ pub struct Row { pub up: bool, } -fn normalized(e: &Entry) -> String { - e.name.replace('\\', "/") +/// The name with forward slashes, and with one on the end for a folder: 7z +/// and the disk list folders by the bare name, ZIP with the slash, and a +/// folder row is matched against what starts with its path and the slash. +pub(crate) fn normalized(e: &Entry) -> String { + let mut name = e.name.replace('\\', "/"); + if e.is_dir && !name.ends_with('/') { + name.push('/'); + } + name } pub fn children_of(entries: &[Entry], dir: &str) -> Vec { diff --git a/arca-gui/src/ui/actions.rs b/arca-gui/src/ui/actions.rs index aef89298..3bc7b3d5 100644 --- a/arca-gui/src/ui/actions.rs +++ b/arca-gui/src/ui/actions.rs @@ -406,6 +406,11 @@ pub(crate) enum DialogKind { name: String, directory: PathBuf, }, + /// Where the picked files on the disk are moved or copied to. + CarryTo { + roots: Vec, + copy: bool, + }, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -447,6 +452,7 @@ pub(crate) enum Shortcut { Up, CloseWindow, TogglePreview, + Duplicate, } /// Reading a key press, with no state and no side effects, so the table of @@ -481,6 +487,7 @@ pub(crate) fn shortcut_for( "i" => Some(Shortcut::Invert), "w" => Some(Shortcut::CloseWindow), "z" if !typing => Some(Shortcut::Undo), + "d" if !typing => Some(Shortcut::Duplicate), _ => None, }; } @@ -533,12 +540,17 @@ pub(crate) fn clipboard_action_allowed( available && idle && has_archive && (!needs_selection || selected > 0) } +/// What an empty list says. An archive still being read has no rows yet, and +/// saying the folder is empty then is saying something false. pub(crate) fn empty_state_aria_label( + listing: bool, error: bool, filter: &str, s: &'static Strings, ) -> &'static str { - if error { + if listing { + s.opening + } else if error { s.cannot_open } else if filter.trim().is_empty() { s.empty_folder diff --git a/arca-gui/src/ui/browser.rs b/arca-gui/src/ui/browser.rs index 12c73268..39996ba2 100644 --- a/arca-gui/src/ui/browser.rs +++ b/arca-gui/src/ui/browser.rs @@ -10,6 +10,8 @@ use egui_phosphor::regular as icon; pub(crate) const ROW_HEIGHT: f32 = 24.0; /// Space around the rows that belongs to the list but to no row. const GUTTER: f32 = 16.0; +// How long the pointer rests on a folder before its peek card opens, in seconds. +const PEEK_REST: f64 = 0.45; pub(crate) fn column_slot(column: SortColumn) -> usize { Columns::ALL @@ -201,13 +203,29 @@ impl Shell { self.focus_filter = true; } + /// What a band starts from: one flag per row on show, in the order they + /// are drawn, because that is the index each row looks itself up by while + /// the band is pulled. The selection itself is kept per entry. + pub(super) fn marquee_base(&self, pane: Option, keep: bool) -> Vec { + let rows = match pane { + Some(pane) => self.controller.pane_rows(pane), + None => self.controller.visible_rows(), + }; + if keep { + rows.iter() + .map(|row| self.controller.is_checked(row)) + .collect() + } else { + vec![false; rows.len()] + } + } + fn peek_card(&mut self, ui: &mut egui::Ui) { - const REST: f64 = 0.45; const SHOWN: usize = 8; let Some((path, rect, since)) = self.peek.clone() else { return; }; - if ui.input(|i| i.time) - since < REST || self.modal_kind().is_some() { + if ui.input(|i| i.time) - since < PEEK_REST || self.modal_kind().is_some() { return; } let s = self.controller.s(); @@ -338,17 +356,57 @@ impl Shell { }); } + /// What the list shows while an archive is being read: the mark breathing + /// in and out over the name of what is being opened, so a large archive + /// does not look like an empty one. + fn opening_state(&self, ui: &mut egui::Ui) { + let s = self.controller.s(); + let p = palette_of(ui.ctx()); + let time = ui.input(|i| i.time); + let alpha = 0.55 + 0.45 * (time * std::f64::consts::TAU / 1.6).cos() as f32; + ui.vertical_centered(|ui| { + ui.add_space((ui.available_height() / 2.0 - 90.0).max(24.0)); + ui.add( + egui::Image::from_bytes("bytes://arca-mark.svg", crate::assets::MARK) + .fit_to_exact_size(Vec2::splat(96.0)) + .tint(egui::Color32::WHITE.gamma_multiply(alpha)), + ); + ui.add_space(12.0); + ui.label(RichText::new(format!("{}\u{2026}", s.opening)).font(theme::display(20.0))); + let name = self + .controller + .state + .archive + .as_ref() + .and_then(|a| a.file_name()) + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + ui.add(egui::Label::new(RichText::new(name).color(hex(p.muted))).truncate()); + ui.add_space(16.0); + let escape = ui.input(|i| i.key_pressed(egui::Key::Escape)); + if secondary_button(ui, s.cancel, true).clicked() || escape { + self.stop_opening.set(true); + } + }); + ui.ctx() + .request_repaint_after(std::time::Duration::from_millis(33)); + } + fn empty_state(&self, ui: &mut egui::Ui) { let s = self.controller.s(); let p = palette_of(ui.ctx()); let state = &self.controller.state; + if state.listing { + self.opening_state(ui); + return; + } let text = if state.archive.is_some() && state.entries.is_empty() && state.active_pane().filter.trim().is_empty() { s.empty_archive } else { - empty_state_aria_label(state.error, &state.active_pane().filter, s) + empty_state_aria_label(false, state.error, &state.active_pane().filter, s) }; ui.add_space(40.0); ui.vertical_centered(|ui| { @@ -399,6 +457,10 @@ impl Shell { let total: f32 = widths.iter().sum(); let order = self.controller.state.active_pane().order; let current_dir = self.controller.row_root().to_string(); + // No entry of a .tar.gz or a .tar.xz is compressed on its own, so it + // has no packed size or saving to show; the status bar has the total. + let whole = self.controller.state.archive.is_some() + && crate::model::whole_stream(self.controller.state.format); egui::ScrollArea::horizontal() .id_salt("details-across") .auto_shrink(false) @@ -594,7 +656,13 @@ impl Shell { } continue; } - let text = column_text(row, *column, s, ¤t_dir); + let text = if whole + && matches!(column, SortColumn::Packed | SortColumn::Saved) + { + String::new() + } else { + column_text(row, *column, s, ¤t_dir) + }; let (anchor, at) = if right_aligned(*column) { ( Align2::RIGHT_CENTER, @@ -846,6 +914,12 @@ impl Shell { } } let rows = self.controller.pane_rows(pane); + if rows.is_empty() && self.controller.state.listing { + if pane == self.controller.state.browser.active { + self.opening_state(ui); + } + return; + } if rows.is_empty() { ui.label(RichText::new(s.empty_folder).color(hex(p.muted))); self.empty_space(ui, Some(pane)); @@ -1029,9 +1103,13 @@ impl Shell { } if let Some(peek) = &mut self.peek { peek.1 = response.rect; + // One more frame when the card is due, and none after it. + let left = PEEK_REST - (now - peek.2); + if left > 0.0 { + ui.ctx() + .request_repaint_after(std::time::Duration::from_secs_f64(left)); + } } - ui.ctx() - .request_repaint_after(std::time::Duration::from_millis(120)); } if let Some(marquee) = self.marquee.as_ref().filter(|m| m.pane == pane) { let swept = response.rect.intersects(marquee.rect()); @@ -1300,11 +1378,7 @@ impl Shell { if !keep { self.controller.dispatch(AppAction::ClearSelection); } - let base = if keep { - self.controller.state.active_pane().checked.clone() - } else { - vec![false; self.controller.state.active_pane().checked.len()] - }; + let base = self.marquee_base(pane, keep); self.marquee = Some(Marquee { from, to: from, diff --git a/arca-gui/src/ui/dialogs.rs b/arca-gui/src/ui/dialogs.rs index dbe74a36..f886587b 100644 --- a/arca-gui/src/ui/dialogs.rs +++ b/arca-gui/src/ui/dialogs.rs @@ -45,6 +45,7 @@ pub(crate) fn shortcut_rows(s: &'static Strings) -> [Vec; 2] { (&["f2"], s.rename_word), (&["delete"], s.delete_word), (&["ctrl-shift-n"], s.new_folder), + (&["ctrl-d"], s.duplicate_word), (&["alt-enter"], s.properties_word), (&["alt-p"], s.preview_panel), (&["ctrl-w"], s.close_window), @@ -507,7 +508,9 @@ impl Shell { }); ui.add_space(10.0); let at_least = |text: String| { - if props.capped { + if props.counting { + "\u{2026}".to_string() + } else if props.capped { format!("≥ {text}") } else { text @@ -532,7 +535,7 @@ impl Shell { let yes_no = |on: bool| if on { s.yes_word } else { s.no_word }; let mut rows: Vec<(&str, String)> = vec![ (s.location_word, location), - (s.col_size, at_least(pretty_bytes(props.size))), + (s.col_size, at_least(crate::model::human(props.size))), ]; if props.is_dir { rows.push(( @@ -791,15 +794,24 @@ impl Shell { let p = palette_of(ui.ctx()); let inputs = self.controller.state.pending_inputs.clone(); let count = inputs.len(); - let sizes: Vec> = inputs - .iter() - .map(|path| { - std::fs::metadata(path) - .ok() - .filter(|m| m.is_file()) - .map(|m| m.len()) - }) - .collect(); + let facts = match &self.input_facts { + Some((read_for, facts)) if *read_for == inputs => facts.clone(), + _ => { + let facts: super::InputFacts = std::rc::Rc::new( + inputs + .iter() + .map(|path| { + let meta = std::fs::metadata(path).ok(); + let size = meta.as_ref().filter(|m| m.is_file()).map(|m| m.len()); + (size, meta.is_some_and(|m| m.is_dir())) + }) + .collect(), + ); + self.input_facts = Some((inputs.clone(), facts.clone())); + facts + } + }; + let sizes: Vec> = facts.iter().map(|(size, _)| *size).collect(); let total: u64 = sizes.iter().flatten().sum(); ui.horizontal(|ui| { ui.label( @@ -1015,12 +1027,21 @@ impl Shell { }); return; } - egui::ScrollArea::vertical() - .max_height(180.0) - .show(ui, |ui| { - for (index, path) in inputs.iter().enumerate() { + // The height of the remove button, the tallest thing in a row. + let row_height = 28.0; + egui::ScrollArea::vertical().max_height(180.0).show_rows( + ui, + row_height, + count, + |ui, shown| { + for (index, path) in inputs + .iter() + .enumerate() + .skip(shown.start) + .take(shown.len()) + { ui.horizontal(|ui| { - let is_dir = sizes[index].is_none() && path.is_dir(); + let is_dir = facts[index].1; let glyph = if is_dir { icon::FOLDER } else { icon::FILE }; ui.label( super::theme::icon_text(glyph) @@ -1054,7 +1075,8 @@ impl Shell { ); }); } - }); + }, + ); }); if let Some(index) = remove { self.controller.state.pending_inputs.remove(index); @@ -2210,21 +2232,6 @@ fn paint_swatch(ui: &egui::Ui, rect: egui::Rect, palette: theme::Palette) { } } -fn pretty_bytes(n: u64) -> String { - const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; - let mut value = n as f64; - let mut unit = 0; - while value >= 1024.0 && unit < UNITS.len() - 1 { - value /= 1024.0; - unit += 1; - } - if unit == 0 { - format!("{n} B") - } else { - format!("{value:.1} {}", UNITS[unit]) - } -} - /// A file's time as the date and time the clock here showed then. fn stamp(when: std::time::SystemTime) -> String { let secs = when diff --git a/arca-gui/src/ui/mod.rs b/arca-gui/src/ui/mod.rs index ca1d8ff0..2b1a7df9 100644 --- a/arca-gui/src/ui/mod.rs +++ b/arca-gui/src/ui/mod.rs @@ -44,8 +44,17 @@ enum DialogResult { }, AddFiles(Option>), SaveCopy(Option), + CarryTo { + roots: Vec, + copy: bool, + to: Option, + }, } +/// For each file waiting to be compressed: its size if it is a file, and +/// whether it is a folder. +pub(crate) type InputFacts = std::rc::Rc, bool)>>; + /// Rows carried by a drag inside the window, as the roots they stand for. #[derive(Clone)] pub(crate) struct DraggedRows(pub Vec); @@ -65,8 +74,8 @@ fn look_of(s: &Settings) -> String { } /// A rectangle being dragged out over the rows from the blank space, which -/// picks every row it crosses. `base` is what was checked when it started, -/// kept when Ctrl is held so the sweep adds to it. +/// picks every row it crosses. `base` is what was checked when it started, one +/// flag per row on show, kept when Ctrl is held so the sweep adds to it. pub(crate) struct Marquee { from: egui::Pos2, to: egui::Pos2, @@ -90,6 +99,9 @@ pub(crate) struct Props { pub(crate) folders: usize, /// The walk stopped early, so size and counts are lower bounds. pub(crate) capped: bool, + /// A folder still being walked on its own thread: size and counts so far + /// say nothing yet. + pub(crate) counting: bool, pub(crate) modified: Option, pub(crate) hidden: bool, pub(crate) pinned: bool, @@ -103,7 +115,7 @@ const PROPS_WALK_MOST: usize = 50_000; /// Sizes up a path without going on for ever: deep trees are cut off at /// `PROPS_WALK_MOST` entries and said to be at least that big. -pub(crate) fn inspect(path: &std::path::Path, pinned: bool) -> Props { +pub(crate) fn inspect(path: &std::path::Path, pinned: bool, walk: bool) -> Props { let meta = std::fs::symlink_metadata(path).ok(); let is_dir = meta.as_ref().is_some_and(|m| m.is_dir()); let name = path @@ -120,6 +132,7 @@ pub(crate) fn inspect(path: &std::path::Path, pinned: bool) -> Props { files: 0, folders: 0, capped: false, + counting: false, modified: meta.as_ref().and_then(|m| m.modified().ok()), hidden: path .file_name() @@ -136,7 +149,9 @@ pub(crate) fn inspect(path: &std::path::Path, pinned: bool) -> Props { props.owner = unix_name("/etc/passwd", meta.uid()); props.group = unix_name("/etc/group", meta.gid()); } - if is_dir { + if is_dir && !walk { + props.counting = true; + } else if is_dir { let mut stack = vec![path.to_path_buf()]; while let Some(dir) = stack.pop() { let Ok(read) = std::fs::read_dir(&dir) else { @@ -189,6 +204,13 @@ pub(crate) struct Shell { marquee: Option, pane_action: Option, properties: Option, + properties_job: Option>, + // Set by the opening screen when somebody gives up waiting for an + // archive to be read; acted on once the frame is drawn. + stop_opening: std::cell::Cell, + // Size and kind of each file waiting to be compressed, read once for the + // list they were read for rather than on every frame. + input_facts: Option<(Vec, InputFacts)>, last_active_pane: Option, // Whether the last thing the user did was press a key. While it was, // the pointer resting on another pane does not steal the focus. @@ -303,6 +325,9 @@ impl Shell { marquee: None, pane_action: None, properties: None, + properties_job: None, + stop_opening: Default::default(), + input_facts: None, last_active_pane: None, keyboard_drove: false, peek: None, @@ -366,6 +391,7 @@ impl Shell { /// Everything that happened on other threads since the last frame. fn poll(&mut self, ctx: &egui::Context) { self.poll_dialog(); + self.poll_properties(); self.controller.ask_about_updates(); let conflict_was_open = self.controller.conflict().is_some(); if self.controller.receive() { @@ -395,10 +421,13 @@ impl Shell { || state.channel.is_some() || state.update_rx.is_some() || matches!(state.preview.status, PreviewStatus::Loading) - || state - .tasks - .iter() - .any(|task| matches!(task.status, TaskStatus::Queued | TaskStatus::Running)); + || state.tasks.iter().any(|task| { + // A paused task says nothing until it is let go, and letting it + // go is a click, which brings a frame of its own. + let paused = task.hold.load(std::sync::atomic::Ordering::Relaxed) + && !task.stop.load(std::sync::atomic::Ordering::Relaxed); + matches!(task.status, TaskStatus::Queued | TaskStatus::Running) && !paused + }); if waiting { ctx.request_repaint_after(TICK); } else if state.cut_pending.is_some() { @@ -925,12 +954,54 @@ impl Shell { .set_directory(&directory) .save_file(), ), + DialogKind::CarryTo { roots, copy } => DialogResult::CarryTo { + roots, + copy, + to: rfd::FileDialog::new().pick_folder(), + }, }; let _ = tx.send(result); ctx.request_repaint(); }); } + /// Opens Properties at once with what one look at the path says, and + /// sizes a folder up on a thread: a tree on a network share or a slow + /// disk can take seconds to walk, and the window keeps drawing meanwhile. + pub(crate) fn show_properties(&mut self, path: PathBuf, pinned: bool) { + let props = inspect(&path, pinned, false); + self.properties_job = None; + if props.counting { + let (tx, rx) = channel(); + let ctx = self.ctx.clone(); + std::thread::spawn(move || { + let _ = tx.send(inspect(&path, pinned, true)); + ctx.request_repaint(); + }); + self.properties_job = Some(rx); + } + self.properties = Some(props); + } + + fn poll_properties(&mut self) { + let Some(job) = &self.properties_job else { + return; + }; + match job.try_recv() { + Ok(done) => { + self.properties_job = None; + if let Some(open) = &mut self.properties { + if open.path == done.path { + let pinned = open.pinned; + *open = Props { pinned, ..done }; + } + } + } + Err(std::sync::mpsc::TryRecvError::Disconnected) => self.properties_job = None, + Err(std::sync::mpsc::TryRecvError::Empty) => {} + } + } + fn poll_dialog(&mut self) { let Some(dialog) = &self.dialog else { return; @@ -961,6 +1032,18 @@ impl Shell { DialogResult::AddFiles(Some(paths)) if !paths.is_empty() => { self.controller.dispatch(AppAction::Add(paths)) } + DialogResult::CarryTo { + roots, + copy, + to: Some(dir), + } => { + let target = crate::controller::disk::disk_dir(&dir); + if copy { + self.controller.copy_on_disk(&roots, &target); + } else { + self.controller.move_on_disk(&roots, &target); + } + } DialogResult::SaveCopy(Some(dest)) => { if let Some(archive) = self.controller.state.archive.clone() { if dest != archive { @@ -1100,7 +1183,7 @@ impl Shell { if name.is_empty() { return; } - for row in &rows { + for row in rows.iter() { if matches_mask(&name, &row.label) { self.controller.dispatch(AppAction::SetChecked { row: row.clone(), @@ -1191,14 +1274,10 @@ impl Shell { if roots.is_empty() { return; } - if let Some(dir) = rfd::FileDialog::new().pick_folder() { - let target = crate::controller::disk::disk_dir(&dir); - if action == RowAction::MoveTo { - self.controller.move_on_disk(&roots, &target); - } else { - self.controller.copy_on_disk(&roots, &target); - } - } + self.begin_dialog(DialogKind::CarryTo { + roots, + copy: action == RowAction::CopyTo, + }); } RowAction::OpenWith => { if let Some(row) = row.filter(|row| !row.is_dir) { @@ -1219,7 +1298,7 @@ impl Shell { _ => disk_path(&self.controller.state.active_pane().directory), }; let pinned = self.controller.is_pinned(&path); - self.properties = Some(inspect(&path, pinned)); + self.show_properties(path, pinned); } RowAction::Copy => self.dispatch_clipboard(false), RowAction::Cut => self.dispatch_clipboard(true), @@ -1806,6 +1885,12 @@ impl Shell { self.row_action(RowAction::Properties, at.unwrap_or(usize::MAX)); } Shortcut::NewFolder => self.row_action(RowAction::NewFolder, usize::MAX), + // The menu offers it on the disk only, and the key does the same. + Shortcut::Duplicate => { + if self.controller.on_disk() && !self.controller.selected_roots().is_empty() { + self.row_action(RowAction::Duplicate, usize::MAX); + } + } Shortcut::Up => { if self.controller.can_go_up() { self.controller.dispatch(AppAction::Up); @@ -2069,9 +2154,20 @@ impl eframe::App for Shell { self.width = ui.available_width(); self.cursor_row = None; let only_form = self.controller.state.one_shot && self.modal_kind() == Some(ModalKind::Add); + if !only_form && self.controller.refresh_volumes() { + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } if !only_form { self.workspace(ui); } + if self.stop_opening.take() { + self.controller.cancel_opening(); + } + // A band ends when the button comes up, wherever that happens: the + // blank space that started it may be gone by then, filled by rows. + if self.marquee.is_some() && !ctx.input(|i| i.pointer.primary_down()) { + self.marquee = None; + } // After the lists, so every row has had its say about a drop. self.settle_blank_drop(&ctx); self.drag_shadow(&ctx); diff --git a/arca-gui/src/ui/sidebar.rs b/arca-gui/src/ui/sidebar.rs index 49878829..ce620277 100644 --- a/arca-gui/src/ui/sidebar.rs +++ b/arca-gui/src/ui/sidebar.rs @@ -57,7 +57,6 @@ impl Shell { let mut relabel: Option = None; let mut format: Option = None; let free = self.controller.state.browser.free; - self.controller.refresh_volumes(); egui::ScrollArea::vertical() .auto_shrink(false) .show(ui, |ui| { @@ -260,7 +259,7 @@ impl Shell { } if let Some(path) = props { let pinned = self.controller.is_pinned(&path); - self.properties = Some(super::inspect(&path, pinned)); + self.show_properties(path, pinned); } if let Some(path) = forget { self.controller diff --git a/arca-gui/src/ui/tests.rs b/arca-gui/src/ui/tests.rs index 76fcc17c..9bfe22e9 100644 --- a/arca-gui/src/ui/tests.rs +++ b/arca-gui/src/ui/tests.rs @@ -403,6 +403,7 @@ fn the_explorer_shortcuts_mean_what_they_mean_there() { let cases = [ (true, true, false, "n", Shortcut::NewFolder), (true, false, false, "w", Shortcut::CloseWindow), + (true, false, false, "d", Shortcut::Duplicate), (false, false, true, "enter", Shortcut::Properties), (false, false, true, "up", Shortcut::Up), (false, false, true, "d", Shortcut::EditPath), @@ -487,10 +488,15 @@ fn empty_folder_label_is_not_reported_as_archive_contents() { // translation that loses the difference is the same bug in Spanish. for lang in super::super::Lang::ALL { let s = crate::i18n::strings(lang); - assert_eq!(empty_state_aria_label(false, "", s), s.empty_folder); - assert_eq!(empty_state_aria_label(false, " ", s), s.empty_folder); - assert_eq!(empty_state_aria_label(false, "zip", s), s.no_matches); - assert_eq!(empty_state_aria_label(true, "", s), s.cannot_open); + assert_eq!(empty_state_aria_label(false, false, "", s), s.empty_folder); + assert_eq!( + empty_state_aria_label(false, false, " ", s), + s.empty_folder + ); + assert_eq!(empty_state_aria_label(false, false, "zip", s), s.no_matches); + assert_eq!(empty_state_aria_label(false, true, "", s), s.cannot_open); + assert_eq!(empty_state_aria_label(true, false, "", s), s.opening); + assert_ne!(s.opening, s.empty_folder); assert_ne!(s.empty_folder, s.cannot_open); } } @@ -665,3 +671,63 @@ fn f2_on_the_row_that_goes_up_renames_nothing() { shell.shortcut(Shortcut::Rename); assert!(shell.controller.state.renaming.is_none()); } + +// Properties opens on one look at the path; a folder is walked separately, +// and only the walk answers for its size and what it holds. +#[test] +fn properties_of_a_folder_wait_for_the_walk_to_give_a_size() { + let room = crate::test_support::Room::new(); + let dir = room.path("tree"); + std::fs::create_dir_all(dir.join("sub")).unwrap(); + std::fs::write(dir.join("a.txt"), b"12345").unwrap(); + std::fs::write(dir.join("sub/b.txt"), b"678").unwrap(); + let glance = inspect(&dir, false, false); + assert!(glance.is_dir && glance.counting); + assert_eq!((glance.files, glance.size), (0, 0)); + let walked = inspect(&dir, false, true); + assert!(!walked.counting); + assert_eq!((walked.files, walked.folders, walked.size), (2, 1, 8)); + let file = inspect(&dir.join("a.txt"), false, false); + assert!(!file.counting); + assert_eq!(file.size, 5); +} + +// A band pulled with Ctrl held keeps what was picked: its starting point is +// read per row on show, the index the rows look themselves up by, and not per +// entry of the archive, which is a different order inside any folder. +#[test] +fn a_ctrl_band_starts_from_the_rows_already_picked() { + let ctx = egui::Context::default(); + let mut shell = Shell::new(&ctx, crate::Startup::Browse(None)); + let mut c = crate::controller::AppController::new(crate::settings::Settings::default()); + c.state.entries = ["dir/a.txt", "dir/b.txt", "dir/c.txt", "root.txt"] + .iter() + .map(|name| arca_core::Entry { + name: name.to_string(), + raw_name: name.as_bytes().to_vec(), + size: 1, + compressed_size: 1, + method: arca_core::Method::Store, + crc32: None, + is_dir: false, + mtime: None, + created: None, + accessed: None, + attributes: 0, + offset: 0, + utf8: true, + encrypted: false, + zipcrypto: false, + }) + .collect(); + c.state.active_pane_mut().checked = vec![true, false, false, false]; + c.state.active_pane_mut().directory = "dir/".into(); + shell.controller = c; + let rows = shell.controller.visible_rows(); + let base = shell.marquee_base(None, true); + assert_eq!(base.len(), rows.len()); + for (row, picked) in rows.iter().zip(&base) { + assert_eq!(*picked, row.path == "dir/a.txt", "{}", row.path); + } + assert!(shell.marquee_base(None, false).iter().all(|picked| !picked)); +} diff --git a/arca-gui/src/ui/widgets.rs b/arca-gui/src/ui/widgets.rs index 52b4d188..fe46166b 100644 --- a/arca-gui/src/ui/widgets.rs +++ b/arca-gui/src/ui/widgets.rs @@ -533,12 +533,12 @@ pub(crate) fn nav_item( /// A thin horizontal bar of progress, with an accessible value. pub(crate) fn progress(ui: &mut Ui, fraction: Option) -> Response { let p = palette_of(ui.ctx()); - let mut bar = egui::ProgressBar::new(fraction.unwrap_or(0.0)) + // Not animated while the amount is unknown: an animated bar asks for a + // frame at the display's rate for as long as it is on screen, and with + // rounded corners egui draws no spinner in it to show for that. + let bar = egui::ProgressBar::new(fraction.unwrap_or(0.0)) .desired_height(6.0) .fill(hex(p.spark)) .corner_radius(3); - if fraction.is_none() { - bar = bar.animate(true); - } ui.add(bar) } diff --git a/arca-rar/Cargo.toml b/arca-rar/Cargo.toml index fbc46f2f..2a87bbc8 100644 --- a/arca-rar/Cargo.toml +++ b/arca-rar/Cargo.toml @@ -13,7 +13,7 @@ rar = ["dep:rars", "dep:tempfile", "dep:libc"] [dependencies] arca-core.workspace = true -rars = { version = "=0.10.2", default-features = false, features = ["encryption", "write"], optional = true } +rars = { version = "=0.10.2", default-features = false, features = ["encryption", "write", "parallel"], optional = true } tempfile = { version = "3", optional = true } [target.'cfg(unix)'.dependencies] diff --git a/arca-rar/fuzz/Cargo.lock b/arca-rar/fuzz/Cargo.lock index 3dea526d..098c9bd0 100644 --- a/arca-rar/fuzz/Cargo.lock +++ b/arca-rar/fuzz/Cargo.lock @@ -172,6 +172,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.2.2" @@ -202,6 +227,12 @@ dependencies = [ "ctutils", ] +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + [[package]] name = "errno" version = "0.3.14" @@ -437,18 +468,37 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rars" version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0665518097392be04a322dec9eba41e88028f6aca85f5d61f8abb068691f2266" dependencies = [ "aes", "aho-corasick", "getrandom", "hmac", + "rayon", "sha1", "sha2", "zeroize", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rustix" version = "1.1.4" diff --git a/arca-rar/fuzz/Cargo.toml b/arca-rar/fuzz/Cargo.toml index ad27efec..7b8b6aeb 100644 --- a/arca-rar/fuzz/Cargo.toml +++ b/arca-rar/fuzz/Cargo.toml @@ -19,3 +19,8 @@ opt-level = 1 [profile.dev.package."*"] opt-level = 3 + +# The same patched RAR backend the workspace builds, so the fuzzer exercises +# the decoder Arca ships and not the one on crates.io. +[patch.crates-io] +rars = { path = "../vendor/rars" } diff --git a/arca-rar/src/lib.rs b/arca-rar/src/lib.rs index ae3849a5..8b33f623 100644 --- a/arca-rar/src/lib.rs +++ b/arca-rar/src/lib.rs @@ -29,7 +29,11 @@ pub mod create_limits { pub const MAX_HEADER_BYTES: u64 = 64 * MIB; /// Managed writer memory: compression workspace, retained headers and /// spool bookkeeping. Not a process-RAM limit; see `rars::WriterResources`. - pub const WRITER_MEMORY_BYTES: u64 = 256 * MIB; + /// The writer charges each compression job about 260 MiB however little + /// it uses, and admits as many jobs as fit, so this is what lets the cores + /// work side by side: sixteen jobs on 144 MiB of mixed files stayed under + /// 230 MiB resident in all. + pub const WRITER_MEMORY_BYTES: u64 = 4096 * MIB; /// Logical bytes the writer may spool to disk beside the output. pub const MAX_SPOOL_BYTES: u64 = 16 * 1024 * MIB; } @@ -46,6 +50,9 @@ pub enum Conflict { pub struct RarArchive { entries: Vec, + // The Unix permissions each member recorded, where it was made on Unix. + #[cfg(feature = "rar")] + modes: Vec>, #[cfg(feature = "rar")] inner: volumes::Volumes, } @@ -91,6 +98,22 @@ impl RarArchive { } } + /// Checks a password against the first encrypted file only, decoding as + /// little of the archive as gets there, instead of all of it as [`test`] + /// does. A wrong password fails here; a right one says nothing about + /// whether the rest of the archive is intact. + /// + /// [`test`]: Self::test + pub fn check_password(&self, password: &str, progress: &Progress<'_>) -> Result<()> { + #[cfg(feature = "rar")] + return reader::check_password(self, password, progress); + #[cfg(not(feature = "rar"))] + { + let _ = (password, progress); + Err(disabled()) + } + } + pub fn read_entry(&self, index: usize, password: Option<&str>) -> Result> { #[cfg(feature = "rar")] return reader::read_entry(self, index, password); diff --git a/arca-rar/src/reader.rs b/arca-rar/src/reader.rs index fbd86757..e4c3e1f5 100644 --- a/arca-rar/src/reader.rs +++ b/arca-rar/src/reader.rs @@ -100,6 +100,7 @@ pub(super) fn open( let mut names = HashSet::new(); let mut files = HashSet::new(); let mut total = 0u64; + let mut modes = Vec::new(); for (index, member) in inner.members(token)?.into_iter().enumerate() { let meta = &member.meta; if token.is_cancelled() { @@ -142,6 +143,10 @@ pub(super) fn open( ArchiveMemberDetail::Rar50Plus { crc32, .. } => crc32, _ => None, }; + modes.push( + (meta.attr_source() == rars::AttrSource::Unix) + .then_some((meta.file_attr & 0o7777) as u32), + ); entries.push(Entry { name, size: meta.unpacked_size, @@ -183,7 +188,11 @@ pub(super) fn open( parent = path.parent(); } } - Ok(RarArchive { inner, entries }) + Ok(RarArchive { + inner, + entries, + modes, + }) }) } @@ -246,6 +255,41 @@ pub(super) fn test( }) } +pub(super) fn check_password( + archive: &RarArchive, + password: &str, + progress: &Progress<'_>, +) -> Result<()> { + let Some(first) = archive + .entries + .iter() + .position(|e| e.encrypted && !e.is_dir) + else { + return Ok(()); + }; + // A set of volumes has no way to stop early, so it is tested whole. + let [single] = archive.inner.archives.as_slice() else { + return test(archive, Some(password), progress); + }; + // Everything up to the first encrypted file is decoded, since in a solid + // archive it is what that file is decoded from, and nothing after it. + let mut index = 0; + controlled(progress, |token| { + single + .extract_with_control(options(Some(password), token), |_| { + let decision = if index > first { + rars::ExtractionDecision::Stop + } else { + rars::ExtractionDecision::Extract(Box::new(io::sink())) + }; + index += 1; + Ok(decision) + }) + .map(|_| ()) + .map_err(|e| archive.inner.error(e)) + }) +} + pub(super) fn read_entry( archive: &RarArchive, index: usize, @@ -462,10 +506,25 @@ pub(super) fn extract( .ok_or_else(|| Error::Format("missing destination parent".into()))?; fs::create_dir_all(parent)?; // Publish only verified bytes, with a same-filesystem atomic replacement. + // A temporary is created readable by its owner alone; the file it + // becomes gets what any new file gets, through the umask, plus the + // execute bits the archive recorded. + #[cfg(unix)] + let mut output = { + use std::os::unix::fs::PermissionsExt; + tempfile::Builder::new() + .permissions(fs::Permissions::from_mode(0o666)) + .tempfile_in(parent)? + }; + #[cfg(not(unix))] let mut output = tempfile::NamedTempFile::new_in(parent)?; let mut input = File::open(staging.path().join(i.to_string()))?; bytes += io::copy(&mut input, &mut output)?; arca_core::extraction::set_mtime(output.as_file(), entry.mtime); + arca_core::extraction::set_executable( + output.as_file(), + archive.modes.get(i).copied().flatten(), + ); output.as_file().sync_all()?; reject_links(&target)?; if overwrite { diff --git a/arca-rar/tests/check-features.py b/arca-rar/tests/check-features.py index 8df52c3b..ae4b1f62 100644 --- a/arca-rar/tests/check-features.py +++ b/arca-rar/tests/check-features.py @@ -1,7 +1,8 @@ -"""Assert default RAR support, the opt-out and the encryption+write feature set. +"""Assert default RAR support, the opt-out and the encryption+write+parallel feature set. -The writer feature is intentional (new RAR5 archives); recovery and parallel -stay off, and no build may fall back to the crate defaults.""" +The writer feature is intentional (new RAR5 archives), and parallel lets it use +every core; recovery stays off, and no build may fall back to the crate +defaults. The crate is the vendored copy in arca-rar/vendor/rars.""" import pathlib import subprocess @@ -24,7 +25,8 @@ rar = [line for line in tree.splitlines() if line.startswith("rars v")] if enabled: assert rar, "RAR feature did not activate the reader" - assert all(line.removesuffix(" (*)") == "rars v0.10.2|encryption,write" for line in rar), rar + assert all(line.removesuffix(" (*)").split("|")[1] == "encryption,parallel,write" + and line.startswith("rars v0.10.2 (") and "vendor" in line for line in rar), rar else: assert not rar, f"{name} unexpectedly includes rars" - print(f"{name}: {'RAR with encryption,write only' if enabled else 'no rars dependency'}") + print(f"{name}: {'vendored RAR with encryption,parallel,write' if enabled else 'no rars dependency'}") diff --git a/arca-rar/tests/create-stress.py b/arca-rar/tests/create-stress.py index 8051295b..af892a64 100644 --- a/arca-rar/tests/create-stress.py +++ b/arca-rar/tests/create-stress.py @@ -565,7 +565,16 @@ def refused(name, args, needle, cwd=None): refused("hide names", ["create", str(area / "h.rar"), str(src), "-p", "secret", "--hide-names"], "") for codec in ("zstd", "deflate", "lzma2"): refused(f"codec {codec}", ["create", str(area / f"{codec}.rar"), str(src), "-c", codec], "auto or store") - refused("threads 4", ["create", str(area / "j.rar"), str(src), "-j", "4"], "sequential") + # -j picks the size of the pool the writer runs on; it is honored, + # not refused, and the archive it makes is an ordinary one. + threaded = area / "j.rar" + run = self.expect_ok(self.arca("create", str(threaded), str(src), "-j", "4"), "threads 4") + if "4 threads" not in run.stdout: + raise Failure(f"threads 4: summary does not say 4 threads: {run.stdout.strip()[-200:]!r}") + unrar = self.tools.get("unrar") + if unrar: + self.expect_ok(self.run([unrar["path"], "t", "-inul", str(threaded)]), "threads 4: unrar t") + detail["threads 4"] = run.stdout.strip()[-160:] refused("output inside source", ["create", str(src / "inside.rar"), str(src)], "output directory") refused("source is the output", ["create", str(existing), str(existing)], "") dup = area / "dup" @@ -582,7 +591,7 @@ def refused(name, args, needle, cwd=None): refused("missing source", ["create", str(area / "missing.rar"), str(area / "absent")], "") refused("missing parent", ["create", str(area / "no-such-dir" / "x.rar"), str(src)], "") refused("unsupported extension", ["create", str(area / "x.rar5"), str(src)], "") - self.expect_clean_dir(area, {"src", "existing.rar", "dup", "link.txt", "fifo"}) + self.expect_clean_dir(area, {"src", "existing.rar", "j.rar", "dup", "link.txt", "fifo"}) return detail def test_limits(self, area): diff --git a/arca-rar/tests/reader.rs b/arca-rar/tests/reader.rs index 423a8d92..ae77e176 100644 --- a/arca-rar/tests/reader.rs +++ b/arca-rar/tests/reader.rs @@ -224,3 +224,45 @@ fn refuses_destination_symlinks_and_preserves_external_files() { assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); } } + +#[test] +fn a_password_is_checked_on_the_first_encrypted_file_alone() { + for (name, password) in [ + ("encrypted.rar", "arca-test-only"), + ("headers.rar", "arca-test-only"), + ("encrypted_multivol.part1.rar", "password"), + ] { + let a = RarArchive::open(&fixture(name), Some(password)).unwrap(); + a.check_password(password, &|_, _, _| true) + .unwrap_or_else(|e| panic!("{name}: {e}")); + let wrong = RarArchive::open(&fixture(name), Some("wrong")) + .and_then(|b| b.check_password("wrong", &|_, _, _| true)); + assert!(wrong.is_err(), "{name}: a wrong password passed"); + } + let plain = RarArchive::open(&fixture("plain.rar"), None).unwrap(); + plain.check_password("anything", &|_, _, _| true).unwrap(); +} + +// Extracted files get the permissions any new file gets, not those of the +// owner-only temporary they were written through. +#[cfg(unix)] +#[test] +fn extracted_files_get_the_usual_permissions() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let a = RarArchive::open(&fixture("plain.rar"), None).unwrap(); + a.extract(dir.path(), &[], None, &|_, _, _| true, &|_| { + Conflict::Overwrite + }) + .unwrap(); + let fresh = dir.path().join("fresh"); + fs::File::create(&fresh).unwrap(); + let usual = fs::metadata(&fresh).unwrap().permissions().mode() & 0o666; + for entry in a.entries().iter().filter(|e| !e.is_dir) { + let mode = fs::metadata(dir.path().join(&entry.name)) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o666, usual, "{}", entry.name); + } +} diff --git a/arca-rar/vendor/PATCHES.md b/arca-rar/vendor/PATCHES.md new file mode 100644 index 00000000..c8ac1d32 --- /dev/null +++ b/arca-rar/vendor/PATCHES.md @@ -0,0 +1,42 @@ +# Vendored rars + +Source: crates.io `rars` **0.10.2**, Apache-2.0 (`COPYING` retained). +Archive SHA-256: +`0665518097392be04a322dec9eba41e88028f6aca85f5d61f8abb068691f2266`. + +Retained: `src/`, `tests/` (the crate's unit tests embed its fixtures), +`README.md`, `COPYING` and the normalized manifest. Omitted: `benches/` and +the `[[bench]]` targets that name them, and the original `Cargo.toml.orig`. + +It is used through `[patch.crates-io]` in the workspace manifest and is left +out of the workspace, as `arca-7z/vendor/sevenz-rust2` is; the manifest gains +an empty `[workspace]` so its own tests run with +`cargo test --manifest-path arca-rar/vendor/rars/Cargo.toml --lib --features write,encryption,parallel`. `arca-rar/fuzz` has +a lockfile of its own and still builds the crates.io copy: the reader it +exercises is unchanged here. + +Every change is in the RAR 5 writer and is marked `Arca:` in the source. The +reader, the format and the integrity checks are as published, and no RARLAB +source was consulted for any of it. + +- **Tree search for a whole member.** The optimal parse searched the first + block of a member with the binary-tree finder and then switched to hash + chains for the rest. A member with no history now keeps the tree for every + block (`codec/rar50/encoder.rs`, `streaming_blocks_with_allowance`), and a + member with no blocks returns before any finder is built. The test that + held the streaming writer byte-identical to the per-block reference encoder + (`parallel_streaming_blocks_are_byte_identical_to_serial_blocks`) now checks + that the writer's stream decodes to the input and is no larger, since the + reference still starts every block afresh. The writer's own output is still + identical whatever the number of jobs. +- **Nice length and parse passes per level.** `EncodeOptions` gains + `nice_length` and `parse_passes`, which default to the crate's constants. + Levels 2 and 3 stop at 64 bytes and price each block once and twice; + level 3 also searches 32 candidates rather than 128 + (`rar50/write/filter_policy.rs`). Levels 0, 1, 4 and 5 are unchanged. + +Measured with `arca create out.rar -l ` on 144 MiB of source and +executables (1,382 files), Windows 11, 16 threads, together with the +`parallel` feature and the 4 GiB writer budget set in `arca-rar`; every +archive passes `UnRAR t`. See the commit that brought this copy in for the +numbers. diff --git a/arca-rar/vendor/rars/.gitattributes b/arca-rar/vendor/rars/.gitattributes new file mode 100644 index 00000000..353faaa7 --- /dev/null +++ b/arca-rar/vendor/rars/.gitattributes @@ -0,0 +1,2 @@ +# Arca: the vendored crate is kept byte for byte as published. +* -text diff --git a/arca-rar/vendor/rars/COPYING b/arca-rar/vendor/rars/COPYING new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/arca-rar/vendor/rars/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arca-rar/vendor/rars/Cargo.toml b/arca-rar/vendor/rars/Cargo.toml new file mode 100644 index 00000000..c6223b8c --- /dev/null +++ b/arca-rar/vendor/rars/Cargo.toml @@ -0,0 +1,301 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.89" +name = "rars" +version = "0.10.2" +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "High-level Rust API for reading, extracting, writing, and repairing RAR archives." +homepage = "https://github.com/bitplane/rars" +documentation = "https://docs.rs/rars" +readme = "README.md" +keywords = [ + "rar", + "archive", + "compression", +] +categories = ["compression"] +license = "Apache-2.0" +repository = "https://github.com/bitplane/rars" + +[package.metadata.docs.rs] +all-features = true + +[features] +default = [ + "write", + "parallel", + "recovery", + "encryption", +] +encryption = [ + "dep:aes", + "dep:hmac", + "dep:sha1", + "dep:sha2", + "dep:zeroize", +] +parallel = ["dep:rayon"] +recovery = ["dep:getrandom"] +write = ["dep:getrandom"] + +[lib] +name = "rars" +path = "src/lib.rs" + +[[test]] +name = "builder_directories" +path = "tests/builder_directories.rs" + +[[test]] +name = "builder_file_comments" +path = "tests/builder_file_comments.rs" + +[[test]] +name = "builder_identity" +path = "tests/builder_identity.rs" + +[[test]] +name = "builder_metadata" +path = "tests/builder_metadata.rs" + +[[test]] +name = "builder_output" +path = "tests/builder_output.rs" + +[[test]] +name = "builder_symlinks" +path = "tests/builder_symlinks.rs" + +[[test]] +name = "builder_timestamps" +path = "tests/builder_timestamps.rs" + +[[test]] +name = "comment_options" +path = "tests/comment_options.rs" + +[[test]] +name = "crypto_api" +path = "tests/crypto_api.rs" + +[[test]] +name = "extraction_control" +path = "tests/extraction_control.rs" + +[[test]] +name = "legacy_builder_sources" +path = "tests/legacy_builder_sources.rs" + +[[test]] +name = "legacy_codec_adapters" +path = "tests/legacy_codec_adapters.rs" + +[[test]] +name = "legacy_name_encoding" +path = "tests/legacy_name_encoding.rs" + +[[test]] +name = "legacy_parallel_retention" +path = "tests/legacy_parallel_retention.rs" + +[[test]] +name = "legacy_writer_cancellation" +path = "tests/legacy_writer_cancellation.rs" + +[[test]] +name = "legacy_writer_errors" +path = "tests/legacy_writer_errors.rs" + +[[test]] +name = "member_selection" +path = "tests/member_selection.rs" + +[[test]] +name = "ppmd_regressions" +path = "tests/ppmd_regressions.rs" + +[[test]] +name = "rar13_fixtures" +path = "tests/rar13_fixtures.rs" + +[[test]] +name = "rar15_40_fixtures" +path = "tests/rar15_40_fixtures.rs" + +[[test]] +name = "rar50_fixtures" +path = "tests/rar50_fixtures.rs" + +[[test]] +name = "rar50_golden" +path = "tests/rar50_golden.rs" + +[[test]] +name = "rar50_split_limits" +path = "tests/rar50_split_limits.rs" + +[[test]] +name = "rarvm_regressions" +path = "tests/rarvm_regressions.rs" + +[[test]] +name = "reader_32bit_lengths" +path = "tests/reader_32bit_lengths.rs" + +[[test]] +name = "reader_cancellation" +path = "tests/reader_cancellation.rs" + +[[test]] +name = "reader_dictionary_limit" +path = "tests/reader_dictionary_limit.rs" + +[[test]] +name = "reader_failure_output" +path = "tests/reader_failure_output.rs" + +[[test]] +name = "reader_failure_paths" +path = "tests/reader_failure_paths.rs" + +[[test]] +name = "reader_header_limits" +path = "tests/reader_header_limits.rs" + +[[test]] +name = "reader_output_limit" +path = "tests/reader_output_limit.rs" + +[[test]] +name = "reader_scratch" +path = "tests/reader_scratch.rs" + +[[test]] +name = "reader_signatures" +path = "tests/reader_signatures.rs" + +[[test]] +name = "reader_source" +path = "tests/reader_source.rs" + +[[test]] +name = "reader_stored_lengths" +path = "tests/reader_stored_lengths.rs" + +[[test]] +name = "reader_verification" +path = "tests/reader_verification.rs" + +[[test]] +name = "reader_workspace" +path = "tests/reader_workspace.rs" + +[[test]] +name = "repair_cancellation" +path = "tests/repair_cancellation.rs" + +[[test]] +name = "rewrite_settings" +path = "tests/rewrite_settings.rs" + +[[test]] +name = "rewrite_staging" +path = "tests/rewrite_staging.rs" + +[[test]] +name = "source_consistency" +path = "tests/source_consistency.rs" + +[[test]] +name = "write_progress" +path = "tests/write_progress.rs" + +[[test]] +name = "writer_aggregate_limit" +path = "tests/writer_aggregate_limit.rs" + +[[test]] +name = "writer_preparation_limit" +path = "tests/writer_preparation_limit.rs" + +[[test]] +name = "writer_spool_limit" +path = "tests/writer_spool_limit.rs" + +[dependencies.aes] +version = "0.9" +features = ["zeroize"] +optional = true + +[dependencies.aho-corasick] +version = "1.1" + +[dependencies.getrandom] +version = "0.4" +optional = true + +[dependencies.hmac] +version = "0.13" +optional = true + +[dependencies.sha1] +version = "0.11" +optional = true + +[dependencies.sha2] +version = "0.11" +optional = true + +[dependencies.zeroize] +version = "1.8" +features = [ + "alloc", + "zeroize_derive", +] +optional = true +default-features = false + +[dev-dependencies.criterion] +version = "0.8" +features = ["cargo_bench_support"] +default-features = false + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies.getrandom] +version = "0.4" +features = ["wasm_js"] +optional = true + +[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies.rayon] +version = "1.12.0" +optional = true + +[target."cfg(unix)".dev-dependencies.libc] +version = "0.2" + +[lints.clippy] +dbg_macro = "warn" +todo = "warn" +unimplemented = "warn" + +[lints.rust] +unsafe_code = "forbid" +unused_must_use = "deny" + +# Arca: tested on its own with --manifest-path, outside the workspace. +[workspace] diff --git a/arca-rar/vendor/rars/README.md b/arca-rar/vendor/rars/README.md new file mode 100644 index 00000000..b3e15eed --- /dev/null +++ b/arca-rar/vendor/rars/README.md @@ -0,0 +1,41 @@ +# rars + +A Rust implementation of RAR. + +* ðŸ�  [home](https://bitplane.net/dev/rust/rars) +* 📦 [downloads](https://github.com/bitplane/rars/releases) + * 🦀 [crates](https://crates.io/crates/rars) + * ðŸ�� [pypi](https://pypi.org/project/rars) + * 📜 [npm](https://www.npmjs.com/package/@bitplane/rars) +* ðŸ�± [source](https://github.com/bitplane/rars) + * 📃 [spec](https://github.com/bitplane/rar-research) + +`rars` is free software for compression, decompression and recovery of RAR +archives. It supports all the archive types I could find - from the early `RE~^` +ones from the DOS days all the way through to RAR 7. It comes with a Rust +library, a CLI, Python and TypeScript bindings. + +It started as an +[agentic development experiment](https://bitplane.net/log/2026/05/rars/), and +has since matured as it gained some users. It's still a bit slower than WinRAR, +uses more memory and has slightly worse compression. It could use more testing +at volume, too. Other than that it's in pretty good shape. + +## Install + +```sh +cargo install rars-cli # the CLI +cargo add rars # the Rust library +pip install rars # Python +npm install @bitplane/rars # JavaScript, as WebAssembly for Node or the browser +``` + +Then `rars --help`. For example: + +```sh +rars x archive.rar out/ +rars a --format rar29 --solid archive.rar files... +rars a --format rar70 --volume-size 10m archive.part1.rar files... +``` + +Licensed under the [Apache License, Version 2.0](COPYING). diff --git a/arca-rar/vendor/rars/src/atomic64.rs b/arca-rar/vendor/rars/src/atomic64.rs new file mode 100644 index 00000000..f4a1ecca --- /dev/null +++ b/arca-rar/vendor/rars/src/atomic64.rs @@ -0,0 +1,125 @@ +//! A 64-bit atomic counter for targets without 64-bit atomics. +//! +//! 32-bit PowerPC, MIPS and older ARM have no `std::sync::atomic::AtomicU64`. +//! Byte budgets can exceed 4 GiB there too, so a narrower atomic would wrap; +//! those targets get a mutex with the same interface instead. + +#[cfg(target_has_atomic = "64")] +pub(crate) use std::sync::atomic::AtomicU64; + +#[cfg(not(target_has_atomic = "64"))] +pub(crate) use fallback::AtomicU64; + +// Also built for tests so 64-bit hosts exercise it. +#[cfg(any(test, not(target_has_atomic = "64")))] +mod fallback { + use std::sync::atomic::Ordering; + use std::sync::Mutex; + + #[derive(Debug, Default)] + pub(crate) struct AtomicU64(Mutex); + + // Which methods are live depends on the enabled features. + #[allow(dead_code)] + impl AtomicU64 { + pub(crate) const fn new(value: u64) -> Self { + Self(Mutex::new(value)) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, u64> { + self.0 + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub(crate) fn load(&self, _: Ordering) -> u64 { + *self.lock() + } + + pub(crate) fn store(&self, value: u64, _: Ordering) { + *self.lock() = value; + } + + pub(crate) fn fetch_add(&self, value: u64, _: Ordering) -> u64 { + let mut current = self.lock(); + let previous = *current; + *current = previous.wrapping_add(value); + previous + } + + pub(crate) fn fetch_sub(&self, value: u64, _: Ordering) -> u64 { + let mut current = self.lock(); + let previous = *current; + *current = previous.wrapping_sub(value); + previous + } + + pub(crate) fn compare_exchange_weak( + &self, + expected: u64, + new: u64, + _: Ordering, + _: Ordering, + ) -> Result { + let mut current = self.lock(); + if *current == expected { + *current = new; + Ok(expected) + } else { + Err(*current) + } + } + } + + #[cfg(test)] + mod tests { + use super::AtomicU64; + use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed}; + + #[test] + fn counts_past_32_bits() { + let value = AtomicU64::new(u64::from(u32::MAX)); + assert_eq!(value.fetch_add(2, Relaxed), u64::from(u32::MAX)); + assert_eq!(value.load(Relaxed), 1 << 32 | 1); + assert_eq!(value.fetch_sub(1 << 32, AcqRel), 1 << 32 | 1); + assert_eq!(value.load(Relaxed), 1); + value.store(5 << 32, Relaxed); + assert_eq!(value.load(Relaxed), 5 << 32); + } + + #[test] + fn compare_exchange_reports_the_current_value() { + let value = AtomicU64::new(7); + assert_eq!(value.compare_exchange_weak(6, 9, AcqRel, Acquire), Err(7)); + assert_eq!(value.compare_exchange_weak(7, 9, AcqRel, Acquire), Ok(7)); + assert_eq!(value.load(Relaxed), 9); + } + + #[test] + fn concurrent_reservations_never_exceed_the_limit() { + let used = AtomicU64::new(0); + let limit = 1000; + let granted = AtomicU64::new(0); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + for _ in 0..500 { + let mut current = used.load(Acquire); + while let Some(next) = Some(current + 3).filter(|n| *n <= limit) { + match used.compare_exchange_weak(current, next, AcqRel, Acquire) { + Ok(_) => { + granted.fetch_add(3, Relaxed); + break; + } + Err(actual) => current = actual, + } + } + } + }); + } + }); + assert_eq!(used.load(Relaxed), 999); + assert_eq!(granted.load(Relaxed), 999); + } + } +} diff --git a/arca-rar/vendor/rars/src/builder.rs b/arca-rar/vendor/rars/src/builder.rs new file mode 100644 index 00000000..11c1d4c0 --- /dev/null +++ b/arca-rar/vendor/rars/src/builder.rs @@ -0,0 +1,2736 @@ +//! One archive builder for every format. +//! +//! The writers underneath are per-family and take different entry types, and +//! choosing between them is the same twenty decisions in every binding. This +//! module makes that choice once: name the format, add members, ask for bytes +//! or a file or a volume set. The Python extension and the WebAssembly package +//! are thin translations of the type below, which is the point of it living +//! here rather than in one of them. + +use crate::write_progress::{check_cancelled, EmissionWriter, ProgressReporter, ResourceProgress}; +use crate::{ + rar13, rar15_40, rar50, ArchiveFamily, ArchiveVersion, EntrySource, Error, FeatureSet, Result, + WriteProgress, WriterResources, +}; +use std::fs; +use std::io::Write; +use std::path::Path; + +struct ConvertedEntries { + entries: Vec, + _charge: Option, +} + +/// The DOS archive bit, which is what a member gets when the caller offers no +/// mode of its own. Zero would be legal and would read as "no attributes", +/// which no real RAR writer emits. +const DOS_ARCHIVE_ATTR: u32 = 0x20; + +// Host IDs are format-specific: legacy RAR uses 3 for Unix, RAR 5 uses 1. +// Keep the host paired with its attributes. Merely correcting the RAR 5 ID +// while retaining DOS_ARCHIVE_ATTR would make reference extractors interpret +// 0x20 as Unix permissions, producing an unexpectedly restricted file. +const RAR15_HOST_UNIX: u8 = 3; +const RAR50_HOST_UNIX: u64 = 1; + +type PendingArchive = + crate::pending_archive::PendingArchive>; + +impl PendingArchive { + fn path_charge( + resources: &WriterResources, + capacity: usize, + ) -> Result> { + let mut charge = resources.execution_charge(); + if let Some(charge) = &mut charge { + charge.grow_to(capacity as u64)?; + } + Ok(charge) + } + + fn with_resources(destination: &Path, resources: &WriterResources) -> Result<(Self, fs::File)> { + Self::with_admission(destination, |capacity| { + Self::path_charge(resources, capacity) + }) + } + + #[cfg(test)] + fn with_sequence( + destination: &Path, + resources: &WriterResources, + next_sequence: impl FnMut() -> u64, + ) -> Result<(Self, fs::File)> { + Self::create_with_sequence( + destination, + |capacity| Self::path_charge(resources, capacity), + next_sequence, + ) + } +} + +#[derive(Debug, Clone, Copy)] +enum EntryAttributes { + Dos(u64), + Unix(u32), +} + +/// A member queued for writing. +/// +/// The bytes are either held directly or fetched from an [`EntrySource`] when +/// the writer reaches the member. Sources are what keep a large archive off the +/// heap. Legacy single archives pass sources to the per-member writers; +/// legacy volume collection still materializes its one input. +#[derive(Debug, Clone)] +struct BuilderEntry { + id: usize, + name: Vec, + data: Vec, + source: Option, + is_directory: bool, + mtime: Option, + mtime_nanoseconds: Option, + file_times: Option, + legacy_extended_times: Option>, + legacy_unicode_name: Option>, + file_comment: Option>, + encryption: Option, + redirection: Option, + redirection_size: Option, + attributes: EntryAttributes, +} + +#[derive(Debug, Clone)] +struct EntryEncryption { + data_password: Option>, + comment_password: Option>, +} + +impl BuilderEntry { + fn attributes(&self) -> u64 { + match self.attributes { + // add_bytes/add_source can receive permission bits alone, whereas + // add_path supplies a complete st_mode. Supply regular-file type + // bits only when absent so both forms describe Unix metadata. + EntryAttributes::Unix(mode) if mode & 0o170000 == 0 => u64::from(mode | 0o100000), + EntryAttributes::Unix(mode) => u64::from(mode), + EntryAttributes::Dos(attributes) => attributes, + } + } + + fn rar50_attr(&self) -> u64 { + self.attributes() + } + + fn rar15_attr(&self) -> u32 { + // set_dos_attributes validates the target family's field width. + self.attributes() as u32 + } + + fn rar13_attr(&self) -> u8 { + match self.attributes { + EntryAttributes::Dos(attributes) => attributes as u8, + EntryAttributes::Unix(_) => DOS_ARCHIVE_ATTR as u8, + } + } + + fn rar50_host_os(&self) -> u64 { + if matches!(self.attributes, EntryAttributes::Unix(_)) { + RAR50_HOST_UNIX + } else { + 0 // Windows host, DOS attributes. + } + } + + fn rar15_host_os(&self) -> u8 { + // The RAR 1.5 writer still downgrades Unix metadata to DOS for old + // extractor compatibility; RAR 2.x-4.x retain the Unix host and mode. + if matches!(self.attributes, EntryAttributes::Unix(_)) { + RAR15_HOST_UNIX + } else { + 0 // MS-DOS host, DOS attributes. + } + } +} + +/// Assembles an archive from members added one at a time. +/// +/// Nothing is encoded until [`to_bytes`](Self::to_bytes), +/// [`write_to_path`](Self::write_to_path) or +/// [`build_volumes`](Self::build_volumes) is called, so the same builder can be +/// written more than once and to more than one format. +/// +/// ```no_run +/// # fn main() -> rars::Result<()> { +/// let mut builder = rars::Builder::new(rars::ArchiveVersion::Rar50); +/// builder.add_bytes(b"hello.txt".to_vec(), b"hello".to_vec(), None, None)?; +/// let archive = builder.to_bytes()?; +/// # let _ = archive; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct Builder { + format: ArchiveVersion, + compression: Option, + store: bool, + solid: bool, + password: Option>, + encrypt_headers: bool, + comment: Option>, + comment_password: Option>, + recovery_percent: Option, + locked: bool, + quick_open: bool, + archive_metadata: Option, + legacy_archive_comment_metadata: Option<(u32, u8)>, + legacy_unpack_version: Option, + rar50_dictionary_size: Option, + volume_size: Option, + entries: Vec, + next_entry_id: usize, + allow_duplicate_names: bool, +} + +impl Builder { + /// A builder for `format`, with the writer's own default compression level. + pub fn new(format: ArchiveVersion) -> Self { + Self { + format, + compression: None, + store: false, + solid: false, + password: None, + encrypt_headers: false, + comment: None, + comment_password: None, + recovery_percent: None, + locked: false, + quick_open: false, + archive_metadata: None, + legacy_archive_comment_metadata: None, + legacy_unpack_version: None, + rar50_dictionary_size: None, + volume_size: None, + entries: Vec::new(), + next_entry_id: 0, + allow_duplicate_names: false, + } + } + + /// The format this builder writes. + pub fn format(&self) -> ArchiveVersion { + self.format + } + + /// Compression level, 0 to 5. `None` leaves the writer's default in place. + pub fn compression_level(mut self, level: Option) -> Self { + self.compression = level; + self + } + + /// Store members without compressing them. Storing is a compression level, + /// not a kind of member, so this wins over any level set above. + pub fn store(mut self, store: bool) -> Self { + self.store = store; + self + } + + /// Compress members against each other rather than independently. + pub fn solid(mut self, solid: bool) -> Self { + self.solid = solid; + self + } + + /// Encrypt member data with `password`. + pub fn password(mut self, password: Option>) -> Self { + self.password = password; + self + } + + /// Encrypt the headers as well as the data, hiding the member names. Needs + /// a password. + pub fn header_encryption(mut self, encrypt: bool) -> Self { + self.encrypt_headers = encrypt; + self + } + + /// An archive comment. + pub fn comment(mut self, comment: Option>) -> Self { + if comment.is_none() { + self.legacy_archive_comment_metadata = None; + self.comment_password = None; + } + self.comment = comment; + self + } + + pub(crate) fn legacy_unpack_version(mut self, version: Option) -> Self { + self.legacy_unpack_version = version; + self + } + + pub(crate) fn rar50_dictionary_size(mut self, size: Option) -> Self { + self.rar50_dictionary_size = size; + self + } + + pub(crate) fn legacy_archive_comment_metadata(mut self, metadata: Option<(u32, u8)>) -> Self { + self.legacy_archive_comment_metadata = metadata; + self + } + + /// Add a recovery record covering this percentage of the archive. + pub fn recovery_percent(mut self, percent: Option) -> Self { + self.recovery_percent = percent; + self + } + + /// Split the output into volumes of at most this many bytes. + /// [`build_volumes`](Self::build_volumes) requires it; the single-archive + /// entry points refuse to run while it is set. + pub fn volume_size(mut self, size: Option) -> Self { + self.volume_size = size; + self + } + + /// How many members are queued. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether no members are queued. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// The queued member names, in the order they will be written. + pub fn names(&self) -> impl Iterator { + self.entries.iter().map(|entry| entry.name.as_slice()) + } + + /// Queue `data` under `name`. + /// + /// `name` is archive identity, not display text: legacy byte names are + /// accepted unchanged. RAR5 names use its wire UTF-8 encoding; use + /// [`crate::filename::encode_rar50`] for native Unix bytes and Unix `mode` + /// for those nonportable mapped names. [`Self::add_path`] handles this + /// conversion automatically for filesystem inputs. + /// + /// `mode` is a Unix mode, with optional file-type bits. Without a mode, + /// the member uses DOS archive attributes and the extractor's default permissions. + /// RAR 1.3-1.5 output uses DOS metadata even when a Unix mode is supplied. + pub fn add_bytes( + &mut self, + name: Vec, + data: Vec, + mtime: Option, + mode: Option, + ) -> Result<()> { + self.push(BuilderEntry { + id: 0, + name: self.validate_name(name)?, + data, + source: None, + is_directory: false, + mtime, + mtime_nanoseconds: None, + file_times: None, + legacy_extended_times: None, + legacy_unicode_name: None, + file_comment: None, + encryption: None, + redirection: None, + redirection_size: None, + attributes: mode.map_or( + EntryAttributes::Dos(u64::from(DOS_ARCHIVE_ATTR)), + EntryAttributes::Unix, + ), + }) + } + + /// Queue a member whose bytes are fetched from `source` when the writer + /// reaches it. + /// `mode` has the same meaning as in [`add_bytes`](Self::add_bytes). + pub fn add_source( + &mut self, + name: Vec, + source: EntrySource, + mtime: Option, + mode: Option, + ) -> Result<()> { + self.push(BuilderEntry { + id: 0, + name: self.validate_name(name)?, + data: Vec::new(), + source: Some(source), + is_directory: false, + mtime, + mtime_nanoseconds: None, + file_times: None, + legacy_extended_times: None, + legacy_unicode_name: None, + file_comment: None, + encryption: None, + redirection: None, + redirection_size: None, + attributes: mode.map_or( + EntryAttributes::Dos(u64::from(DOS_ARCHIVE_ATTR)), + EntryAttributes::Unix, + ), + }) + } + + /// Replace a queued file's payload while retaining its position and metadata. + /// Directories and redirections cannot be changed into payload entries. + /// The source is opened only when writing. Errors leave the entry unchanged. + pub fn set_source(&mut self, name: &[u8], source: EntrySource) -> Result<()> { + let id = self.entry_id(name)?; + self.set_source_by_id(id, source) + } + + /// Identity-based variant of [`set_source`](Self::set_source). + pub fn set_source_by_id(&mut self, id: usize, source: EntrySource) -> Result<()> { + let index = self.index_by_id(id)?; + let name = self.entries[index].name.clone(); + + let entry = &mut self.entries[index]; + if entry.is_directory || entry.redirection.is_some() { + return Err( + Error::InvalidArgument("payload source requires a regular entry") + .at_entry(name.to_vec(), "replacing payload"), + ); + } + entry.data = Vec::new(); + entry.source = Some(source); + Ok(()) + } + + /// Queue an explicit RAR1.3–4.x or RAR5/7 directory, including an empty one. + /// `mode` supplies Unix permission bits; otherwise DOS directory flags are used. + /// Legacy timestamps use raw DOS values; legacy volume output is unsupported. + pub fn add_directory( + &mut self, + name: Vec, + mtime: Option, + mode: Option, + ) -> Result<()> { + if matches!( + self.format, + ArchiveVersion::Rar13 | ArchiveVersion::Rar14 | ArchiveVersion::Rar15 + ) && mode.is_some() + { + return Err(Error::InvalidArgument( + "RAR1.x directory output requires DOS attributes", + )); + } + if self.format.family() != ArchiveFamily::Rar50Plus && self.volume_size.is_some() { + return Err(Error::InvalidArgument( + "explicit directories require RAR5/7 or single-archive RAR1.3–4.x output", + )); + } + self.push(BuilderEntry { + id: 0, + name: self.validate_name(name)?, + data: Vec::new(), + source: None, + is_directory: true, + mtime, + mtime_nanoseconds: None, + file_times: None, + legacy_extended_times: None, + legacy_unicode_name: None, + file_comment: None, + encryption: None, + redirection: None, + redirection_size: None, + attributes: mode.map_or(EntryAttributes::Dos(0x10), |mode| { + EntryAttributes::Unix((mode & 0o7777) | 0o040000) + }), + }) + } + + /// Queue a RAR2.0–4.x or RAR5/7 Unix symbolic link without following its target. + /// Name and target use archive wire bytes. Relative targets remain unchanged + /// through renames; the directory flag describes the target, not this entry. + /// Legacy targets use native bytes and cannot carry a target-directory flag. + pub fn add_unix_symlink( + &mut self, + name: Vec, + target: Vec, + target_is_directory: bool, + mtime: Option, + mode: Option, + ) -> Result<()> { + if matches!( + self.format, + ArchiveVersion::Rar20 + | ArchiveVersion::Rar29 + | ArchiveVersion::Rar30 + | ArchiveVersion::Rar40 + ) { + if self.volume_size.is_some() + || target_is_directory + || target.is_empty() + || target.contains(&0) + { + return Err(Error::InvalidArgument("legacy symbolic links require single-archive output, a nonempty native target without NUL and no target-directory flag")); + } + return self.add_bytes( + name, + target, + mtime, + Some(0o120000 | (mode.unwrap_or(0o777) & 0o7777)), + ); + } + let link = rar50::FileRedirection { + redirection_type: 1, + flags: u64::from(target_is_directory), + target_name: target, + }; + if self.format.family() != ArchiveFamily::Rar50Plus + || self.volume_size.is_some() + || !link.is_supported_unix_symlink() + { + return Err(Error::InvalidArgument("symbolic links require single-archive RAR5/7 output and a nonempty UTF-8 wire target without NUL")); + } + self.push(BuilderEntry { + id: 0, + name: self.validate_name(name)?, + data: Vec::new(), + source: None, + is_directory: false, + mtime, + mtime_nanoseconds: None, + file_times: None, + legacy_extended_times: None, + legacy_unicode_name: None, + file_comment: None, + encryption: None, + redirection: Some(link), + redirection_size: None, + attributes: EntryAttributes::Unix(0o120000 | (mode.unwrap_or(0o777) & 0o7777)), + }) + } + + /// Retain a supported RAR5 redirection and its header metadata, without + /// reading a target. Archive-internal targets are checked before output. + pub fn add_archive_redirection(&mut self, member: &crate::ArchiveMember) -> Result<()> { + let link = member + .supported_redirection() + .ok_or(Error::InvalidArgument("unsupported redirection metadata"))?; + if self.format.family() != ArchiveFamily::Rar50Plus || self.volume_size.is_some() { + return Err(Error::InvalidArgument( + "redirections require single-archive RAR5/7 output", + )); + } + let meta = &member.meta; + self.push(BuilderEntry { + id: 0, + name: self.validate_name(meta.name.clone())?, + data: Vec::new(), + source: None, + is_directory: meta.is_directory, + mtime: meta.file_time, + mtime_nanoseconds: meta.mtime_refinement.map(|time| time.nanoseconds), + file_times: None, + legacy_extended_times: None, + legacy_unicode_name: None, + file_comment: None, + encryption: None, + redirection: Some(link.clone()), + redirection_size: Some(meta.unpacked_size), + attributes: if meta.host_os == Some(1) { + EntryAttributes::Unix(meta.file_attr as u32) + } else { + EntryAttributes::Dos(meta.file_attr) + }, + }) + } + + fn check_redirection_targets(&self) -> Result<()> { + if !self.entries.iter().any(|entry| { + entry + .redirection + .as_ref() + .is_some_and(|link| link.redirection_type >= 4) + }) { + return Ok(()); + } + let mut preceding = std::collections::HashMap::new(); + for entry in &self.entries { + let size = entry.redirection_size.unwrap_or(match &entry.source { + Some(source) => source.len()?, + None => entry.data.len() as u64, + }); + if let Some(link) = &entry.redirection { + if link.redirection_type >= 4 && preceding.get(&link.target_name) != Some(&size) { + return Err(Error::InvalidArgument("hard-link and file-copy targets must precede the link and have the same size") + .at_entry(entry.name.clone(), "validating redirection target")); + } + } + if !entry.is_directory + && entry + .redirection + .as_ref() + .is_none_or(|link| link.redirection_type >= 4) + { + preceding.insert(entry.name.clone(), size); + } + } + Ok(()) + } + + /// Set per-member encryption for RAR1.3–4.x or RAR5/7 output. + /// Legacy output supports data passwords only, for single archives. + /// Explicit None passwords retain plaintext even when the builder has a default password. + pub fn set_entry_encryption( + &mut self, + name: &[u8], + data_password: Option>, + comment_password: Option>, + ) -> Result<()> { + let id = self.entry_id(name)?; + self.set_entry_encryption_by_id(id, data_password, comment_password) + } + + /// Identity-based variant of [`set_entry_encryption`](Self::set_entry_encryption). + pub fn set_entry_encryption_by_id( + &mut self, + id: usize, + data_password: Option>, + comment_password: Option>, + ) -> Result<()> { + let index = self.index_by_id(id)?; + + if (self.format.family() != ArchiveFamily::Rar50Plus + && (comment_password.is_some() || self.volume_size.is_some())) + || data_password.as_ref().is_some_and(Vec::is_empty) + || comment_password.as_ref().is_some_and(Vec::is_empty) + { + return Err(Error::InvalidArgument( + "per-entry encryption requires supported output and nonempty passwords; legacy output supports data encryption in single archives only", + )); + } + let entry = &mut self.entries[index]; + entry.encryption = Some(EntryEncryption { + data_password, + comment_password, + }); + Ok(()) + } + + /// Retain RAR5/7 archive-level metadata and advisory lock/index settings. + pub fn archive_metadata( + mut self, + metadata: Option, + locked: bool, + quick_open: bool, + ) -> Result { + if self.format.family() != ArchiveFamily::Rar50Plus { + return Err(Error::InvalidArgument( + "archive metadata settings require RAR5/7", + )); + } + if let Some(metadata) = &metadata { + rar50::write::headers::retained_archive_metadata( + metadata, + &WriterResources::default(), + )?; + } + self.archive_metadata = metadata; + self.locked = locked; + self.quick_open = quick_open; + Ok(self) + } + + /// Encrypt only the archive comment with this password. + pub fn archive_comment_password(mut self, password: Option>) -> Self { + self.comment_password = password; + self + } + + /// Retain a validated native Unicode name record for a queued legacy entry. + /// Renaming re-encodes Unicode; unchanged names retain their original bytes. + pub fn set_legacy_unicode_name(&mut self, name: &[u8], raw: Vec) -> Result<()> { + let id = self.entry_id(name)?; + self.set_legacy_unicode_name_by_id(id, raw) + } + + /// Identity-based variant of [`set_legacy_unicode_name`](Self::set_legacy_unicode_name). + pub fn set_legacy_unicode_name_by_id(&mut self, id: usize, raw: Vec) -> Result<()> { + let index = self.index_by_id(id)?; + let name = self.entries[index].name.clone(); + + if !matches!( + self.format, + ArchiveVersion::Rar20 + | ArchiveVersion::Rar29 + | ArchiveVersion::Rar30 + | ArchiveVersion::Rar40 + ) || self.volume_size.is_some() + { + return Err(Error::InvalidArgument( + "legacy Unicode names require single-archive RAR2–4 output", + )); + } + rar15_40::validate_unicode_name(&raw, &name)?; + let entry = &mut self.entries[index]; + entry.legacy_unicode_name = Some(raw); + Ok(()) + } + + /// Retain a native RAR2.9–4.x extended-time record, including archival time. + /// DOS values and fractional precision are copied without timezone conversion. + /// Unsupported output or invalid records leave the entry unchanged. + pub fn set_legacy_extended_times(&mut self, name: &[u8], raw: Option>) -> Result<()> { + let id = self.entry_id(name)?; + self.set_legacy_extended_times_by_id(id, raw) + } + + /// Identity-based variant of [`set_legacy_extended_times`](Self::set_legacy_extended_times). + pub fn set_legacy_extended_times_by_id( + &mut self, + id: usize, + raw: Option>, + ) -> Result<()> { + let index = self.index_by_id(id)?; + + if !matches!( + self.format, + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 + ) || self.volume_size.is_some() + { + return Err(Error::InvalidArgument( + "legacy extended timestamps require single-archive RAR2.9–4.x output", + )); + } + if let Some(raw) = &raw { + crate::file_times::validate_legacy_extended_times(raw)?; + } + let entry = &mut self.entries[index]; + entry.legacy_extended_times = raw; + Ok(()) + } + + /// Set complete RAR5/7 timestamps without narrowing FILETIME or discarding fractions. + pub fn set_file_times(&mut self, name: &[u8], times: Option) -> Result<()> { + let id = self.entry_id(name)?; + self.set_file_times_by_id(id, times) + } + + /// Identity-based variant of [`set_file_times`](Self::set_file_times). + pub fn set_file_times_by_id( + &mut self, + id: usize, + times: Option, + ) -> Result<()> { + let index = self.index_by_id(id)?; + + if self.format.family() != ArchiveFamily::Rar50Plus { + return Err(Error::InvalidArgument( + "complete file times require RAR5/7 output", + )); + } + if let Some(times) = times { + times.encode()?; + } + let entry = &mut self.entries[index]; + if times.is_some_and(|times| times.modified.is_some()) { + entry.mtime = None; + } + // Preserve an existing fractional mtime when adding only ctime/atime. + let times = if let (Some(mut times), Some(seconds), Some(nanoseconds)) = + (times, entry.mtime, entry.mtime_nanoseconds) + { + times.modified = Some(crate::FileTimestamp::Unix { + seconds, + nanoseconds, + }); + times.encode()?; + entry.mtime = None; + Some(times) + } else { + times + }; + entry.mtime_nanoseconds = None; + entry.file_times = times; + Ok(()) + } + + /// Add nanosecond precision to a queued RAR5/7 modification time. + /// The member must already have whole seconds; legacy output is unsupported. + pub fn set_mtime_nanoseconds(&mut self, name: &[u8], nanoseconds: u32) -> Result<()> { + let id = self.entry_id(name)?; + self.set_mtime_nanoseconds_by_id(id, nanoseconds) + } + + /// Identity-based variant of [`set_mtime_nanoseconds`](Self::set_mtime_nanoseconds). + pub fn set_mtime_nanoseconds_by_id(&mut self, id: usize, nanoseconds: u32) -> Result<()> { + let index = self.index_by_id(id)?; + + if self.format.family() != crate::ArchiveFamily::Rar50Plus || nanoseconds >= 1_000_000_000 { + return Err(Error::InvalidArgument( + "nanosecond modification times require RAR5/7 and a fraction below one second", + )); + } + let entry = &mut self.entries[index]; + if entry.mtime.is_none() { + return Err(Error::InvalidArgument( + "fractional modification time requires whole seconds", + )); + } + entry.mtime_nanoseconds = Some(nanoseconds); + Ok(()) + } + + /// Sets or removes a queued member's comment. Comments follow the entry + /// through renames and reordering; `Some(Vec::new())` is an explicit empty comment. + /// RAR3/4 and volume output do not support comments. Errors leave the entry unchanged. + pub fn set_file_comment(&mut self, name: &[u8], comment: Option>) -> Result<()> { + let id = self.entry_id(name)?; + self.set_file_comment_by_id(id, comment) + } + + /// Identity-based variant of [`set_file_comment`](Self::set_file_comment). + pub fn set_file_comment_by_id(&mut self, id: usize, comment: Option>) -> Result<()> { + let index = self.index_by_id(id)?; + + if comment.is_some() { + crate::write_plan::validate_option( + self.format, + crate::write_plan::WriterOption::FileComment, + crate::write_plan::PlanShape::new().volumes(self.volume_size.is_some()), + )?; + } + let entry = &mut self.entries[index]; + entry.file_comment = comment; + Ok(()) + } + + /// Set DOS/Windows attributes for a queued entry, replacing any Unix mode. + /// + /// The output host is paired with these flags. Attribute width is checked + /// against the target format (8 bits for RAR 1.3/1.4, 32 for RAR 1.5-4.x). + /// The directory bit must agree with the entry kind: changing attributes + /// cannot change a file into a directory. Errors leave metadata unchanged. + pub fn set_dos_attributes(&mut self, name: &[u8], attributes: u64) -> Result<()> { + let id = self.entry_id(name)?; + self.set_dos_attributes_by_id(id, attributes) + } + + /// Identity-based variant of [`set_dos_attributes`](Self::set_dos_attributes). + pub fn set_dos_attributes_by_id(&mut self, id: usize, attributes: u64) -> Result<()> { + let index = self.index_by_id(id)?; + + use crate::ArchiveFamily; + let max = match self.format.family() { + ArchiveFamily::Rar13 => u64::from(u8::MAX), + ArchiveFamily::Rar15To40 => u64::from(u32::MAX), + ArchiveFamily::Rar50Plus => u64::MAX, + }; + if attributes > max { + return Err(Error::InvalidArgument( + "DOS attributes exceed the target format's field width", + )); + } + let entry = &mut self.entries[index]; + if entry + .redirection + .as_ref() + .is_some_and(|link| link.redirection_type == 1) + { + return Err(Error::InvalidArgument( + "Unix symbolic links require Unix attributes", + )); + } + if (attributes & 0x10 != 0) != entry.is_directory { + return Err(Error::InvalidArgument( + "DOS directory attributes must match the entry kind", + )); + } + entry.attributes = EntryAttributes::Dos(attributes); + Ok(()) + } + + /// Queue a file, or every file under a directory, named `archive_name` in + /// the archive. Children are added in sorted order so the same tree gives + /// the same archive twice. + /// `archive_name` and child names use native filename bytes on Unix and + /// UTF-8 elsewhere. Legacy output preserves these bytes; RAR5/7 applies + /// the reversible Unix byte mapping. Member lookup uses the encoded name. + /// + /// Symlinks are refused rather than followed, at the root and at every + /// level below it: a link is a name for someone else's file, and copying + /// what it points at is not what the caller asked for. + /// Other special files are refused instead of being silently omitted. + pub fn add_path(&mut self, path: &Path, archive_name: &[u8]) -> Result<()> { + let link_meta = fs::symlink_metadata(path)?; + if link_meta.file_type().is_symlink() { + return Err(Error::AtEntry { + name: archive_name.to_vec(), + operation: "adding", + source: Box::new(Error::InputSymlink), + }); + } + let meta = fs::metadata(path)?; + if meta.is_dir() { + let mut children = fs::read_dir(path)?.collect::, _>>()?; + children.sort_by_key(|entry| entry.file_name()); + for child in children { + let file_name = child.file_name(); + let file_name = crate::filename::native_bytes(&file_name)?; + let mut child_name = archive_name.to_vec(); + child_name.push(b'/'); + child_name.extend_from_slice(file_name); + self.add_path(&child.path(), &child_name)?; + } + } else if meta.is_file() { + // Validate native path syntax before a RAR5 mapping marker can + // precede the first component of the encoded archive name. + crate::filename::validate_relative(archive_name)?; + let name = if cfg!(unix) && self.format.family() == ArchiveFamily::Rar50Plus { + crate::filename::encode_rar50(archive_name).into_owned() + } else { + archive_name.to_vec() + }; + self.add_source(name, EntrySource::from_path(path), None, unix_mode(&meta))?; + } else { + return Err( + Error::InvalidArgument("input path is not a regular file or directory") + .at_entry(archive_name.to_vec(), "adding"), + ); + } + Ok(()) + } + + /// Stable IDs in current archive order, including directories and links. + /// IDs survive renames/removals and are never reused within this builder. + pub fn member_ids(&self) -> impl Iterator + '_ { + self.entries.iter().map(|entry| entry.id) + } + + /// Permit duplicate names when importing entries. Name-based editing still + /// rejects ambiguity; callers must select those members by ID. + pub fn allow_duplicate_names(mut self, allow: bool) -> Self { + self.allow_duplicate_names = allow; + self + } + + /// Resolve a unique queued name to its stable ID. + pub fn entry_id(&self, name: &[u8]) -> Result { + let mut matches = self.entries.iter().filter(|entry| entry.name == name); + let entry = matches + .next() + .ok_or_else(|| Error::EntryNotFound.at_entry(name.to_vec(), "selecting"))?; + if matches.next().is_some() { + return Err( + Error::InvalidArgument("ambiguous member name; select by member ID") + .at_entry(name.to_vec(), "selecting"), + ); + } + Ok(entry.id) + } + + fn index_by_id(&self, id: usize) -> Result { + self.entries + .iter() + .position(|entry| entry.id == id) + .ok_or(Error::EntryNotFound) + } + + /// Drop the uniquely named member. Ambiguous names are rejected. + pub fn remove(&mut self, name: &[u8]) -> Result<()> { + self.remove_by_id(self.entry_id(name)?) + } + + /// Drop exactly one member, identified by its stable ID. + pub fn remove_by_id(&mut self, id: usize) -> Result<()> { + let index = self.index_by_id(id)?; + let name = &self.entries[index].name; + if self + .entries + .iter() + .any(|entry| entry.id != id && entry.name == *name) + { + for entry in &self.entries[index + 1..] { + if entry + .redirection + .as_ref() + .is_some_and(|link| link.redirection_type >= 4 && link.target_name == *name) + { + return Err(Error::InvalidArgument( + "removing this duplicate would retarget a hard link or file copy", + )); + } + if entry.name == *name && !entry.is_directory { + break; + } + } + } + self.entries.remove(index); + Ok(()) + } + + /// Rename the uniquely named member. Ambiguous names are rejected. + pub fn rename(&mut self, old: &[u8], new: Vec) -> Result<()> { + self.rename_by_id(self.entry_id(old)?, new) + } + + /// Rename exactly one member. A new duplicate name is rejected. Name-based + /// hard-link/file-copy targets follow the nearest preceding target identity. + pub fn rename_by_id(&mut self, id: usize, new: Vec) -> Result<()> { + let new = self.validate_name(new)?; + let index = self.index_by_id(id)?; + let old = self.entries[index].name.clone(); + if old != new { + self.reject_duplicate_name(&new)?; + } + let unicode_name = if old != new && self.entries[index].legacy_unicode_name.is_some() { + Some(crate::filename::encode_legacy_unicode(&new)?) + } else { + self.entries[index].legacy_unicode_name.clone() + }; + // A later member with the same name shadows this target for later links. + let mut target = None; + for entry in &mut self.entries { + if let Some(link) = &mut entry.redirection { + if link.redirection_type >= 4 && link.target_name == old && target == Some(id) { + link.target_name = new.clone(); + } + } + if entry.name == old + && !entry.is_directory + && entry + .redirection + .as_ref() + .is_none_or(|link| link.redirection_type >= 4) + { + target = Some(entry.id); + } + } + self.entries[index].legacy_unicode_name = unicode_name; + self.entries[index].name = new; + Ok(()) + } + + fn validate_name(&self, name: Vec) -> Result> { + let name = validate_entry_name(name)?; + if self.format.family() == ArchiveFamily::Rar50Plus && std::str::from_utf8(&name).is_err() { + return Err(Error::InvalidArgument( + "RAR5 archive names require wire UTF-8; use filename::encode_rar50 for native Unix bytes", + )); + } + Ok(name) + } + + fn push(&mut self, mut entry: BuilderEntry) -> Result<()> { + if !self.allow_duplicate_names { + self.reject_duplicate_name(&entry.name)?; + } + let next = self + .next_entry_id + .checked_add(1) + .ok_or(Error::InvalidArgument("member ID overflow"))?; + entry.id = self.next_entry_id; + self.next_entry_id = next; + self.entries.push(entry); + Ok(()) + } + + fn reject_duplicate_name(&self, name: &[u8]) -> Result<()> { + if self.entries.iter().any(|entry| entry.name == name) { + return Err(Error::AtEntry { + name: name.to_vec(), + operation: "adding", + source: Box::new(Error::DuplicateEntry), + }); + } + Ok(()) + } + + /// Encode the whole archive into memory. + pub fn to_bytes(&self) -> Result> { + self.to_bytes_with_progress(None) + } + + /// As [`to_bytes`](Self::to_bytes), reporting progress as it goes. + pub fn to_bytes_with_progress(&self, progress: Option<&dyn WriteProgress>) -> Result> { + self.check_single()?; + if self.streams_rar50() { + let mut output = Vec::new(); + self.write_streaming_rar50(&mut output, &WriterResources::default(), progress)?; + return Ok(output); + } + let mut output = Vec::new(); + self.write_to(&mut output, &WriterResources::default(), progress)?; + Ok(output) + } + + /// Encode into a collector that remains charged until handoff or drop. + pub fn to_output( + &self, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result { + let mut output = crate::WriterOutput::new(resources); + self.write_to(&mut output, resources, progress)?; + Ok(output) + } + + /// Encode with an explicit resource policy, transferring output ownership + /// to the caller when this method returns. + pub fn to_bytes_with_resources( + &self, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result> { + Ok(self.to_output(resources, progress)?.into_vec()) + } + + /// Write the archive to `output`. + /// + /// RAR 5 and RAR 7 stream using the supplied workspace and spool policy; + /// see [`WriterResources`] for its scope and bare-WASM memory limits. + /// Legacy families emit headers and members directly, retaining compressed + /// members in memory. Stored sources are read twice to prepare and verify + /// checksums. Legacy managed-memory quotas are not supported. + /// On failure, `output` may contain a partial archive. Use + /// [`write_to_path`](Self::write_to_path) for atomic file publication. + pub fn write_to( + &self, + output: &mut dyn Write, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + self.check_single()?; + if self.streams_rar50() { + return self.write_streaming_rar50(output, resources, progress); + } + if resources.max_preparation_bytes().is_some() || resources.max_memory_bytes().is_some() { + return Err(Error::UnsupportedFamilyFeature { + family: self.format.family(), + feature: if resources.max_memory_bytes().is_some() { + "aggregate managed-memory limit" + } else { + "preparation memory quota" + }, + }); + } + let progress = ResourceProgress::new(resources, progress.map(ProgressReporter)); + let reporting = Some(ProgressReporter(&progress)); + check_cancelled(reporting)?; + let mut output = EmissionWriter::new(output, reporting); + self.write_legacy_to(&mut output, resources, Some(&progress))?; + output.finish() + } + + /// Write the archive to `path`, streaming where the format allows it. + /// The completed archive replaces the destination only after writing and syncing succeed. + /// + /// Any spooling the writer needs goes beside the output rather than in the + /// system temporary directory, which is often a memory-backed filesystem + /// too small for the archive being written. + pub fn write_to_path(&self, path: &Path, progress: Option<&dyn WriteProgress>) -> Result<()> { + let resources = match path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + Some(parent) => WriterResources::default().with_temp_dir(parent), + None => WriterResources::default(), + }; + self.write_to_path_with_resources(path, &resources, progress) + } + + /// Publish atomically using an explicit writer policy and scratch directory. + pub fn write_to_path_with_resources( + &self, + path: &Path, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + if self.format.family() != ArchiveFamily::Rar50Plus + && resources.max_memory_bytes().is_some() + { + return Err(Error::UnsupportedFamilyFeature { + family: self.format.family(), + feature: "aggregate managed memory quota", + }); + } + let (mut pending, output) = PendingArchive::with_resources(path, resources)?; + // Close before rename or cleanup on platforms that disallow removing + // open files. Declaration order also closes it first during unwinding. + { + let mut output = output; + self.write_to(&mut output, resources, progress)?; + output.sync_all()?; + } + check_cancelled(progress.map(ProgressReporter))?; + fs::rename(pending.path.as_ref().unwrap(), path)?; + pending.path = None; + Ok(()) + } + + /// Encode the archive as a volume set, one `Vec` per volume. + /// + /// Requires [`volume_size`](Self::volume_size). Naming the parts on disk is + /// the caller's job, because the two families number them differently. + pub fn build_volumes(&self, progress: Option<&dyn WriteProgress>) -> Result>> { + self.build_volumes_with_resources(&WriterResources::default(), progress) + } + + pub fn build_volumes_with_resources( + &self, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result>> { + if self.format.family() == ArchiveFamily::Rar50Plus { + return self.to_volume_output(resources, progress)?.into_vec(); + } + if resources.max_memory_bytes().is_some() || resources.max_preparation_bytes().is_some() { + return Err(Error::UnsupportedFamilyFeature { + family: self.format.family(), + feature: "managed writer memory policy", + }); + } + let volume_size = self.volume_payload_size()?; + let control = ResourceProgress::new(resources, progress.map(ProgressReporter)); + let progress = Some(&control as &dyn WriteProgress); + check_cancelled(progress.map(ProgressReporter))?; + self.single_volume_entry()?; + let this = self.materialized_volume_input(progress.map(ProgressReporter))?; + let result = match self.format.family() { + ArchiveFamily::Rar15To40 => this.build_rar15_volumes(volume_size, progress), + ArchiveFamily::Rar13 => this.build_rar13_volumes(volume_size, progress), + ArchiveFamily::Rar50Plus => unreachable!("modern volume output handled above"), + }?; + check_cancelled(progress.map(ProgressReporter))?; + Ok(result) + } + + /// Collect modern volumes, retaining their charges for a binding handoff. + pub fn to_volume_output( + &self, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result { + if self.format.family() != ArchiveFamily::Rar50Plus { + return Err(Error::UnsupportedFamilyFeature { + family: self.format.family(), + feature: "streaming managed volume output", + }); + } + let mut sink = crate::streaming::output::VolumeCollector::new(resources)?; + self.write_volumes_to(&mut sink, resources, progress)?; + sink.finish() + } + + /// Stream RAR5/7 volumes to a caller-owned sink under the resource policy. + pub fn write_volumes_to( + &self, + sink: &mut dyn rar50::VolumeSink, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + if self.format.family() != ArchiveFamily::Rar50Plus { + return Err(Error::UnsupportedFamilyFeature { + family: self.format.family(), + feature: "streaming managed volume output", + }); + } + let volume_size = self.volume_payload_size()?; + if self.comment.is_some() { + return Err(Error::InvalidArgument( + "RAR 5 volume comments are not supported", + )); + } + let converted = self.rar50_entries_with_resources(resources)?; + rar50::write_streaming_volumes_with_progress( + &converted.entries, + self.rar50_options(), + rar50::ArchiveExtras::default() + .with_recovery_percent(self.recovery_percent) + .with_filter_policy(self.rar50_filter_policy()), + volume_size as u64, + sink, + resources, + progress, + ) + } + + fn volume_payload_size(&self) -> Result { + self.check_encryption_option()?; + self.check_recovery_option()?; + if self.legacy_unpack_version.is_some() { + return Err(Error::InvalidArgument( + "retained legacy unpacker version requires single-archive output", + )); + } + if self.entries.iter().any(|entry| { + entry.legacy_extended_times.is_some() || entry.legacy_unicode_name.is_some() + }) { + return Err(Error::InvalidArgument( + "legacy extended timestamps and Unicode names are unsupported in volume output", + )); + } + + if self.format.family() != ArchiveFamily::Rar50Plus + && self.entries.iter().any(|entry| entry.encryption.is_some()) + { + return Err(Error::InvalidArgument( + "legacy per-entry encryption is unsupported in volume output", + )); + } + if self.format.family() != ArchiveFamily::Rar50Plus && self.entries.iter().any(|entry| entry.is_directory || matches!(entry.attributes, EntryAttributes::Unix(mode) if mode & 0o170000 == 0o120000)) { + return Err(Error::InvalidArgument("legacy directories and symbolic links are unsupported in volume output")); + } + let volume_size = self + .volume_size + .ok_or(Error::InvalidArgument("volume_size is required"))?; + if self + .entries + .iter() + .any(|entry| entry.file_comment.is_some()) + { + crate::write_plan::validate_option( + self.format, + crate::write_plan::WriterOption::FileComment, + crate::write_plan::PlanShape::new().volumes(true), + )?; + } + if self.entries.is_empty() { + return Err(Error::InvalidArgument("archive builder has no entries")); + } + if self.archive_metadata.is_some() || self.locked || self.quick_open { + return Err(Error::InvalidArgument( + "archive metadata settings are not supported in volume output", + )); + } + if self.entries.iter().any(|entry| entry.redirection.is_some()) { + return Err(Error::InvalidArgument( + "symbolic links are not supported in volume output", + )); + } + Ok(volume_size) + } + + /// Whether writing goes through the streaming RAR 5 writer, which serves + /// everything this builder can ask for except volume sets. It validates + /// missing header passwords before opening member sources. + pub fn streams_rar50(&self) -> bool { + matches!(self.format, ArchiveVersion::Rar50 | ArchiveVersion::Rar70) + && self.volume_size.is_none() + } + + fn check_recovery_option(&self) -> Result<()> { + if self.recovery_percent.is_some() { + crate::write_plan::validate_option( + self.format, + crate::write_plan::WriterOption::RecoveryRecord, + crate::write_plan::PlanShape::new().volumes(self.volume_size.is_some()), + )?; + } + Ok(()) + } + + fn check_encryption_option(&self) -> Result<()> { + if cfg!(feature = "encryption") { + return Ok(()); + } + let encrypted_entry = self.entries.iter().any(|entry| { + let data_password = entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |encryption| { + encryption.data_password.as_deref() + }); + let comment_password = entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |encryption| { + encryption.comment_password.as_deref() + }); + data_password.is_some() || (entry.file_comment.is_some() && comment_password.is_some()) + }); + if self.encrypt_headers || self.comment_password.is_some() || encrypted_entry { + crate::crypto::require_encryption()?; + } + Ok(()) + } + + fn check_single(&self) -> Result<()> { + self.check_encryption_option()?; + self.check_recovery_option()?; + self.check_redirection_targets()?; + if !self.streams_rar50() + && (self.archive_metadata.is_some() || self.locked || self.quick_open) + { + return Err(Error::InvalidArgument( + "archive metadata settings require the RAR5/7 streaming writer", + )); + } + if self.entries.is_empty() && !self.streams_rar50() { + return Err(Error::InvalidArgument("archive builder has no entries")); + } + if self.volume_size.is_some() { + return Err(Error::InvalidArgument( + "use build_volumes for multivolume archives", + )); + } + Ok(()) + } + + /// Legacy volume collection needs its one input as a slice. Borrow an + /// inline input; read a source without copying any other member payloads. + fn materialized_volume_input( + &self, + progress: Option>, + ) -> Result> { + check_cancelled(progress)?; + // The caller has validated the legacy volume path and its one input. + if !self.entries.iter().any(|entry| entry.source.is_some()) { + return Ok(std::borrow::Cow::Borrowed(self)); + } + let mut owned = self.clone(); + for entry in &mut owned.entries { + if let Some(source) = entry.source.take() { + entry.data = crate::write_stream::MemberBytes::Source(&source) + .load_with_progress(progress) + .map_err(|error| { + crate::write_stream::member_error(error, &entry.name, "reading source") + })? + .into_owned(); + source.release(); + } + } + Ok(std::borrow::Cow::Owned(owned)) + } + + fn write_legacy_to( + &self, + output: &mut dyn Write, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + match self.format.family() { + ArchiveFamily::Rar50Plus => unreachable!("RAR 5/7 use the streaming writer"), + ArchiveFamily::Rar15To40 => self.write_rar15_single(output, resources, progress), + ArchiveFamily::Rar13 => self.write_rar13_single(output, resources, progress), + } + } + + fn features(&self) -> FeatureSet { + let mut features = FeatureSet::store_only(); + features.solid = self.solid; + features.header_encryption = self.encrypt_headers; + features.quick_open = self.quick_open; + features + } + + /// Storing is a compression level, not a kind of member, so `store` wins + /// over any level the caller asked for. + fn rar50_options(&self) -> rar50::WriterOptions { + let mut options = rar50::WriterOptions::new(self.format, self.features()); + if let Some(size) = self.rar50_dictionary_size { + options = options.with_dictionary_size(size); + } + if let Some(level) = self.compression { + options = options.with_compression_level(level); + } + if self.store { + options = options.with_compression_level(0); + } + options + } + + /// Whether to look for a data filter, which the archive has to be able to + /// carry. One answer for every RAR 5 path: when the streaming writer worked + /// this out for itself, an archive built from bytes came out unfiltered and + /// larger than the same archive built from files. + fn rar50_filter_policy(&self) -> rar50::FilterPolicy { + if self.solid || self.store { + rar50::FilterPolicy::None + } else { + rar50::FilterPolicy::Auto + } + } + + fn rar50_entries_with_resources( + &self, + resources: &WriterResources, + ) -> Result { + let mut charge = resources.execution_charge(); + if let Some(charge) = &mut charge { + let mut capacity = self + .entries + .len() + .checked_mul(std::mem::size_of::()) + .ok_or(Error::InvalidArgument("converted entry capacity overflows"))?; + for entry in &self.entries { + let data_password = entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |e| e.data_password.as_deref()); + let comment_password = entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |e| e.comment_password.as_deref()); + for bytes in [ + entry.name.len(), + entry + .redirection + .as_ref() + .map_or(0, |r| r.target_name.len()), + data_password.map_or(0, <[u8]>::len), + ] { + capacity = capacity + .checked_add(bytes) + .ok_or(Error::InvalidArgument("converted entry capacity overflows"))?; + } + if let Some(comment) = &entry.file_comment { + for bytes in [ + std::mem::size_of::(), + 3, + comment.len(), + comment_password.map_or(0, <[u8]>::len), + ] { + capacity = capacity.checked_add(bytes).ok_or(Error::InvalidArgument( + "converted service capacity overflows", + ))?; + } + } + } + charge.grow_to(capacity as u64)?; + } + let mut entries = Vec::with_capacity(self.entries.len()); + for entry in &self.entries { + let source = match &entry.source { + Some(source) => source.clone(), + None => EntrySource::copy_for_writer(&entry.data, resources)?, + }; + let built = rar50::ArchiveEntry::new(entry.name.clone(), source) + .with_directory(entry.is_directory) + .with_redirection(entry.redirection.clone()) + .with_redirection_size(entry.redirection_size) + .with_mtime(entry.mtime) + .with_mtime_nanoseconds(entry.mtime_nanoseconds) + .with_file_times(entry.file_times) + .with_attributes(entry.rar50_attr()) + .with_host_os(entry.rar50_host_os()); + let data_password = entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |encryption| { + encryption.data_password.as_deref() + }); + let comment_password = entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |encryption| { + encryption.comment_password.as_deref() + }); + let built = match &entry.file_comment { + Some(comment) => { + let service = rar50::ServiceEntry::new(b"CMT".to_vec(), comment.clone()); + let service = match comment_password { + Some(password) => service.with_password(password.to_vec()), + None => service, + }; + let mut built = built; + built.services = vec![service]; + built + } + None => built, + }; + entries.push(match data_password { + Some(password) => built.with_password(password.to_vec()), + None => built, + }); + } + Ok(ConvertedEntries { + entries, + _charge: charge, + }) + } + + fn write_streaming_rar50( + &self, + output: &mut dyn Write, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + let mut extras = rar50::ArchiveExtras::default() + .with_recovery_percent(self.recovery_percent) + .with_filter_policy(self.rar50_filter_policy()); + extras.metadata_record = self.archive_metadata.as_ref(); + extras.locked = self.locked; + if self.encrypt_headers { + extras.header_password = self.password.as_deref(); + } + if let Some(comment) = self.comment.as_deref() { + extras = match self.comment_password.as_deref() { + Some(password) => extras.with_encrypted_comment(comment, password), + None => extras.with_comment(comment), + }; + } + let converted = self.rar50_entries_with_resources(resources)?; + rar50::write_streaming_archive_with_progress( + &converted.entries, + self.rar50_options(), + extras, + resources, + progress, + output, + ) + } + + fn rar15_options(&self) -> rar15_40::WriterOptions { + let mut options = rar15_40::WriterOptions::new(self.format, self.features()); + options.archive_comment_metadata = self.legacy_archive_comment_metadata; + if let Some(level) = self.compression { + options = options.with_compression_level(level); + } + options + } + + fn write_rar15_single( + &self, + output: &mut dyn Write, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + let entries: Vec<_> = self + .entries + .iter() + .map(|entry| rar15_40::RetainedFileEntry { + source: entry.source.as_ref(), + file: rar15_40::FileEntry { + name: &entry.name, + data: &entry.data, + file_time: entry.mtime.unwrap_or(0), + file_attr: entry.rar15_attr(), + host_os: entry.rar15_host_os(), + password: entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |encryption| { + encryption.data_password.as_deref() + }), + file_comment: entry.file_comment.as_deref(), + }, + metadata: rar15_40::RetainedMemberMetadata { + unicode_name: entry.legacy_unicode_name.as_deref(), + unpack_version: self.legacy_unpack_version, + extended_times: entry.legacy_extended_times.as_deref(), + is_directory: entry.is_directory, + is_symlink: matches!(entry.attributes, EntryAttributes::Unix(mode) if mode & 0o170000 == 0o120000), + }, + }) + .collect(); + rar15_40::write_archive_with_retained_metadata( + &entries, + self.rar15_options(), + if self.store { + crate::write_plan::MemberCoding::Stored + } else { + crate::write_plan::MemberCoding::Compressed + }, + self.comment.as_deref(), + resources, + progress, + self.encrypt_headers + .then_some(self.password.as_deref()) + .flatten(), + self.comment_password.as_deref(), + output, + ) + } + + fn rar13_options(&self) -> rar13::WriterOptions { + let mut options = rar13::WriterOptions::new(self.format, self.features()); + if let Some(level) = self.compression { + options = options.with_compression_level(level); + } + options + } + + fn write_rar13_single( + &self, + output: &mut dyn Write, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + ) -> Result<()> { + let entries: Vec<_> = self + .entries + .iter() + .map(|entry| rar13::BorrowedFileEntry { + file: rar13::FileEntry { + name: &entry.name, + data: &entry.data, + file_time: entry.mtime.unwrap_or(0), + file_attr: entry.rar13_attr(), + password: entry + .encryption + .as_ref() + .map_or(self.password.as_deref(), |encryption| { + encryption.data_password.as_deref() + }), + file_comment: entry.file_comment.as_deref(), + }, + source: entry.source.as_ref(), + }) + .collect(); + rar13::write_archive_with_sources( + &entries, + self.rar13_options(), + if self.store { + crate::write_plan::MemberCoding::Stored + } else { + crate::write_plan::MemberCoding::Compressed + }, + self.comment.as_deref(), + resources, + progress, + output, + ) + } + + fn single_volume_entry(&self) -> Result<&BuilderEntry> { + match self.entries.as_slice() { + [entry] => Ok(entry), + _ => Err(Error::InvalidArgument("legacy volumes support one input")), + } + } + + fn build_rar15_volumes( + &self, + volume_size: usize, + progress: Option<&dyn WriteProgress>, + ) -> Result>> { + let entry = self.single_volume_entry()?; + let options = self.rar15_options(); + if self.store { + rar15_40::write_stored_volumes_with_progress( + rar15_40::StoredEntry { + name: &entry.name, + data: &entry.data, + file_time: entry.mtime.unwrap_or(0), + file_attr: entry.rar15_attr(), + host_os: entry.rar15_host_os(), + password: self.password.as_deref(), + file_comment: entry.file_comment.as_deref(), + }, + options, + volume_size, + progress, + ) + } else { + rar15_40::write_compressed_volumes_with_progress( + rar15_40::FileEntry { + name: &entry.name, + data: &entry.data, + file_time: entry.mtime.unwrap_or(0), + file_attr: entry.rar15_attr(), + host_os: entry.rar15_host_os(), + password: self.password.as_deref(), + file_comment: entry.file_comment.as_deref(), + }, + options, + volume_size, + progress, + ) + } + } + + fn build_rar13_volumes( + &self, + volume_size: usize, + progress: Option<&dyn WriteProgress>, + ) -> Result>> { + let entry = self.single_volume_entry()?; + let options = self.rar13_options(); + if self.store { + rar13::write_stored_volumes_with_progress( + rar13::StoredEntry { + name: &entry.name, + data: &entry.data, + file_time: entry.mtime.unwrap_or(0), + file_attr: entry.rar13_attr(), + password: self.password.as_deref(), + file_comment: entry.file_comment.as_deref(), + }, + options, + volume_size, + progress, + ) + } else { + rar13::write_compressed_volumes_with_progress( + rar13::FileEntry { + name: &entry.name, + data: &entry.data, + file_time: entry.mtime.unwrap_or(0), + file_attr: entry.rar13_attr(), + password: self.password.as_deref(), + file_comment: entry.file_comment.as_deref(), + }, + options, + volume_size, + progress, + ) + } + } +} + +// Retain the established public paths while the implementations live with +// the reader filename utilities. +pub use crate::filename::{entry_relative_path, validate_entry_name}; + +#[cfg(unix)] +fn unix_mode(metadata: &fs::Metadata) -> Option { + use std::os::unix::fs::PermissionsExt; + Some(metadata.permissions().mode()) +} + +#[cfg(not(unix))] +fn unix_mode(_metadata: &fs::Metadata) -> Option { + None +} + +#[cfg(test)] +mod tests { + #[test] + fn pending_archive_retries_name_collisions_and_reports_exhaustion() { + let root = crate::scratch::case("pending-archive-collision"); + let destination = root.join("archive.rar"); + let name = |sequence| format!(".rars-writing-{}-{sequence:016x}", std::process::id()); + std::fs::write(root.join(name(0)), b"occupied").unwrap(); + let resources = crate::WriterResources::default().with_temp_dir(&*root); + let mut sequence = 0; + let (pending, file) = + super::PendingArchive::with_sequence(&destination, &resources, || { + let value = sequence; + sequence += 1; + value + }) + .unwrap(); + assert_eq!(sequence, 2); + assert_eq!(pending.path.as_deref(), Some(root.join(name(1)).as_path())); + drop(file); + drop(pending); + + for sequence in 1..128 { + std::fs::write(root.join(name(sequence)), b"occupied").unwrap(); + } + let mut sequence = 0; + let error = super::PendingArchive::with_sequence(&destination, &resources, || { + let value = sequence; + sequence += 1; + value + }) + .err() + .unwrap(); + assert_eq!(sequence, 128); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!(std::fs::read(root.join(name(0))).unwrap(), b"occupied"); + + let missing = root.join("missing/archive.rar"); + let error = super::PendingArchive::with_sequence(&missing, &resources, || 0) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + } + + #[test] + fn converted_rar50_entries_charge_encrypted_comment_storage() { + let resources = crate::WriterResources::default().with_max_memory_bytes(1 << 20); + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50); + builder + .add_source( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".as_slice()), + None, + None, + ) + .unwrap(); + let baseline = builder.rar50_entries_with_resources(&resources).unwrap(); + let baseline_charge = resources.managed_memory_in_use(); + drop(baseline); + assert_eq!(resources.managed_memory_in_use(), 0); + + let comment = b"private comment"; + let password = b"password"; + builder + .set_file_comment(b"file", Some(comment.to_vec())) + .unwrap(); + builder + .set_entry_encryption(b"file", None, Some(password.to_vec())) + .unwrap(); + let converted = builder.rar50_entries_with_resources(&resources).unwrap(); + assert_eq!( + resources.managed_memory_in_use() - baseline_charge, + (std::mem::size_of::() + 3 + comment.len() + password.len()) + as u64 + ); + drop(converted); + assert_eq!(resources.managed_memory_in_use(), 0); + } + + #[test] + fn builder_rejects_legacy_volume_directory_and_metadata_settings() { + use crate::{ArchiveVersion, Builder, ErrorKind}; + + let mut volume = Builder::new(ArchiveVersion::Rar29).volume_size(Some(64)); + assert_eq!( + volume + .add_directory(b"dir".to_vec(), None, None) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + assert_eq!( + Builder::new(ArchiveVersion::Rar29) + .archive_metadata(None, false, false) + .err() + .unwrap() + .kind(), + ErrorKind::InvalidArgument + ); + } + + #[test] + fn builder_rejects_invalid_retained_archive_metadata_at_setter() { + let record = crate::rar50::ArchiveMetadataRecord { + flags: 2, + name: None, + creation_time: None, + }; + let error = crate::Builder::new(crate::ArchiveVersion::Rar50) + .archive_metadata(Some(record), false, false) + .err() + .unwrap(); + assert_eq!( + error, + crate::Error::InvalidArgument("unsupported archive metadata record") + ); + } + + #[test] + fn builder_refuses_retained_archive_metadata_in_volume_output() { + let record = crate::rar50::ArchiveMetadataRecord { + flags: 2, + name: None, + creation_time: Some(123), + }; + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50) + .archive_metadata(Some(record), false, false) + .unwrap() + .volume_size(Some(64)); + builder + .add_source( + b"file".to_vec(), + crate::EntrySource::from_opener(1, || panic!("volume refusal opened source")), + None, + None, + ) + .unwrap(); + assert_eq!( + builder.to_bytes().unwrap_err(), + crate::Error::InvalidArgument( + "archive metadata settings require the RAR5/7 streaming writer" + ) + ); + assert_eq!( + builder.build_volumes(None).unwrap_err(), + crate::Error::InvalidArgument( + "archive metadata settings are not supported in volume output" + ) + ); + } + + #[test] + fn renaming_a_member_to_its_existing_name_preserves_its_identity() { + use crate::{ArchiveReader, ArchiveVersion, Builder}; + + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let id = builder.entry_id(b"file").unwrap(); + builder.rename_by_id(id, b"file".to_vec()).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } + + #[test] + fn builder_rejects_unsupported_per_entry_password_shapes() { + use crate::{ArchiveVersion, Builder, ErrorKind}; + + for format in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert_eq!( + builder + .set_entry_encryption(b"file", Some(Vec::new()), None) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + assert_eq!( + builder + .set_entry_encryption(b"file", None, Some(Vec::new())) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + if format != ArchiveVersion::Rar50 { + assert_eq!( + builder + .set_entry_encryption(b"file", None, Some(b"comment".to_vec())) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + } + } + + let mut volume = Builder::new(ArchiveVersion::Rar29) + .store(true) + .volume_size(Some(64)); + volume + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert_eq!( + volume + .set_entry_encryption(b"file", Some(b"password".to_vec()), None) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn rar13_entry_encryption_override_can_clear_the_builder_password() { + use crate::{ArchiveReader, ArchiveVersion, Builder}; + + for store in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar13) + .store(store) + .password(Some(b"default password".to_vec())); + builder + .add_bytes(b"plain".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.set_entry_encryption(b"plain", None, None).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"plain", None).unwrap().unwrap(), + b"payload" + ); + } + } + + #[test] + fn legacy_builder_combines_inline_and_source_members() { + use crate::{ArchiveReader, ArchiveVersion, Builder, EntrySource}; + + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes(b"bytes".to_vec(), b"in memory".to_vec(), None, None) + .unwrap(); + builder + .add_source( + b"source".to_vec(), + EntrySource::from_bytes(b"from source".as_slice()), + None, + None, + ) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"bytes", None).unwrap().unwrap(), + b"in memory" + ); + assert_eq!( + archive.read_member(b"source", None).unwrap().unwrap(), + b"from source" + ); + } + + #[test] + fn retained_unpack_version_cannot_be_written_to_legacy_volumes() { + use crate::{ArchiveVersion, Builder, Error}; + + let mut builder = Builder::new(ArchiveVersion::Rar29) + .store(true) + .volume_size(Some(64)) + .legacy_unpack_version(Some(29)); + builder + .add_bytes(b"file".to_vec(), b"data".to_vec(), None, None) + .unwrap(); + assert_eq!( + builder.build_volumes(None).unwrap_err(), + Error::InvalidArgument( + "retained legacy unpacker version requires single-archive output" + ) + ); + } + + #[test] + fn replacing_source_keeps_metadata_and_refuses_entry_kind_changes() { + use crate::{ArchiveReader, ArchiveVersion, Builder, EntrySource, ErrorKind}; + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"old".to_vec(), Some(123), Some(0o640)) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"comment".to_vec())) + .unwrap(); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_unix_symlink(b"link".to_vec(), b"file".to_vec(), false, None, None) + .unwrap(); + for name in [b"dir".as_slice(), b"link"] { + assert_eq!( + builder + .set_source(name, EntrySource::from_bytes(b"bad".as_slice())) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + } + assert_eq!( + builder + .set_source(b"missing", EntrySource::from_bytes(b"bad".as_slice())) + .unwrap_err() + .kind(), + ErrorKind::EntryNotFound + ); + builder + .set_source(b"file", EntrySource::from_bytes(b"replacement".as_slice())) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert_eq!(members[0].meta.name, b"file"); + assert_eq!(members[0].meta.file_time, Some(123)); + assert_eq!(members[0].meta.file_attr & 0o7777, 0o640); + assert_eq!( + archive.member_comment_at(0, None).unwrap(), + Some(b"comment".to_vec()) + ); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"replacement" + ); + assert!(members[1].meta.is_directory); + assert!(members[2].meta.is_redirection); + } + + use super::*; + + #[test] + fn legacy_directory_metadata_and_link_payloads_survive_solid_output() { + let mut builder = Builder::new(ArchiveVersion::Rar29).solid(true); + builder + .add_bytes(b"file".to_vec(), b"payload".repeat(100), Some(0), None) + .unwrap(); + builder + .add_directory(b"directory".to_vec(), Some(0), Some(0o2750)) + .unwrap(); + builder + .set_file_comment(b"directory", Some(b"comment".to_vec())) + .unwrap(); + builder + .set_legacy_extended_times(b"directory", Some(vec![0, 0xb0, 1, 2, 3])) + .unwrap(); + builder + .add_unix_symlink(b"link".to_vec(), b"file".to_vec(), false, Some(0), None) + .unwrap(); + builder + .add_bytes(b"last".to_vec(), b"payload".repeat(100), Some(0), None) + .unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + let files: Vec<_> = archive.as_rar15_40().unwrap().files().collect(); + assert!(files[1].is_directory()); + assert_eq!(files[1].attr, 0o042750); + assert_eq!(files[1].pack_size, 0); + assert_eq!(files[1].ext_time, [0, 0xb0, 1, 2, 3]); + assert_eq!( + archive.member_comment_at(1, None).unwrap(), + Some(b"comment".to_vec()) + ); + assert!(files[2].is_stored()); + assert_eq!( + archive.legacy_symlink_target_at(2, None).unwrap(), + Some(b"file".to_vec()) + ); + assert_eq!( + archive.read_member(b"last", None).unwrap().unwrap(), + b"payload".repeat(100) + ); + assert!(builder.volume_size(Some(128)).build_volumes(None).is_err()); + } + + #[test] + #[cfg(feature = "encryption")] + fn legacy_comments_coexist_with_salt_and_extended_times() { + let mut builder = Builder::new(ArchiveVersion::Rar29) + .password(Some(b"secret".to_vec())) + .comment(Some(b"archive".to_vec())); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), Some(0), None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"comment".to_vec())) + .unwrap(); + builder + .set_legacy_extended_times(b"file", Some(vec![0, 0xb0, 1, 2, 3])) + .unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + assert_eq!( + archive.member_comment_at(0, Some(b"secret")).unwrap(), + Some(b"comment".to_vec()) + ); + assert_eq!( + archive + .read_member(b"file", Some(b"secret")) + .unwrap() + .unwrap(), + b"payload" + ); + assert_eq!( + archive + .as_rar15_40() + .unwrap() + .files() + .next() + .unwrap() + .ext_time, + [0, 0xb0, 1, 2, 3] + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn legacy_header_encryption_survives_removing_encrypted_member() { + let mut builder = Builder::new(ArchiveVersion::Rar30) + .password(Some(b"secret".to_vec())) + .header_encryption(true); + builder + .add_bytes(b"plain".to_vec(), b"public".to_vec(), Some(0), None) + .unwrap(); + builder.set_entry_encryption(b"plain", None, None).unwrap(); + builder + .add_bytes(b"encrypted".to_vec(), b"private".to_vec(), Some(0), None) + .unwrap(); + // Salt and native extended times must coexist in encrypted file headers. + builder + .set_legacy_extended_times(b"encrypted", Some(vec![0, 0xb0, 1, 2, 3])) + .unwrap(); + for remove in [false, true] { + if remove { + builder.remove(b"encrypted").unwrap(); + } + let data = builder.to_bytes().unwrap(); + assert!(matches!( + crate::ArchiveReader::read(&data), + Err(Error::NeedPassword) + )); + let archive = crate::ArchiveReader::read_owned_with_options( + data, + crate::ArchiveReadOptions::with_password(b"secret"), + ) + .unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + assert!(archive.as_rar15_40().unwrap().main.has_encrypted_headers()); + assert!(!archive + .as_rar15_40() + .unwrap() + .files() + .next() + .unwrap() + .is_encrypted()); + assert_eq!( + archive + .read_member(b"plain", if remove { None } else { Some(b"secret") }) + .unwrap() + .unwrap(), + b"public" + ); + if !remove { + assert_eq!( + archive + .read_member(b"encrypted", Some(b"secret")) + .unwrap() + .unwrap(), + b"private" + ); + assert_eq!( + archive + .as_rar15_40() + .unwrap() + .files() + .nth(1) + .unwrap() + .ext_time, + [0, 0xb0, 1, 2, 3] + ); + } + } + assert!(builder + .clone() + .volume_size(Some(128)) + .build_volumes(None) + .is_err()); + builder + .set_entry_encryption(b"plain", Some(b"different".to_vec()), None) + .unwrap(); + assert!(builder.to_bytes().is_err()); + } + + #[test] + fn native_legacy_times_survive_writes_and_rejected_changes() { + // Fractional odd-second mtime and an archival DOS timestamp. + let raw = vec![0x08, 0xf0, 1, 2, 3, 0, 0, 0, 0]; + for store in [true, false] { + let mut builder = Builder::new(ArchiveVersion::Rar40).store(store); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), Some(0), None) + .unwrap(); + builder + .set_legacy_extended_times(b"file", Some(raw.clone())) + .unwrap(); + let before = builder.to_bytes().unwrap(); + assert!(builder + .set_legacy_extended_times(b"file", Some(vec![0, 0xf0])) + .is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + let archive = crate::ArchiveReader::read_owned(before).unwrap(); + assert_eq!( + archive + .as_rar15_40() + .unwrap() + .files() + .next() + .unwrap() + .ext_time, + raw + ); + assert!(archive.rewrite_preservation_issues().is_empty()); + assert!(builder + .clone() + .volume_size(Some(128)) + .build_volumes(None) + .is_err()); + builder.set_legacy_extended_times(b"file", None).unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(!archive + .as_rar15_40() + .unwrap() + .files() + .next() + .unwrap() + .has_ext_time()); + } + } + + #[test] + fn empty_rewrites_retain_archive_settings() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for encrypted in crate::test_features::encryption_cases() { + let password = encrypted.then_some(b"secret".as_slice()); + let builder = Builder::new(format) + .solid(true) + .password(password.map(<[u8]>::to_vec)) + .header_encryption(encrypted) + .recovery_percent(cfg!(feature = "recovery").then_some(5)) + .comment(Some(b"comment".to_vec())); + let options = crate::ArchiveReadOptions { + password, + ..Default::default() + }; + let archive = crate::ArchiveReader::read_owned_with_options( + builder.to_bytes().unwrap(), + options, + ) + .unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + let rewritten = archive + .preserving_builder(password) + .unwrap() + .comment(Some(b"comment".to_vec())) + .to_bytes() + .unwrap(); + let output = + crate::ArchiveReader::read_owned_with_options(rewritten, options).unwrap(); + let raw = output.as_rar50().unwrap(); + assert!(raw.main.is_solid()); + assert_eq!(raw.main.has_recovery_record(), cfg!(feature = "recovery")); + assert_eq!(raw.main.encrypted_headers, encrypted); + assert_eq!(raw.files().count(), 0); + assert!(output.rewrite_preservation_issues().is_empty()); + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn header_encryption_does_not_require_encrypted_members() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .password(Some(b"secret".to_vec())) + .header_encryption(true); + builder + .add_bytes(b"plain".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.set_entry_encryption(b"plain", None, None).unwrap(); + let archive = crate::ArchiveReader::read_owned_with_options( + builder.to_bytes().unwrap(), + crate::ArchiveReadOptions::with_password(b"secret"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.encrypted_headers); + assert!(!raw.files().next().unwrap().encrypted); + assert_eq!( + archive.read_member(b"plain", None).unwrap().unwrap(), + b"payload" + ); + assert!(archive.rewrite_preservation_issues().is_empty()); + } + + #[cfg(unix)] + #[test] + fn add_path_preserves_non_utf8_child_names() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let root = crate::scratch::case("builder-non-utf8-name"); + let raw_source = root.join(OsStr::from_bytes(b"bad-\xff")); + let native_name = match fs::write(&raw_source, b"payload") { + Ok(()) => b"bad-\xff".as_slice(), + Err(error) + if error.kind() == std::io::ErrorKind::InvalidInput + || error.raw_os_error() == Some(libc::EILSEQ) => + { + // Some Unix filesystems reject byte names. The builder must + // report the missing/unrepresentable source, never queue a + // substituted filename or silently claim it was archived. + for format in ArchiveVersion::ALL { + let mut builder = Builder::new(format); + let error = builder.add_path(&raw_source, b"explicit").unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert!(builder.is_empty()); + } + // Continue exercising collection and explicit naming in every + // format with a name this filesystem can actually represent. + let name = "valid-日本語".as_bytes(); + fs::write(root.join(OsStr::from_bytes(name)), b"payload").unwrap(); + name + } + Err(error) => panic!("raw filename probe failed unexpectedly: {error}"), + }; + let source = root.join(OsStr::from_bytes(native_name)); + for format in ArchiveVersion::ALL { + let mut builder = Builder::new(format); + for unsafe_name in [b"../bad-\xff".as_slice(), b"/bad-\xff"] { + assert!(builder.add_path(&source, unsafe_name).is_err()); + assert!(builder.is_empty()); + } + builder.add_path(&root, b"root").unwrap(); + let native_archive_name = [b"root/".as_slice(), native_name].concat(); + let name = if format.family() == ArchiveFamily::Rar50Plus { + crate::filename::encode_rar50(&native_archive_name).into_owned() + } else { + native_archive_name + }; + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(&name, None).unwrap().unwrap(), + b"payload" + ); + // The same native source can also have an explicit archive name. + builder.add_path(&source, b"explicit").unwrap(); + assert_eq!( + builder.names().collect::>(), + vec![name.as_slice(), &b"explicit"[..]] + ); + } + } + + #[test] + fn add_path_preserves_unicode_child_names_and_contents() { + let root = crate::scratch::case("builder-unicode-names"); + fs::create_dir(root.join("日本語")).unwrap(); + for name in ["日本語", "кириллица", "replacement-\u{fffd}"] { + fs::write(root.join("日本語").join(name), name.as_bytes()).unwrap(); + } + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder.add_path(&root, b"root").unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for name in ["日本語", "кириллица", "replacement-\u{fffd}"] { + let member_name = format!("root/日本語/{name}"); + assert_eq!( + archive + .read_member(member_name.as_bytes(), None) + .unwrap() + .unwrap(), + name.as_bytes() + ); + } + } + + fn builder_with(format: ArchiveVersion) -> Builder { + let mut builder = Builder::new(format); + builder + .add_bytes(b"a.txt".to_vec(), b"hello world".repeat(64), None, None) + .unwrap(); + builder + .add_bytes(b"dir/b.txt".to_vec(), b"second member".to_vec(), None, None) + .unwrap(); + builder + } + + /// Every version this crate can write, which is all of them. The loop is + /// over `ArchiveVersion::ALL` so a tenth version is a failing test rather + /// than a gap. + #[test] + fn writes_every_family() { + for format in ArchiveVersion::ALL { + let archive = builder_with(format).to_bytes().unwrap(); + let read = crate::ArchiveReader::read_owned(archive).unwrap(); + let names: Vec<_> = read + .members() + .map(|member| member.meta.name_bytes().to_vec()) + .collect(); + assert_eq!( + names, + vec![b"a.txt".to_vec(), b"dir/b.txt".to_vec()], + "{format}" + ); + } + } + + #[test] + fn stored_and_compressed_round_trip_to_the_same_bytes() { + for store in [false, true] { + let archive = builder_with(ArchiveVersion::Rar50) + .store(store) + .to_bytes() + .unwrap(); + let read = crate::ArchiveReader::read_owned(archive).unwrap(); + let data = read.read_member(b"a.txt", None).unwrap().unwrap(); + assert_eq!(data, b"hello world".repeat(64)); + } + } + + #[test] + fn rejects_a_duplicate_name() { + let mut builder = Builder::new(ArchiveVersion::Rar50); + builder + .add_bytes(b"a".to_vec(), vec![], None, None) + .unwrap(); + let error = builder + .add_bytes(b"a".to_vec(), vec![], None, None) + .unwrap_err(); + assert!(error.to_string().contains("duplicate archive entry name")); + assert_eq!(builder.names().collect::>(), vec![&b"a"[..]]); + } + + #[test] + fn rejected_edits_preserve_a_usable_archive() { + let mut builder = builder_with(ArchiveVersion::Rar50).store(true); + let before = builder.to_bytes().unwrap(); + assert!(builder.rename(b"a.txt", b"dir/b.txt".to_vec()).is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + assert!(builder + .add_bytes(b"a.txt".to_vec(), b"replacement".to_vec(), None, None) + .is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + + builder.rename(b"a.txt", b"a.txt".to_vec()).unwrap(); + assert_eq!(builder.to_bytes().unwrap(), before); + builder.rename(b"a.txt", b"renamed.txt".to_vec()).unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"renamed.txt", None).unwrap().unwrap(), + b"hello world".repeat(64) + ); + } + + #[test] + fn legacy_byte_identity_survives_edits_without_unicode_validation() { + for format in ArchiveVersion::ALL { + if format.family() == ArchiveFamily::Rar50Plus { + continue; + } + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"old-\xff".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.rename(b"old-\xff", b"new-\xfe".to_vec()).unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"new-\xfe", None).unwrap().unwrap(), + b"payload" + ); + builder.remove(b"new-\xfe").unwrap(); + assert!(builder.is_empty()); + } + let mut modern = Builder::new(ArchiveVersion::Rar50); + assert!(modern + .add_bytes(b"old-\xff".to_vec(), vec![], None, None) + .is_err()); + } + + #[test] + fn rejects_an_escaping_name() { + let mut builder = Builder::new(ArchiveVersion::Rar50); + for name in [&b"../escape"[..], b"/absolute", b"C:\\drive"] { + assert!(builder + .add_bytes(name.to_vec(), vec![], None, None) + .is_err()); + } + } + + #[test] + fn renames_and_removes() { + let mut builder = builder_with(ArchiveVersion::Rar50); + builder.rename(b"a.txt", b"c.txt".to_vec()).unwrap(); + builder.remove(b"dir/b.txt").unwrap(); + assert_eq!(builder.names().collect::>(), vec![&b"c.txt"[..]]); + assert!(builder.remove(b"gone").is_err()); + assert!(builder.rename(b"gone", b"x".to_vec()).is_err()); + } + + #[test] + fn refuses_a_single_archive_when_a_volume_size_is_set() { + let error = builder_with(ArchiveVersion::Rar50) + .volume_size(Some(4096)) + .to_bytes() + .unwrap_err(); + assert!(error.to_string().contains("build_volumes")); + } + + #[test] + fn splits_into_volumes() { + let mut builder = Builder::new(ArchiveVersion::Rar50); + builder + .add_bytes(b"big.bin".to_vec(), vec![7u8; 300_000], None, None) + .unwrap(); + let volumes = builder + .store(true) + .volume_size(Some(64 * 1024)) + .build_volumes(None) + .unwrap(); + assert!(volumes.len() > 1, "expected a split, got {}", volumes.len()); + } +} + +#[cfg(test)] +#[test] +fn parentless_output_path_uses_default_resources_and_publishes_readable_archives() { + struct Destination(std::path::PathBuf); + impl Drop for Destination { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let sequence = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let destination = Destination(std::path::PathBuf::from(format!( + ".rars-parentless-test-{}-{sequence}.rar", + std::process::id() + ))); + assert!(destination.0.parent().unwrap().as_os_str().is_empty()); + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.write_to_path(&destination.0, None).unwrap(); + let archive = + crate::ArchiveReader::read_owned(std::fs::read(&destination.0).unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } +} + +#[cfg(test)] +#[test] +fn converted_redirections_charge_targets_and_preserve_them_in_output() { + let resources = WriterResources::default().with_max_memory_bytes(1 << 20); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_unix_symlink(b"link".to_vec(), b"a".to_vec(), false, None, None) + .unwrap(); + let short = builder.rar50_entries_with_resources(&resources).unwrap(); + let short_charge = resources.managed_memory_in_use(); + drop(short); + assert_eq!(resources.managed_memory_in_use(), 0); + let target = "destination-😀".as_bytes(); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_unix_symlink(b"link".to_vec(), target.to_vec(), false, None, None) + .unwrap(); + let converted = builder.rar50_entries_with_resources(&resources).unwrap(); + assert_eq!( + resources.managed_memory_in_use() - short_charge, + (target.len() - 1) as u64 + ); + assert_eq!( + converted.entries[0] + .redirection + .as_ref() + .unwrap() + .target_name, + target + ); + drop(converted); + assert_eq!(resources.managed_memory_in_use(), 0); + let bytes = builder.to_bytes_with_resources(&resources, None).unwrap(); + let archive = crate::ArchiveReader::read_owned(bytes).unwrap(); + let member = archive.members().next().unwrap(); + assert_eq!(member.unix_symlink().unwrap().target_name, target); + assert_eq!(resources.managed_memory_in_use(), 0); +} + +#[cfg(all(test, not(all(target_arch = "wasm32", target_os = "unknown"))))] +#[test] +fn pending_archive_accepts_maximum_sequence_without_a_destination_parent() { + let destination = Path::new(""); + assert!(destination.parent().is_none()); + let expected = std::path::PathBuf::from(format!( + ".rars-writing-{}-{:016x}", + std::process::id(), + u64::MAX + )); + assert!( + !expected.exists(), + "unique maximum-sequence path is already occupied" + ); + let resources = WriterResources::default().with_max_memory_bytes(1 << 20); + let (pending, file) = + PendingArchive::with_sequence(destination, &resources, || u64::MAX).unwrap(); + assert_eq!(pending.path.as_ref().unwrap(), &expected); + assert!(expected.is_file()); + assert!(expected.as_os_str().len() <= 41); + drop(file); + drop(pending); + assert!(!expected.exists()); + assert_eq!(resources.managed_memory_in_use(), 0); +} diff --git a/arca-rar/vendor/rars/src/codec/canonical.rs b/arca-rar/vendor/rars/src/codec/canonical.rs new file mode 100644 index 00000000..495895ed --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/canonical.rs @@ -0,0 +1,218 @@ +//! Canonical decoder mechanics shared by the explicit-table RAR formats. +//! Code-length parsing, diagnostics and bit input remain format-specific. +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; + +#[derive(Debug)] +pub(super) struct Huffman { + symbols: Buffer, + first_code: [u16; 16], + first_index: [usize; 16], + counts: [u16; 16], +} + +impl Huffman { + pub(super) fn with_allowance(allowance: &B) -> Self { + Self { + symbols: Buffer::new(allowance), + first_code: [0; 16], + first_index: [0; 16], + counts: [0; 16], + } + } + + /// `counts` has been checked for oversubscription and corresponds to + /// `lengths`; only nonzero lengths 1..=15 contribute symbols. Incomplete + /// tables are accepted: unused alphabets and unassigned prefixes occur in + /// the supported formats. Invalid prefixes are rejected when decoded. + pub(super) fn from_counts(lengths: &[u8], counts: [u16; 16], allowance: &B) -> Result { + let mut first_code = [0u16; 16]; + let mut code = 0u16; + for len in 1..=15 { + code = (code + counts[len - 1]) << 1; + first_code[len] = code; + } + let mut first_index = [0usize; 16]; + let mut index = 0usize; + for len in 1..=15 { + first_index[len] = index; + index += usize::from(counts[len]); + } + // Codes of one length follow alphabet order. Scatter directly into + // those ranges instead of storing (code, length, symbol) and sorting. + // Keep usize indices: the public RAR5 constructor also accepts sparse + // alphabets whose symbol index exceeds a 16-bit value. + let mut symbols = Buffer::filled(index, 0, allowance)?; + let mut next_index = first_index; + for (symbol, &len) in lengths.iter().enumerate() { + if len == 0 { + continue; + } + symbols[next_index[len as usize]] = symbol; + next_index[len as usize] += 1; + } + Ok(Self { + symbols, + first_code, + first_index, + counts, + }) + } + + pub(super) fn try_clone(&self) -> Result { + Ok(Self { + symbols: Buffer::copied(&self.symbols, &self.symbols.allowance())?, + first_code: self.first_code, + first_index: self.first_index, + counts: self.counts, + }) + } + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.symbols.len() + } + pub(super) fn is_empty(&self) -> bool { + self.symbols.is_empty() + } + + /// Read only the bits of the selected code. A speculative wide peek would + /// change truncation and incremental-input behavior near an input boundary. + pub(super) fn decode( + &self, + mut read_bit: impl FnMut() -> Result, + empty: &'static str, + invalid: &'static str, + ) -> Result { + if self.is_empty() { + return Err(Error::InvalidData(empty)); + } + let mut code = 0u16; + for len in 1..=15 { + code = (code << 1) | read_bit()?; + let count = self.counts[len]; + if count != 0 { + let offset = code.wrapping_sub(self.first_code[len]); + if offset < count { + let index = self.first_index[len] + usize::from(offset); + return Ok(self.symbols[index]); + } + } + } + Err(Error::InvalidData(invalid)) + } +} + +pub(super) fn validate_counts(counts: &[u16; 16], oversubscribed: &'static str) -> Result<()> { + let mut available = 1i32; + for &count in counts.iter().skip(1) { + available = (available << 1) - i32::from(count); + if available < 0 { + return Err(Error::InvalidData(oversubscribed)); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pinned_codes_keep_alphabet_indices_and_consume_only_their_bits() { + let lengths = [3, 0, 1, 3, 2]; + let mut counts = [0; 16]; + counts[1] = 1; + counts[2] = 1; + counts[3] = 2; + let table = Huffman::from_counts(&lengths, counts, &Allowance::default()).unwrap(); + assert_eq!(table.len(), 4); + // Canonical order is (length, alphabet index): 2=0, 4=10, + // 0=110, 3=111. Each input ends exactly at the selected code. + for (symbol, bits) in [ + (2, vec![0]), + (4, vec![1, 0]), + (0, vec![1, 1, 0]), + (3, vec![1, 1, 1]), + ] { + let mut input = bits.into_iter(); + assert_eq!( + table.decode( + || input.next().ok_or(Error::NeedMoreInput), + "empty", + "invalid" + ), + Ok(symbol) + ); + assert_eq!(input.next(), None); + } + } + + #[test] + fn empty_invalid_truncated_and_cancelled_codes_keep_their_errors() { + let table = Huffman::with_allowance(&Allowance::default()); + assert_eq!( + table.decode( + || panic!("empty table must not read input"), + "empty", + "invalid" + ), + Err(Error::InvalidData("empty")) + ); + let mut counts = [0; 16]; + counts[2] = 1; + validate_counts(&counts, "oversubscribed").unwrap(); + let table = Huffman::from_counts(&[2], counts, &Allowance::default()).unwrap(); + assert_eq!( + table.decode(|| Ok(1), "empty", "invalid"), + Err(Error::InvalidData("invalid")) + ); + assert_eq!( + table.decode(|| Err(Error::NeedMoreInput), "empty", "invalid"), + Err(Error::NeedMoreInput) + ); + assert_eq!( + table.decode(|| Err(Error::Cancelled), "empty", "invalid"), + Err(Error::Cancelled) + ); + } + + #[test] + fn table_and_checkpoint_allocations_are_admitted_and_released() { + let mut counts = [0; 16]; + counts[1] = 2; + let allowance = Allowance::limited(64 * 1024); + let table = Huffman::from_counts(&[1, 1], counts, &allowance).unwrap(); + let retained = allowance.used(); + assert_eq!(retained, (2 * std::mem::size_of::()) as u64); + let copy = table.try_clone().unwrap(); + assert_eq!(allowance.used(), retained * 2); + drop(table); + assert_eq!(copy.decode(|| Ok(1), "empty", "invalid"), Ok(1)); + assert_eq!(allowance.used(), retained); + drop(copy); + assert_eq!(allowance.used(), 0); + let allowance = Allowance::limited(retained); + let table = Huffman::from_counts(&[1, 1], counts, &allowance).unwrap(); + assert!(matches!( + table.try_clone(), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(allowance.used(), retained); + drop(table); + assert_eq!(allowance.used(), 0); + assert!(matches!( + Huffman::from_counts(&[1, 1], counts, &Allowance::limited(0)), + Err(Error::WorkspaceLimitExceeded(_)) + )); + } + + #[test] + fn sparse_alphabet_keeps_the_full_symbol_index() { + let mut lengths = vec![0; 65_538]; + lengths[65_537] = 1; + let mut counts = [0; 16]; + counts[1] = 1; + let table = Huffman::from_counts(&lengths, counts, &Allowance::default()).unwrap(); + assert_eq!(table.decode(|| Ok(0), "empty", "invalid"), Ok(65_537)); + } +} diff --git a/arca-rar/vendor/rars/src/codec/fast.rs b/arca-rar/vendor/rars/src/codec/fast.rs new file mode 100644 index 00000000..a1c08871 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/fast.rs @@ -0,0 +1,246 @@ +/// Append the low `count` bits most-significant first. Complete bytes are +/// extracted from a word; only the two edge bytes need masking. The final +/// partial byte stays zero-padded so callers can inspect or resume the buffer. +#[inline] +#[cfg(any(test, feature = "write"))] +pub(crate) fn write_msb_bits( + bytes: &mut Vec, + bit_pos: &mut usize, + value: u64, + mut count: usize, +) { + debug_assert!(count <= 64); + debug_assert_eq!(bytes.len(), (*bit_pos).div_ceil(8)); + if count == 0 { + return; + } + let used = *bit_pos % 8; + *bit_pos += count; + if used != 0 { + let take = count.min(8 - used); + count -= take; + let mask = ((1u16 << take) - 1) as u8; + *bytes.last_mut().unwrap() |= (((value >> count) as u8) & mask) << (8 - used - take); + } + if count == 0 { + return; + } + let whole = count / 8; + if whole != 0 { + let aligned = (value << (64 - count)).to_be_bytes(); + bytes.extend_from_slice(&aligned[..whole]); + } + let tail = count % 8; + if tail != 0 { + bytes.push((value as u8) << (8 - tail)); + } +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn match_length(input: &[u8], pos: usize, distance: usize, max_length: usize) -> usize { + // Encoders supply older match-finder candidates or remembered distances + // already admitted at an earlier position in this forward-moving parse. + debug_assert!(distance != 0 && distance <= pos); + + let max_length = max_length.min(input.len().saturating_sub(pos)); + match_length_scalar(input, pos, distance, max_length, 0) +} + +#[cfg(any(test, feature = "write"))] +fn match_length_scalar( + input: &[u8], + pos: usize, + distance: usize, + max_length: usize, + mut length: usize, +) -> usize { + while length + 32 <= max_length { + for offset in [0, 8, 16, 24] { + let current = u64::from_le_bytes( + input[pos + length + offset..pos + length + offset + 8] + .try_into() + .unwrap(), + ); + let previous = u64::from_le_bytes( + input[pos + length + offset - distance..pos + length + offset - distance + 8] + .try_into() + .unwrap(), + ); + let difference = current ^ previous; + if difference != 0 { + return length + offset + (difference.trailing_zeros() / 8) as usize; + } + } + length += 32; + } + while length + 8 <= max_length { + let current = u64::from_le_bytes(input[pos + length..pos + length + 8].try_into().unwrap()); + let previous = u64::from_le_bytes( + input[pos + length - distance..pos + length - distance + 8] + .try_into() + .unwrap(), + ); + let difference = current ^ previous; + if difference != 0 { + return length + (difference.trailing_zeros() / 8) as usize; + } + length += 8; + } + while length < max_length && input[pos + length] == input[pos + length - distance] { + length += 1; + } + length +} + +pub(crate) fn next_x86_opcode( + data: &[u8], + start: usize, + end_exclusive: usize, + cmp_mask: u8, +) -> Option { + let end = end_exclusive.min(data.len()); + if start >= end { + return None; + } + + data[start..end] + .iter() + .position(|&byte| byte & cmp_mask == 0xe8) + .map(|offset| start + offset) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn opcode_scan_clamps_the_end_and_handles_empty_extents() { + assert_eq!(next_x86_opcode(b"a\xe8b", 0, usize::MAX, 0xff), Some(1)); + assert_eq!(next_x86_opcode(b"a\xe8b", 2, usize::MAX, 0xff), None); + assert_eq!( + next_x86_opcode(b"a\xe8b", usize::MAX, usize::MAX, 0xff), + None + ); + assert_eq!(next_x86_opcode(b"a\xe8b", 1, 1, 0xff), None); + assert_eq!(next_x86_opcode(b"", 0, usize::MAX, 0xff), None); + } + + fn reference_match_length( + input: &[u8], + pos: usize, + distance: usize, + max_length: usize, + ) -> usize { + let mut length = 0usize; + while length < max_length && input[pos + length] == input[pos + length - distance] { + length += 1; + } + length + } + + #[test] + fn match_length_clamps_the_requested_extent_at_input_end() { + let input = b"abcdabcdabcdabcd"; + for pos in 1..=input.len() { + for distance in 1..=pos { + for requested in [0, 1, 3, 8, 32, usize::MAX] { + let available = requested.min(input.len() - pos); + assert_eq!( + match_length(input, pos, distance, requested), + reference_match_length(input, pos, distance, available), + "pos={pos}, distance={distance}, requested={requested}" + ); + } + } + } + // The public RAR13 match finder also permits positions past EOF. + // There are no bytes to compare, so no indexing is attempted. + assert_eq!(match_length(input, input.len() + 1, 1, usize::MAX), 0); + assert_eq!(match_length(input, usize::MAX, usize::MAX, usize::MAX), 0); + } + + #[test] + fn match_length_matches_scalar_around_chunk_boundaries() { + let mut input = Vec::new(); + input.extend((0..192).map(|index| (index % 251) as u8)); + input.extend_from_within(64..192); + + for distance in 1..=64 { + let pos = 192usize; + let max = (input.len() - pos).min(96); + let expected = reference_match_length(&input, pos, distance, max); + assert_eq!(match_length(&input, pos, distance, max), expected); + } + } + + #[test] + fn match_length_stops_at_first_mismatch_in_chunk_tail() { + let mut input = b"abcdefghijklmnopqrstuvwxyz012345".repeat(4); + let pos = 64; + input[pos + 37] ^= 0x55; + + assert_eq!( + match_length(&input, pos, 32, 64), + reference_match_length(&input, pos, 32, 64) + ); + } + + #[test] + fn x86_opcode_scan_matches_scalar_for_e8_and_e8e9() { + let mut data = vec![0x41u8; 96]; + for pos in [0, 31, 32, 33, 63, 64, 91] { + data[pos] = 0xe8; + } + data[47] = 0xe9; + + for &include_e9 in &[false, true] { + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let mut pos = 0usize; + let mut found = Vec::new(); + while let Some(next) = next_x86_opcode(&data, pos, data.len() - 4, cmp_mask) { + found.push(next); + pos = next + 1; + } + + let expected: Vec<_> = data + .iter() + .take(data.len() - 4) + .enumerate() + .filter_map(|(pos, &byte)| (byte & cmp_mask == 0xe8).then_some(pos)) + .collect(); + assert_eq!(found, expected); + } + } +} + +#[cfg(test)] +mod bit_output_tests { + use super::write_msb_bits; + + #[test] + fn word_output_matches_bit_reference_at_every_alignment_and_width() { + for offset in 0..8 { + for width in 0..=64 { + for value in [0, u64::MAX, 0x0123456789abcdef, 0xfedcba9876543210] { + let mut actual = Vec::new(); + let mut at = 0; + let mut expected = Vec::new(); + let mut bits = 0usize; + for (value, count) in [(0x55, offset), (value, width), (0x1234, 13)] { + write_msb_bits(&mut actual, &mut at, value, count); + for bit in (0..count).rev() { + if bits.is_multiple_of(8) { + expected.push(0); + } + *expected.last_mut().unwrap() |= + (((value >> bit) & 1) as u8) << (7 - bits % 8); + bits += 1; + } + assert_eq!(actual, expected, "offset={offset} width={width}"); + assert_eq!(at, bits); + } + } + } + } + } +} diff --git a/arca-rar/vendor/rars/src/codec/filters.rs b/arca-rar/vendor/rars/src/codec/filters.rs new file mode 100644 index 00000000..3f8c9480 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/filters.rs @@ -0,0 +1,526 @@ +use super::{Error, Result}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg(any(test, feature = "write"))] +pub(crate) enum FilterOp { + #[cfg(test)] + E8, + #[cfg(test)] + E8E9, + Delta { + channels: usize, + }, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct DeltaErrorMessages { + pub invalid_channels: &'static str, + pub zero_channels: &'static str, + pub truncated_source: &'static str, +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn encode_in_place( + op: FilterOp, + data: &mut [u8], + file_offset: u32, + messages: DeltaErrorMessages, +) -> Result<()> { + let _ = file_offset; + match op { + #[cfg(test)] + FilterOp::E8 => e8e9_encode(data, file_offset, false), + #[cfg(test)] + FilterOp::E8E9 => e8e9_encode(data, file_offset, true), + FilterOp::Delta { channels } => { + let encoded = delta_encode(data, channels, messages)?; + data.copy_from_slice(&encoded); + } + } + Ok(()) +} + +/// Channel ceiling on the decode side. +/// +/// RAR 5 cannot exceed 32: its filter record stores `channels - 1` in five +/// bits. The RAR 2.9 VM takes the count from register R[0] instead, and the +/// reference decoder accepts up to 1024 there, so a decoder that stops at 32 +/// refuses archives the official tools extract. The bound still has to exist, +/// because R[0] is a full 32-bit value from the archive and the deinterleave +/// loop runs once per channel whether or not the channel has any bytes. +pub(crate) const MAX_DELTA_CHANNELS: usize = 1024; + +#[cfg(test)] +pub(crate) fn decode_in_place( + op: FilterOp, + data: &mut Vec, + file_offset: u32, + messages: DeltaErrorMessages, +) -> Result<()> { + decode_in_place_with_control( + op, + data, + file_offset, + messages, + &crate::read_control::ReadControl::default(), + ) +} + +#[cfg(test)] +pub(crate) fn decode_in_place_with_control( + op: FilterOp, + data: &mut Vec, + file_offset: u32, + messages: DeltaErrorMessages, + control: &crate::read_control::ReadControl, +) -> Result<()> { + control.check_codec()?; + + match op { + #[cfg(test)] + FilterOp::E8 => e8e9_decode_with_control(data, file_offset, false, control)?, + #[cfg(test)] + FilterOp::E8E9 => e8e9_decode_with_control(data, file_offset, true, control)?, + FilterOp::Delta { channels } => { + *data = delta_decode_with_control(data, channels, messages, control)?; + } + } + Ok(()) +} + +#[cfg(test)] +pub(crate) fn e8e9_decode(data: &mut [u8], file_offset: u32, include_e9: bool) { + e8e9_decode_with_control( + data, + file_offset, + include_e9, + &crate::read_control::ReadControl::default(), + ) + .expect("uncancelled filter"); +} + +pub(crate) fn e8e9_decode_with_control( + data: &mut [u8], + file_offset: u32, + include_e9: bool, + control: &crate::read_control::ReadControl, +) -> Result<()> { + control.check_codec()?; + let mut poller = control.poller(); + if data.len() <= 4 { + return Ok(()); + } + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let opcode_limit = data.len() - 4; + let mut opcode_pos = 0usize; + while opcode_pos < opcode_limit { + poller.check_codec(opcode_pos)?; + let scan_end = if control.is_enabled() { + opcode_limit.min(opcode_pos.saturating_add(64 * 1024)) + } else { + opcode_limit + }; + let Some(pos) = super::fast::next_x86_opcode(data, opcode_pos, scan_end, cmp_mask) else { + opcode_pos = scan_end; + continue; + }; + let cur_pos = pos + 1; + let offset = file_offset.wrapping_add(cur_pos as u32); + let addr = u32::from_le_bytes([ + data[cur_pos], + data[cur_pos + 1], + data[cur_pos + 2], + data[cur_pos + 3], + ]); + let new_addr = if addr < 0x0100_0000 { + Some(addr.wrapping_sub(offset)) + } else if addr & 0x8000_0000 != 0 && addr.wrapping_add(offset) & 0x8000_0000 == 0 { + Some(addr.wrapping_add(0x0100_0000)) + } else { + None + }; + if let Some(value) = new_addr { + data[cur_pos..cur_pos + 4].copy_from_slice(&value.to_le_bytes()); + } + opcode_pos = pos + 5; + } + + Ok(()) +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn e8e9_encode(data: &mut [u8], file_offset: u32, include_e9: bool) { + if data.len() <= 4 { + return; + } + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let opcode_limit = data.len() - 4; + let mut opcode_pos = 0usize; + while let Some(pos) = super::fast::next_x86_opcode(data, opcode_pos, opcode_limit, cmp_mask) { + let cur_pos = pos + 1; + let offset = file_offset.wrapping_add(cur_pos as u32); + let addr = u32::from_le_bytes([ + data[cur_pos], + data[cur_pos + 1], + data[cur_pos + 2], + data[cur_pos + 3], + ]); + let candidate = addr.wrapping_add(offset); + if candidate < 0x0100_0000 { + data[cur_pos..cur_pos + 4].copy_from_slice(&candidate.to_le_bytes()); + } else { + let candidate = addr.wrapping_sub(0x0100_0000); + if candidate & 0x8000_0000 != 0 && candidate.wrapping_add(offset) & 0x8000_0000 == 0 { + data[cur_pos..cur_pos + 4].copy_from_slice(&candidate.to_le_bytes()); + } + } + opcode_pos = pos + 5; + } +} + +#[cfg(test)] +pub(crate) fn delta_decode_with_control( + data: &[u8], + channels: usize, + messages: DeltaErrorMessages, + control: &crate::read_control::ReadControl, +) -> Result> { + delta_decode_with_allowance( + data, + channels, + messages, + control, + &super::workspace::Allowance::default(), + ) + .map(super::workspace::Buffer::into_vec) +} + +pub(crate) fn delta_decode_with_allowance( + data: &[u8], + channels: usize, + messages: DeltaErrorMessages, + control: &crate::read_control::ReadControl, + allowance: &B, +) -> Result> { + control.check_codec()?; + let mut poller = control.poller(); + if channels == 0 { + return Err(Error::InvalidData(messages.zero_channels)); + } + if channels > MAX_DELTA_CHANNELS { + return Err(Error::InvalidData(messages.invalid_channels)); + } + let mut out = super::workspace::Buffer::filled(data.len(), 0, allowance)?; + let mut src = 0usize; + for channel in 0..channels { + let mut prev = 0u8; + let mut dest = channel; + while dest < out.len() { + poller.check_codec(src)?; + let byte = *data + .get(src) + .ok_or(Error::InvalidData(messages.truncated_source))?; + prev = prev.wrapping_sub(byte); + out[dest] = prev; + src += 1; + dest += channels; + } + } + Ok(out) +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn delta_encode( + data: &[u8], + channels: usize, + messages: DeltaErrorMessages, +) -> Result> { + delta_encode_with_allowance( + data, + channels, + messages, + &super::workspace::Allowance::default(), + ) + .map(super::workspace::Buffer::into_vec) +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn delta_encode_with_allowance( + data: &[u8], + channels: usize, + messages: DeltaErrorMessages, + allowance: &B, +) -> Result> { + if channels == 0 || channels > 32 { + return Err(Error::InvalidData(messages.invalid_channels)); + } + let mut out = super::workspace::Buffer::with_capacity(data.len(), allowance)?; + for channel in 0..channels { + let mut prev = 0u8; + let mut src = channel; + while src < data.len() { + let byte = data[src]; + out.push(prev.wrapping_sub(byte)).map_err(Into::into)?; + prev = byte; + src += channels; + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + #[test] + fn short_x86_operands_are_unchanged_and_still_observe_cancellation() { + for length in 0..=4 { + for include_e9 in [false, true] { + let original = vec![0xe8; length]; + let mut data = original.clone(); + super::e8e9_encode(&mut data, u32::MAX, include_e9); + assert_eq!(data, original); + super::e8e9_decode(&mut data, u32::MAX, include_e9); + assert_eq!(data, original); + let token = crate::ReadCancellation::new(); + token.cancel(); + let control = crate::read_control::ReadControl::new(Some(&token)); + assert_eq!( + super::e8e9_decode_with_control(&mut data, 0, include_e9, &control), + Err(super::Error::Cancelled) + ); + assert_eq!(data, original); + } + } + } + + #[test] + fn configured_x86_scanning_matches_default_across_poll_boundaries() { + let mut input = vec![0; 192 * 1024]; + for pos in [65530, 65535, 65541, 131070, 131080] { + input[pos] = 0xe8; + input[pos + 1..pos + 5].copy_from_slice(&123456u32.to_le_bytes()); + } + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + let mut expected = input.clone(); + e8e9_decode(&mut expected, 4096, true); + e8e9_decode_with_control(&mut input, 4096, true, &control).unwrap(); + assert_eq!(input, expected); + } + + #[test] + fn cancellation_interrupts_delta_and_opcode_free_x86_scans() { + for x86 in [false, true] { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(2); + let mut bytes = vec![0; 384 * 1024]; + let result = if x86 { + e8e9_decode_with_control(&mut bytes, 0, true, &control) + } else { + delta_decode_with_control(&bytes, 2, generic_messages(), &control).map(|_| ()) + }; + assert_eq!(result.unwrap_err(), Error::Cancelled); + } + } + use super::*; + + fn generic_messages() -> DeltaErrorMessages { + DeltaErrorMessages { + invalid_channels: "DELTA filter channel count is invalid", + zero_channels: "DELTA filter has zero channels", + truncated_source: "DELTA filter source is truncated", + } + } + + /// Filters a copy, going through the same dispatch the writers use. + fn encode(op: FilterOp, data: &[u8], file_offset: u32) -> Result> { + let mut out = data.to_vec(); + encode_in_place(op, &mut out, file_offset, generic_messages())?; + Ok(out) + } + + fn x86_sample() -> Vec { + let mut data = b"prefix ".to_vec(); + data.extend_from_slice(&[0xe8, 0x10, 0x20, 0x00, 0x00]); + data.extend_from_slice(b" middle "); + data.extend_from_slice(&[0xe9, 0xf0, 0xff, 0xff, 0xff]); + data.extend_from_slice(b" suffix"); + data + } + + fn reference_e8e9_encode(data: &mut [u8], file_offset: u32, include_e9: bool) { + if data.len() <= 4 { + return; + } + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let mut cur_pos = 0usize; + while cur_pos < data.len() - 4 { + cur_pos += 1; + let opcode = data[cur_pos - 1]; + if opcode & cmp_mask == 0xe8 { + let offset = file_offset.wrapping_add(cur_pos as u32); + let addr = u32::from_le_bytes([ + data[cur_pos], + data[cur_pos + 1], + data[cur_pos + 2], + data[cur_pos + 3], + ]); + let candidate = addr.wrapping_add(offset); + if candidate < 0x0100_0000 { + data[cur_pos..cur_pos + 4].copy_from_slice(&candidate.to_le_bytes()); + } else { + let candidate = addr.wrapping_sub(0x0100_0000); + if candidate & 0x8000_0000 != 0 + && candidate.wrapping_add(offset) & 0x8000_0000 == 0 + { + data[cur_pos..cur_pos + 4].copy_from_slice(&candidate.to_le_bytes()); + } + } + cur_pos += 4; + } + } + } + + fn reference_e8e9_decode(data: &mut [u8], file_offset: u32, include_e9: bool) { + if data.len() <= 4 { + return; + } + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let mut cur_pos = 0usize; + while cur_pos < data.len() - 4 { + cur_pos += 1; + let opcode = data[cur_pos - 1]; + if opcode & cmp_mask == 0xe8 { + let offset = file_offset.wrapping_add(cur_pos as u32); + let addr = u32::from_le_bytes([ + data[cur_pos], + data[cur_pos + 1], + data[cur_pos + 2], + data[cur_pos + 3], + ]); + let new_addr = if addr < 0x0100_0000 { + Some(addr.wrapping_sub(offset)) + } else if addr & 0x8000_0000 != 0 && addr.wrapping_add(offset) & 0x8000_0000 == 0 { + Some(addr.wrapping_add(0x0100_0000)) + } else { + None + }; + if let Some(value) = new_addr { + data[cur_pos..cur_pos + 4].copy_from_slice(&value.to_le_bytes()); + } + cur_pos += 4; + } + } + } + + #[test] + fn e8_transform_round_trips_representative_bytes() { + let input = x86_sample(); + let mut filtered = encode(FilterOp::E8, &input, 4096).unwrap(); + + decode_in_place(FilterOp::E8, &mut filtered, 4096, generic_messages()).unwrap(); + + assert_eq!(filtered, input); + } + + #[test] + fn e8e9_transform_round_trips_representative_bytes() { + let input = x86_sample(); + let mut filtered = encode(FilterOp::E8E9, &input, 8192).unwrap(); + + decode_in_place(FilterOp::E8E9, &mut filtered, 8192, generic_messages()).unwrap(); + + assert_eq!(filtered, input); + } + + #[test] + fn e8e9_transform_matches_reference_at_chunk_boundaries_and_skips_payloads() { + let mut input = vec![0x41u8; 104]; + for (pos, address) in [ + (0usize, 0x0000_00e8u32), + (31, 0x0000_0100), + (36, 0x0000_0200), + (64, 0x0000_0300), + (96, 0xffff_ff00), + ] { + input[pos] = if pos == 36 { 0xe9 } else { 0xe8 }; + input[pos + 1..pos + 5].copy_from_slice(&address.to_le_bytes()); + } + input[32] = 0xe8; + input[65] = 0xe9; + + for &include_e9 in &[false, true] { + let mut expected = input.clone(); + let mut actual = input.clone(); + reference_e8e9_encode(&mut expected, 0x1000, include_e9); + e8e9_encode(&mut actual, 0x1000, include_e9); + assert_eq!(actual, expected); + + reference_e8e9_decode(&mut expected, 0x1000, include_e9); + e8e9_decode(&mut actual, 0x1000, include_e9); + assert_eq!(actual, expected); + assert_eq!(actual, input); + } + } + + #[test] + fn delta_transform_round_trips_interleaved_channels() { + let input = b"abcdefghijklmnopqrstuvwxyz0123456789".repeat(3); + let mut filtered = encode(FilterOp::Delta { channels: 3 }, &input, 0).unwrap(); + + decode_in_place( + FilterOp::Delta { channels: 3 }, + &mut filtered, + 0, + generic_messages(), + ) + .unwrap(); + + assert_eq!(filtered, input); + } + + #[test] + fn delta_decode_accepts_channel_counts_the_rar29_vm_can_ask_for() { + // 33 is past what RAR 5 can encode and well inside what the RAR 2.9 + // VM can put in R[0]. Refusing it here refused whole archives. + let mut filtered = vec![0; 64]; + + assert_eq!( + decode_in_place( + FilterOp::Delta { channels: 33 }, + &mut filtered, + 0, + generic_messages(), + ), + Ok(()) + ); + } + + #[test] + fn delta_decode_rejects_channel_counts_above_the_vm_limit() { + let mut filtered = vec![0; 64]; + + assert_eq!( + decode_in_place( + FilterOp::Delta { + channels: MAX_DELTA_CHANNELS + 1, + }, + &mut filtered, + 0, + generic_messages(), + ), + Err(Error::InvalidData("DELTA filter channel count is invalid")) + ); + } + + #[test] + fn encode_in_place_matches_allocating_encode() { + let input = x86_sample(); + let expected = encode(FilterOp::E8E9, &input, 1234).unwrap(); + let mut actual = input; + + encode_in_place(FilterOp::E8E9, &mut actual, 1234, generic_messages()).unwrap(); + + assert_eq!(actual, expected); + } +} diff --git a/arca-rar/vendor/rars/src/codec/huffman.rs b/arca-rar/vendor/rars/src/codec/huffman.rs new file mode 100644 index 00000000..51700bd8 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/huffman.rs @@ -0,0 +1,617 @@ +#[cfg(any(test, feature = "write"))] +use super::workspace::Allowance; +use super::workspace::{Budget, Buffer}; +use super::Result; + +/// Like [`lengths_for_frequencies`], but guarantees the returned code lengths +/// form a *complete* canonical prefix code (Kraft equality) whenever at least +/// one symbol is used. +/// +/// Strict RAR 5 decoders build their tables with 7-Zip's +/// `k_BuildMode_Full_or_Empty`, which rejects any under-full table (a table +/// whose codes leave part of the code space unassigned). A frequency table +/// with a single used symbol otherwise yields one length-1 code — Kraft sum +/// `0.5` — and such archives decode fine in unRAR but fail in 7-Zip / WinRAR +/// with a spurious "Data Error". Real Huffman codes for two or more symbols are +/// already complete, so this only adjusts the degenerate single-symbol case (and +/// the rare uniform-length fallback), padding with phantom codes that are never +/// emitted. +#[cfg(test)] +pub(crate) fn complete_lengths_for_frequencies(frequencies: &[usize], max_bits: u8) -> Vec { + complete_lengths_with_allowance(frequencies, max_bits, &Allowance::default()) + .expect("unlimited Huffman allocation") + .into_vec() +} + +/// Returns true if the non-zero code lengths form a complete prefix code +/// (Kraft sum exactly 1), or if the table is empty (no used symbol). +fn is_complete_code(lengths: &[u8]) -> bool { + // Kraft sum in units of 2^-max_len, accumulated as an integer to avoid + // floating point. A complete code has sum == 2^max_len. + let max_len = lengths.iter().copied().max().unwrap_or(0); + if max_len == 0 { + return true; // empty table + } + let mut sum: u64 = 0; + for &len in lengths { + if len != 0 { + sum += 1u64 << (max_len - len); + } + } + sum == (1u64 << max_len) +} + +/// Overwrites `lengths` with a complete near-uniform canonical code over the +/// currently-used symbols (those with a non-zero length), preserving symbol +/// order. A single used symbol is padded with one phantom length-1 code so the +/// result satisfies Kraft equality. The alphabet must have at least two slots +/// and at least one used symbol. Callers that need a guaranteed-complete code +/// (e.g. tables a strict decoder builds +/// with `Full`/`Full_or_Empty`) can mark used symbols with any non-zero length +/// and call this to normalise them. +pub(crate) fn assign_flat_complete_code(lengths: &mut [u8]) { + let n = lengths.iter().filter(|&&length| length != 0).count(); + // Level tables always encode a nonempty main-table description. Empty + // frequency tables in complete_lengths_with_allowance skip this helper. + debug_assert!(n != 0 && lengths.len() >= 2); + if n == 1 { + let symbol = lengths.iter().position(|&length| length != 0).unwrap(); + lengths[symbol] = 1; + let phantom = usize::from(symbol == 0); + lengths[phantom] = 1; + return; + } + let k = (usize::BITS - (n - 1).leading_zeros()) as u8; + let short_count = (1usize << k) - n; + for (index, length) in lengths + .iter_mut() + .filter(|length| **length != 0) + .enumerate() + { + *length = if index < short_count { k - 1 } else { k }; + } +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn lengths_for_frequencies(frequencies: &[usize], max_bits: u8) -> Vec { + lengths_with_allowance(frequencies, max_bits, &Allowance::default()) + .expect("unlimited Huffman allocation") + .into_vec() +} + +pub(crate) fn complete_lengths_with_allowance( + frequencies: &[usize], + max_bits: u8, + allowance: &B, +) -> Result> { + let mut lengths = lengths_with_allowance(frequencies, max_bits, allowance)?; + if !is_complete_code(&lengths) { + assign_flat_complete_code(&mut lengths); + } + Ok(lengths) +} + +pub(crate) fn lengths_with_allowance( + frequencies: &[usize], + max_bits: u8, + allowance: &B, +) -> Result> { + let used_count = frequencies + .iter() + .filter(|&&frequency| frequency != 0) + .count(); + let mut lengths = Buffer::filled(frequencies.len(), 0u8, allowance)?; + if used_count <= 1 { + uniform_lengths_into(&mut lengths, frequencies); + return Ok(lengths); + } + + // Parents replace the old per-node symbol vectors. Node order breaks ties + // exactly as the original heap did; every parent is created after its children. + let capacity = frequencies + .len() + .checked_add(used_count - 1) + .ok_or(super::Error::InvalidData("Huffman node count overflows"))?; + let mut parents = Buffer::filled(capacity, usize::MAX, allowance)?; + let mut heap = Buffer::with_capacity(used_count, allowance)?; + let mut order = 0; + for (symbol, &frequency) in frequencies.iter().enumerate() { + if frequency != 0 { + heap.push((frequency, order, symbol)).map_err(Into::into)?; + order += 1; + } + } + // Sorting once constructs a valid min-heap without auxiliary allocation. + heap.sort_unstable(); + let mut next = frequencies.len(); + while heap.len() > 1 { + let (left_frequency, _, left) = pop_min(&mut heap); + let (right_frequency, _, right) = pop_min(&mut heap); + parents[left] = next; + parents[right] = next; + heap.push((left_frequency.saturating_add(right_frequency), order, next)) + .map_err(Into::into)?; + let mut child = heap.len() - 1; + while child != 0 { + let parent = (child - 1) / 2; + if heap[parent] <= heap[child] { + break; + } + heap.swap(parent, child); + child = parent; + } + next += 1; + order += 1; + } + // Replace parent indices with depths in reverse creation order. + for node in (0..next).rev() { + parents[node] = if parents[node] == usize::MAX { + 0 + } else { + parents[parents[node]] + 1 + }; + } + for (length, &depth) in lengths.iter_mut().zip(parents.iter()) { + *length = depth as u8; + } + if lengths.iter().any(|&length| length > max_bits) { + limit_code_lengths(&mut lengths, frequencies, max_bits); + } + Ok(lengths) +} + +fn pop_min(heap: &mut Buffer<(usize, usize, usize), B>) -> (usize, usize, usize) { + let last = heap.pop().expect("frequency heap has a node"); + if heap.is_empty() { + return last; + } + let first = std::mem::replace(&mut heap[0], last); + let mut parent = 0; + loop { + let left = parent * 2 + 1; + if left >= heap.len() { + break; + } + let right = left + 1; + let child = if right < heap.len() && heap[right] < heap[left] { + right + } else { + left + }; + if heap[parent] <= heap[child] { + break; + } + heap.swap(parent, child); + parent = child; + } + first +} + +fn uniform_lengths_into(lengths: &mut [u8], frequencies: &[usize]) { + let bits = bits_for_symbol_count( + frequencies + .iter() + .filter(|&&frequency| frequency != 0) + .count(), + ); + for (length, &frequency) in lengths.iter_mut().zip(frequencies) { + *length = if frequency == 0 { 0 } else { bits }; + } +} + +/// Bring every code length down to `max_bits`, keeping the code complete. +/// +/// A code deeper than the format allows used to be replaced by a uniform one +/// over the same symbols. That is a far worse code: on a skewed 289 symbol main +/// table a flat code spends nine bits on every token where the Huffman code it +/// replaced averaged closer to six, and the codes that trip the limit overflow +/// it by a bit or two, not by enough to justify giving up. +/// +/// Clamping the over-long lengths breaks Kraft equality, so this pays the +/// difference back by lengthening the symbols that cost least per bit, then +/// spends whatever is left over on the most frequent ones so the code comes out +/// complete. Package-merge would be optimal; this is within a fraction of a +/// percent of it and much shorter. +fn limit_code_lengths(lengths: &mut [u8], frequencies: &[usize], max_bits: u8) { + // Writers use the format's 15-bit limit. The generic limiter requires a + // positive limit representable as a prefix-space size on this host. + debug_assert!((1..usize::BITS as u8).contains(&max_bits)); + // No prefix code over this many symbols fits in this many bits however it + // is arranged, so there is nothing to repair towards. + let used = lengths.iter().filter(|&&length| length != 0).count(); + if used > (1usize << max_bits) { + uniform_lengths_into(lengths, frequencies); + return; + } + + for length in lengths.iter_mut() { + if *length > max_bits { + *length = max_bits; + } + } + + // Kraft sum in units of 2^-max_bits, so it stays exact in integers. + let target: u64 = 1 << max_bits; + let mut sum: u64 = lengths + .iter() + .filter(|&&length| length != 0) + .map(|&length| 1u64 << (max_bits - length)) + .sum(); + + while sum > target { + // If every used length were max_bits, sum would equal used, which + // the capacity check above bounds by target. An overfull code must + // therefore have a symbol whose length can still be increased. + let symbol = (0..lengths.len()) + .filter(|&symbol| lengths[symbol] != 0 && lengths[symbol] < max_bits) + .min_by_key(|&symbol| (frequencies[symbol], std::cmp::Reverse(lengths[symbol]))) + .expect("overfull prefix code has a short symbol"); + sum -= 1u64 << (max_bits - lengths[symbol] - 1); + lengths[symbol] += 1; + } + + while let Some(symbol) = (0..lengths.len()) + .filter(|&symbol| lengths[symbol] > 1) + .filter(|&symbol| sum + (1u64 << (max_bits - lengths[symbol])) <= target) + .max_by_key(|&symbol| (frequencies[symbol], lengths[symbol])) + { + sum += 1u64 << (max_bits - lengths[symbol]); + lengths[symbol] -= 1; + } +} + +#[cfg(feature = "write")] +pub(crate) fn lengths_for_frequency_array( + frequencies: &[usize; N], + max_bits: u8, +) -> [u8; N] { + let mut lengths = [0u8; N]; + lengths.copy_from_slice(&lengths_for_frequencies(frequencies, max_bits)); + lengths +} + +pub(crate) fn bits_for_symbol_count(count: usize) -> u8 { + match count { + 0 | 1 => 1, + _ => usize::BITS as u8 - (count - 1).leading_zeros() as u8, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn charged_tree_preserves_heap_ties_and_saturating_frequencies() { + // Independent reference for the replaced per-node symbol-list tree. + fn reference(frequencies: &[usize]) -> Vec { + use std::{cmp::Reverse, collections::BinaryHeap}; + let mut lengths = vec![0u8; frequencies.len()]; + let mut heap = BinaryHeap::new(); + let mut order = 0; + for (symbol, &frequency) in frequencies.iter().enumerate() { + if frequency != 0 { + heap.push(Reverse((frequency, order, vec![symbol]))); + order += 1; + } + } + if heap.len() <= 1 { + uniform_lengths_into(&mut lengths, frequencies); + return lengths; + } + while heap.len() > 1 { + let Reverse((left, _, mut symbols)) = heap.pop().unwrap(); + let Reverse((right, _, mut other)) = heap.pop().unwrap(); + for &symbol in symbols.iter().chain(&other) { + lengths[symbol] += 1; + } + symbols.append(&mut other); + heap.push(Reverse((left.saturating_add(right), order, symbols))); + order += 1; + } + if lengths.iter().any(|&length| length > 15) { + limit_code_lengths(&mut lengths, frequencies, 15); + } + lengths + } + let allowance = Allowance::limited(1024 * 1024); + let mut state = 71u64; + for count in [0, 1, 2, 20, 64, 306, 1024] { + for shape in 0..8 { + let frequencies: Vec<_> = (0..count) + .map(|_| { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1); + match shape { + 0 => 0, + 1 => 1, + 2 => usize::MAX, + 3 => (state % 3) as usize, + _ => (state >> (shape * 7)) as usize, + } + }) + .collect(); + let actual = lengths_with_allowance(&frequencies, 15, &allowance).unwrap(); + assert_eq!(&*actual, reference(&frequencies)); + assert_eq!(allowance.used(), count as u64); + drop(actual); + assert_eq!(allowance.used(), 0); + } + } + } + + #[test] + fn tree_refusal_releases_scratch_and_success_retains_only_lengths() { + let frequencies = [1, 3, 7, 11]; + let peak = 4 + + 7 * std::mem::size_of::() as u64 + + 4 * std::mem::size_of::<(usize, usize, usize)>() as u64; + for limit in [0, 4, peak - 1] { + let allowance = Allowance::limited(limit); + assert!(matches!( + lengths_with_allowance(&frequencies, 15, &allowance), + Err(super::super::Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(allowance.used(), 0); + } + let allowance = Allowance::limited(peak); + let lengths = complete_lengths_with_allowance(&frequencies, 15, &allowance).unwrap(); + assert_eq!(allowance.used(), 4); + drop(lengths); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn weighted_lengths_favour_common_symbols() { + let frequencies = [1, 1, 16, 1]; + let lengths = lengths_for_frequencies(&frequencies, 15); + + assert!(lengths[2] < lengths[0]); + assert!(lengths.iter().all(|&length| length <= 15)); + } + + #[test] + fn lengths_too_deep_to_limit_fall_back_to_uniform_lengths() { + // 1024 symbols cannot be spelled in one bit however they are arranged, + // so there is no length-limited code to find. + let frequencies = (1..=1024).collect::>(); + let lengths = lengths_for_frequencies(&frequencies, 1); + + assert!(lengths.iter().all(|&length| length == 10)); + } + + #[test] + fn deep_codes_are_limited_rather_than_flattened() { + // Fibonacci frequencies are the worst case for Huffman depth, and this + // alphabet wants codes far deeper than fifteen bits. + let mut frequencies = vec![1usize, 1]; + while frequencies.len() < 40 { + let next = frequencies[frequencies.len() - 1] + frequencies[frequencies.len() - 2]; + frequencies.push(next); + } + let mut sparse = vec![0; 306]; + for (symbol, &frequency) in frequencies.iter().enumerate() { + sparse[symbol * 7 + 3] = frequency; + } + for frequencies in [frequencies, sparse] { + let lengths = lengths_for_frequencies(&frequencies, 15); + + assert!(lengths.iter().all(|&length| length <= 15), "{lengths:?}"); + assert!(is_complete_code(&lengths), "{lengths:?}"); + for (&frequency, &length) in frequencies.iter().zip(&lengths) { + assert_eq!(frequency == 0, length == 0); + } + // A flat code over forty symbols is six bits each, and on frequencies + // this skewed the limited code has to beat that. + let flat: usize = frequencies.iter().sum::() * 6; + let limited: usize = frequencies + .iter() + .zip(&lengths) + .map(|(&frequency, &length)| frequency * usize::from(length)) + .sum(); + assert!(limited < flat, "limited {limited} flat {flat}"); + } + } + + #[test] + fn limited_codes_stay_decodable_for_every_shape_of_input() { + // rar29 feeds this straight into a table it writes, with no validation + // pass behind it, so the one thing that must always hold is that the + // result is a usable prefix code inside the format's bit limit. + let mut seed = 0x2545_f491_4f6c_dd1du64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for symbols in [2usize, 3, 17, 64, 255, 299] { + for skew in 0..8 { + let frequencies: Vec = (0..symbols) + .map(|_| match skew { + // Increasingly lopsided distributions, which is what + // drives a Huffman code past the depth limit. + 0 => 1, + s => 1 + (next() % (1u64 << (s * 8).min(60))) as usize, + }) + .collect(); + let lengths = lengths_for_frequencies(&frequencies, 15); + + assert_eq!(lengths.len(), symbols); + assert!( + lengths.iter().all(|&length| length <= 15), + "over the limit: {symbols} symbols, skew {skew}" + ); + let deepest = lengths.iter().copied().max().unwrap_or(0); + if deepest > 0 { + let kraft: u64 = lengths + .iter() + .filter(|&&length| length != 0) + .map(|&length| 1u64 << (deepest - length)) + .sum(); + assert!( + kraft <= 1u64 << deepest, + "not a prefix code: {symbols} symbols, skew {skew}" + ); + } + } + } + } + + fn kraft_sum_is_one(lengths: &[u8]) -> bool { + let max_len = lengths.iter().copied().max().unwrap_or(0); + if max_len == 0 { + return lengths.iter().all(|&len| len == 0); + } + let sum: u64 = lengths + .iter() + .filter(|&&len| len != 0) + .map(|&len| 1u64 << (max_len - len)) + .sum(); + sum == (1u64 << max_len) + } + + #[test] + fn flat_codes_complete_every_small_nonempty_symbol_subset() { + for slots in 2..=8 { + for mask in 1usize..1 << slots { + let mut lengths: Vec<_> = (0..slots) + .map(|symbol| if mask & (1 << symbol) != 0 { 7 } else { 0 }) + .collect(); + assign_flat_complete_code(&mut lengths); + let used = mask.count_ones() as usize; + assert_eq!(lengths.iter().filter(|&&n| n != 0).count(), used.max(2)); + for (symbol, &length) in lengths.iter().enumerate() { + if mask & (1 << symbol) != 0 { + assert_ne!(length, 0); + } else if used > 1 { + assert_eq!(length, 0); + } + } + // Count concrete fixed-width bit strings owned by each code + // length. A complete prefix code fills the entire space. + let deepest = lengths.iter().copied().max().unwrap(); + let mut ordered: Vec<_> = lengths.iter().copied().filter(|&n| n != 0).collect(); + ordered.sort_unstable(); + let mut next = 0usize; + for length in ordered { + let width = 1usize << (deepest - length); + assert_eq!(next % width, 0); + next += width; + } + assert_eq!(next, 1 << deepest); + let shallowest = lengths.iter().copied().filter(|&n| n != 0).min().unwrap(); + assert!(deepest - shallowest <= 1); + } + } + } + + #[test] + fn small_alphabets_have_non_overlapping_canonical_codes() { + for symbols in 2u32..=7 { + for mut pattern in 0..3usize.pow(symbols) { + let frequencies: Vec<_> = (0..symbols) + .map(|_| { + let frequency = pattern % 3; + pattern /= 3; + frequency + }) + .collect(); + let used = frequencies + .iter() + .filter(|&&frequency| frequency != 0) + .count(); + for max_bits in 1..=4 { + let lengths = lengths_for_frequencies(&frequencies, max_bits); + for (&frequency, &length) in frequencies.iter().zip(&lengths) { + assert_eq!(frequency == 0, length == 0); + } + if used > 1usize << max_bits { + // An impossible limit retains the documented uniform fallback. + let uniform = (usize::BITS - (used - 1).leading_zeros()) as u8; + assert!(lengths + .iter() + .all(|&length| length == 0 || length == uniform)); + } else { + assert!(lengths.iter().all(|&length| length <= max_bits)); + } + + // Assign canonical integers independently, then occupy every + // suffix below each prefix. No two symbols may share a slot. + let depth = lengths.iter().copied().max().unwrap_or(0); + let mut occupied = vec![false; 1usize << depth]; + let mut ordered: Vec<_> = lengths.iter().copied().filter(|&n| n != 0).collect(); + ordered.sort_unstable(); + let mut code = 0usize; + let mut previous = 0; + for length in ordered { + code <<= length - previous; + let start = code << (depth - length); + let end = start + (1usize << (depth - length)); + assert!(end <= occupied.len(), "{frequencies:?}: {lengths:?}"); + for slot in &mut occupied[start..end] { + assert!(!*slot, "{frequencies:?}: {lengths:?}"); + *slot = true; + } + code += 1; + previous = length; + } + } + } + } + } + + #[test] + fn single_symbol_table_is_completed_with_a_phantom_code() { + // A lone used symbol would otherwise get one length-1 code (Kraft 0.5), + // which strict RAR 5 decoders reject. It must be padded to a complete + // code without disturbing the used symbol's own length. + for used in [0usize, 1, 7, 40] { + let mut frequencies = vec![0usize; 44]; + frequencies[used] = 123; + let lengths = complete_lengths_for_frequencies(&frequencies, 15); + assert_eq!(lengths[used], 1, "used symbol {used} keeps a length-1 code"); + assert_eq!( + lengths.iter().filter(|&&len| len != 0).count(), + 2, + "exactly one phantom code was added for used symbol {used}" + ); + assert!( + kraft_sum_is_one(&lengths), + "used symbol {used} yields a complete code" + ); + } + } + + #[test] + fn empty_table_stays_empty() { + let lengths = complete_lengths_for_frequencies(&[0usize; 16], 15); + assert!(lengths.iter().all(|&len| len == 0)); + } + + #[test] + fn completed_codes_are_always_complete_for_any_symbol_count() { + for used_count in 1..=64usize { + let mut frequencies = vec![0usize; 306]; + for (i, freq) in frequencies.iter_mut().take(used_count).enumerate() { + *freq = 1 + i; // distinct frequencies, still a valid Huffman input + } + let lengths = complete_lengths_for_frequencies(&frequencies, 15); + assert!( + kraft_sum_is_one(&lengths), + "code for {used_count} symbols must be complete" + ); + assert!(lengths.iter().all(|&len| len <= 15)); + } + } + + #[test] + fn multi_symbol_huffman_code_is_left_optimal() { + // A skewed distribution already yields a complete Huffman code; the + // completeness pass must not flatten it into a uniform code. + let frequencies = [100usize, 1, 1, 1, 1]; + let optimal = lengths_for_frequencies(&frequencies, 15); + let completed = complete_lengths_for_frequencies(&frequencies, 15); + assert_eq!(optimal, completed); + assert!(kraft_sum_is_one(&completed)); + } +} diff --git a/arca-rar/vendor/rars/src/codec/match_finder.rs b/arca-rar/vendor/rars/src/codec/match_finder.rs new file mode 100644 index 00000000..ec5d11f2 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/match_finder.rs @@ -0,0 +1,479 @@ +//! Match finders shared by the LZ encoders. +//! +//! [`MatchFinder`] is a hash chain. Positions are indexed into a +//! single-bucket-per-hash chain: `head` maps a hash of the `MIN_MATCH` bytes +//! at a position to the most recently inserted position, and `prev` links each +//! inserted position to the previous position with the same hash. Walking a +//! chain therefore visits candidates from the smallest distance to the +//! largest, so callers can stop as soon as a candidate falls outside their +//! window. Inserting costs one store, so a caller can skip past bytes it has +//! already decided about for free, which is what the lazy encoders do. +//! +//! [`TreeMatchFinder`] keeps the positions sharing a hash in a binary tree +//! ordered by the bytes that follow them, after LZMA. One descent finds every +//! improving match and re-hangs the tree under the new position, and each node +//! on the way down narrows a proven-prefix bound, so the descent never re-reads +//! bytes it has already matched. That is what a deep chain walk cannot avoid, +//! and why the tree serves the optimal parse, which searches at every position +//! anyway. The price is that inserting *is* a descent, so history cannot be +//! seeded cheaply, and the tree costs eight bytes per byte of window against +//! the chain's four. +//! +//! `prev` holds one link per position in a window rather than one per position +//! in the input, indexed by position modulo the window. A link is only followed +//! while the candidate it names is inside the window, and a slot is only reused +//! once the position it held has fallen out, so the two never overlap. That +//! bounds the finder by the window instead of by the data, which is what lets +//! one finder span every block of a member: rebuilding it per block meant +//! rehashing the whole history each time, which measured at about 40% of the +//! encode on a 16 MiB member. +//! +//! Links are 32 bits, so the window costs four bytes per byte it covers rather +//! than eight, and twice as many links share a cache line. Positions past four +//! gigabytes keep only their low half; `resolve` measures back from the newest +//! position inserted to recover the rest. + +use super::workspace::{Allowance, Budget, Buffer}; +use super::Result; + +/// Sentinel for "no position" in `head`/`prev` chains. +pub(crate) const NO_POSITION: usize = usize::MAX; + +/// The sentinel as it is stored. A position whose low 32 bits are all set is +/// indistinguishable from it and ends its chain one link early. That costs one +/// candidate every four gigabytes and cannot cost correctness: a caller +/// confirms every candidate by comparing the bytes at it. +const NO_LINK: u32 = u32::MAX; + +/// Reads a stored link back as the position it names, given the newest position +/// the finder holds. +/// +/// Only the low 32 bits of a position survive storage. No link names a position +/// newer than `newest`, so measuring back from `newest` recovers it exactly +/// whenever the input fits in 32 bits, and within four gigabytes of the true one +/// when it does not. Neither case can underflow: below four gigabytes the stored +/// link is the position itself and so no higher than `newest`, and above it the +/// distance is a `u32` and `newest` is not. +fn resolve(newest: usize, link: u32) -> usize { + if link == NO_LINK { + return NO_POSITION; + } + newest - (newest as u32).wrapping_sub(link) as usize +} + +/// The multiplicative mix every finder hashes with. +fn mix(value: u32, bits: u32) -> usize { + (value.wrapping_mul(0x9E37_79B1) >> (32 - bits)) as usize +} + +#[derive(Debug)] +pub(crate) struct MatchFinder { + head: Buffer, + prev: Buffer, + mask: usize, + /// The highest position inserted so far, and so at least as high as any + /// position a link can name. Truncated links are read back against it. + newest: usize, +} + +#[cfg(any(test, feature = "write"))] +impl MatchFinder { + pub(crate) fn new(window: usize) -> Self { + Self::with_allowance(window, &Allowance::default()).expect("unlimited finder allocation") + } +} + +impl MatchFinder { + const HASH_BITS: u32 = match MIN_MATCH { + 3 => 16, + 4 => 17, + _ => panic!("match finder supports MIN_MATCH of 3 or 4"), + }; + + /// Builds a finder that remembers the last `window` positions. + /// + /// The caller must not accept a match further back than `window`, which is + /// the check it already makes against its own maximum distance. A link to a + /// position that has fallen out of the window is still readable, and still + /// names the position it named, so that check rejects it the same way it + /// rejects a match that is merely too far away. + pub(crate) fn with_allowance(window: usize, allowance: &B) -> Result { + let window = window + .max(1) + .checked_next_power_of_two() + .ok_or(super::Error::InvalidData("finder window overflows"))?; + Ok(Self { + head: Buffer::filled(1 << Self::HASH_BITS, NO_LINK, allowance)?, + // Zero rather than the sentinel, and nothing reads it either way: a + // link is only ever followed from a position that has been + // inserted, and inserting a position writes its slot first. Zero + // lets the allocator hand back pages it has not had to touch, so a + // window wider than the data has reached costs nothing yet. + prev: Buffer::filled(window, 0, allowance)?, + mask: window - 1, + newest: 0, + }) + } + + #[cfg(feature = "write")] + pub(crate) fn allowance(&self) -> B { + self.head.allowance() + } + + fn hash(input: &[u8], pos: usize) -> usize { + let value = if MIN_MATCH == 3 { + u32::from(input[pos]) + | (u32::from(input[pos + 1]) << 8) + | (u32::from(input[pos + 2]) << 16) + } else { + u32::from_le_bytes([input[pos], input[pos + 1], input[pos + 2], input[pos + 3]]) + }; + mix(value, Self::HASH_BITS) + } + + /// Records `pos` as a future match candidate. Positions too close to the + /// end of the input to fit `MIN_MATCH` bytes are ignored. + pub(crate) fn insert(&mut self, input: &[u8], pos: usize) { + if pos + MIN_MATCH <= input.len() { + let hash = Self::hash(input, pos); + self.prev[pos & self.mask] = self.head[hash]; + self.head[hash] = pos as u32; + self.newest = self.newest.max(pos); + } + } + + /// Returns the most recently inserted candidate sharing `pos`'s hash, or + /// [`NO_POSITION`]. The caller must ensure at least `MIN_MATCH` bytes are + /// readable at `pos`. + pub(crate) fn first(&self, input: &[u8], pos: usize) -> usize { + resolve(self.newest, self.head[Self::hash(input, pos)]) + } + + /// Returns the next-older candidate in `candidate`'s chain, or + /// [`NO_POSITION`]. + pub(crate) fn previous(&self, candidate: usize) -> usize { + let older = resolve(self.newest, self.prev[candidate & self.mask]); + // Chains run newest to oldest. A link that does not step back is one + // whose slot has been reused, or whose position wrapped past four + // gigabytes; either way the chain ends here, which is what keeps the + // walk finite. This also covers the sentinel, which is above every + // position. + if older >= candidate { + return NO_POSITION; + } + older + } +} + +/// A binary-tree match finder, after LZMA's BT4. +/// +/// Positions sharing a hash of their first four bytes form a binary search +/// tree ordered by the bytes at them, newest position at the root. The two +/// children of each position live in `son`, one window slot per position as +/// with the chain finder's `prev`, and the same rules keep slot reuse and the +/// window from ever overlapping. Truncated links are read back with the +/// position being searched for as the reference, which is always the newest. +#[derive(Debug)] +pub(crate) struct TreeMatchFinder { + head: Buffer, + /// Two links per window slot: the child whose bytes compare lesser first, + /// then the greater-or-equal one, in LZMA's layout. + son: Buffer, + mask: usize, +} + +impl TreeMatchFinder { + #[cfg(test)] + pub(crate) fn new(window: usize) -> Self { + Self::with_allowance(window, &Allowance::default()).expect("unlimited finder allocation") + } +} + +impl TreeMatchFinder { + const HASH_BITS: u32 = 17; + const MIN_MATCH: usize = 4; + + /// Builds a finder that remembers the last `window` positions, at eight + /// bytes of links per byte of window. + pub(crate) fn with_allowance(window: usize, allowance: &B) -> Result { + let window = window + .max(1) + .checked_next_power_of_two() + .ok_or(super::Error::InvalidData("finder window overflows"))?; + Ok(Self { + head: Buffer::filled(1 << Self::HASH_BITS, NO_LINK, allowance)?, + // Zero rather than the sentinel for the same reason as the chain + // finder's `prev`: a slot is always written during its position's + // own insertion before any link can lead to it, and untouched + // zeroes let the allocator defer the pages. + son: Buffer::filled( + window + .checked_mul(2) + .ok_or(super::Error::InvalidData("finder window overflows"))?, + 0, + allowance, + )?, + mask: window - 1, + }) + } + + #[cfg(feature = "write")] + pub(crate) fn allowance(&self) -> B { + self.head.allowance() + } + + /// Finds the matches at `pos` and inserts `pos`, in one descent. + /// + /// Pushes `(length, distance)` pairs onto `out` with both strictly + /// increasing, so each pair is the nearest distance found that reaches its + /// length, and nothing shorter than four bytes or further than + /// `max_distance` is reported. Comparing stops at `len_limit`, which must + /// leave `pos + len_limit` readable; a pair whose length equals it may + /// really extend further, which the caller measures if it cares. A node + /// that matches the whole limit gives its place to the new position, since + /// the two are interchangeable prefixes and the new one is nearer + /// everything to come. `cut` bounds the nodes visited, + /// and whatever hangs below the last one is cut off rather than left + /// dangling. The caller must skip positions with fewer than four bytes + /// left, as the tree collector does. On allocation refusal, discard the finder: + /// the insertion may have been partially applied before output growth failed. + pub(crate) fn matches( + &mut self, + input: &[u8], + pos: usize, + len_limit: usize, + max_distance: usize, + cut: usize, + out: &mut Buffer<(u32, u32), B>, + ) -> std::result::Result<(), B::Failure> { + // The tree collector calls this only when at least four bytes remain. + // Unlike the chain finder, it skips near-end positions before calling. + debug_assert!(len_limit >= Self::MIN_MATCH && pos + len_limit <= input.len()); + let hash = mix( + u32::from_le_bytes([input[pos], input[pos + 1], input[pos + 2], input[pos + 3]]), + Self::HASH_BITS, + ); + let mut current = resolve(pos, self.head[hash]); + self.head[hash] = pos as u32; + // The two attachment points still waiting for a subtree, starting as + // the new position's own child slots. Each step down hangs the node + // just compared on one of them and moves that side into the node's + // matching child slot. + let mut ptr0 = ((pos & self.mask) << 1) + 1; + let mut ptr1 = (pos & self.mask) << 1; + // How much of the prefix is proven to match on each side of the + // descent. Everything below a node compares the same way the node + // did up to its recorded length, so bytes before the smaller of the + // two never need reading again. + let mut len0 = 0usize; + let mut len1 = 0usize; + let mut longest = Self::MIN_MATCH - 1; + let mut budget = cut; + let mut floor = pos; + loop { + // A candidate that does not step back is a reused slot or the + // sentinel; one further back than the window has fallen out of it. + // Either ends the descent, sealing both attachment points so no + // stale link survives below them. A spent budget ends it the same + // way, dropping whatever subtree the budget could not reach. + if current >= floor || pos - current > self.mask || budget == 0 { + self.son[ptr0] = NO_LINK; + self.son[ptr1] = NO_LINK; + return Ok(()); + } + budget -= 1; + floor = current; + let pair = (current & self.mask) << 1; + let mut len = len0.min(len1); + if input[current + len] == input[pos + len] { + len += 1; + while len < len_limit && input[current + len] == input[pos + len] { + len += 1; + } + if len > longest { + if pos - current <= max_distance { + out.push((len as u32, (pos - current) as u32))?; + } + longest = len; + if len == len_limit { + // The node's whole comparable prefix is the new + // position's prefix, so the new position adopts its + // children and the node drops out as the farther of + // two interchangeable candidates. + self.son[ptr1] = self.son[pair]; + self.son[ptr0] = self.son[pair + 1]; + return Ok(()); + } + } + } + if input[current + len] < input[pos + len] { + self.son[ptr1] = current as u32; + ptr1 = pair + 1; + len1 = len; + current = resolve(pos, self.son[ptr1]); + } else { + self.son[ptr0] = current as u32; + ptr0 = pair; + len0 = len; + current = resolve(pos, self.son[ptr0]); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::{resolve, MatchFinder, TreeMatchFinder, NO_LINK, NO_POSITION}; + + /// Four repeats of the same four bytes, so every position shares a hash and + /// they form one chain. + const REPEATED: &[u8] = b"abcdabcdabcdabcd"; + + #[test] + fn a_chain_walks_from_the_nearest_candidate_to_the_furthest() { + let mut finder = MatchFinder::<4>::new(REPEATED.len()); + for pos in [0, 4, 8] { + finder.insert(REPEATED, pos); + } + + let mut walked = Vec::new(); + let mut candidate = finder.first(REPEATED, 12); + while candidate != NO_POSITION { + walked.push(candidate); + candidate = finder.previous(candidate); + } + assert_eq!(walked, [8, 4, 0]); + } + + #[test] + fn a_position_that_has_fallen_out_of_the_window_still_names_itself() { + // A candidate three thousand bytes back, found through a window of + // sixty-four. It reads back as the position it named rather than as + // something inside the window, which is what leaves the caller to + // reject it on distance. + let data = vec![b'x'; 4096]; + let mut finder = MatchFinder::<4>::new(64); + finder.insert(&data, 0); + finder.insert(&data, 2048); + assert_eq!(finder.first(&data, 3000), 2048); + assert_eq!(finder.previous(2048), 0); + } + + #[test] + fn resolving_recovers_a_position_stored_below_four_gigabytes() { + assert_eq!(resolve(1_000, 0), 0); + assert_eq!(resolve(1_000, 999), 999); + assert_eq!(resolve(1_000, 1_000), 1_000); + assert_eq!(resolve(1_000, NO_LINK), NO_POSITION); + } + + /// Every improving match at each position of `input`, walked in order the + /// way the optimal parse's collector does. + fn tree_matches_at_every_position(input: &[u8], cut: usize) -> Vec> { + let mut finder = TreeMatchFinder::new(input.len()); + let mut all = Vec::new(); + for pos in 0..input.len() { + let mut out = super::Buffer::new(&super::Allowance::default()); + if pos + 4 <= input.len() { + let len_limit = input.len() - pos; + finder + .matches(input, pos, len_limit, pos, cut, &mut out) + .unwrap(); + } + all.push(out.into_vec()); + } + all + } + + #[test] + fn the_tree_reports_the_nearest_distance_for_each_length() { + // At the last "abcd" the nearest candidate matches everything there is, + // so it is the only report; nothing farther can improve on it. + let all = tree_matches_at_every_position(REPEATED, usize::MAX); + assert_eq!(all[12], [(4, 4)]); + // At position 8 both distances 4 and 8 match 8 bytes; only the nearer + // is worth reporting. + assert_eq!(all[8], [(8, 4)]); + } + + #[test] + fn the_tree_reports_runs_of_increasing_length_and_distance() { + // "abc" twelve bytes back, "abcde" twenty-two back. The parse wants + // both: the nearer is cheaper at its length, the farther reaches + // longer. + let input = b"abcdeXXXXX_abcdfYYYY__abcdeZZ"; + let all = tree_matches_at_every_position(input, usize::MAX); + assert_eq!(all[22], [(4, 11), (5, 22)]); + } + + #[test] + fn a_spent_budget_ends_the_descent_but_keeps_what_it_found() { + let input = b"abcdeXXXXX_abcdfYYYY__abcdeZZ"; + let all = tree_matches_at_every_position(input, 1); + assert_eq!(all[22], [(4, 11)]); + } + + #[test] + fn the_tree_drops_candidates_outside_its_window() { + let input = vec![b'x'; 4096]; + let mut finder = TreeMatchFinder::new(64); + let mut matches = super::Buffer::new(&super::Allowance::default()); + finder.matches(&input, 0, 16, 0, 8, &mut matches).unwrap(); + finder + .matches(&input, 2048, 16, 2048, 8, &mut matches) + .unwrap(); + assert!(matches.is_empty()); + } + + #[test] + fn the_tree_does_not_report_matches_beyond_the_requested_distance() { + let input = vec![b'x'; 64]; + let mut finder = TreeMatchFinder::new(64); + let mut matches = super::Buffer::new(&super::Allowance::default()); + finder.matches(&input, 0, 16, 0, 8, &mut matches).unwrap(); + finder.matches(&input, 16, 16, 8, 8, &mut matches).unwrap(); + assert!(matches.is_empty()); + } + + #[test] + fn a_refused_tree_child_table_releases_its_head_table() { + let head_bytes = (1 << 17) * std::mem::size_of::(); + let allowance = super::Allowance::limited(head_bytes as u64); + assert!(TreeMatchFinder::with_allowance(64, &allowance).is_err()); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn a_node_matching_the_whole_limit_hands_its_children_to_the_new_position() { + // Degenerate data: every position matches every earlier one to the + // limit, so each descent must stop at its first node rather than walk + // them all. Matching to the limit and improving on nothing are the + // only exits that leave a single report. + let input = vec![b'z'; 512]; + let all = tree_matches_at_every_position(&input, usize::MAX); + for (pos, matches) in all.iter().enumerate().skip(1).take(507) { + assert_eq!( + matches.as_slice(), + [((input.len() - pos) as u32, 1)], + "position {pos} did not stop at its nearest candidate", + ); + } + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn resolving_recovers_a_position_stored_past_four_gigabytes() { + // The one path no member that fits in this machine's memory reaches: an + // input long enough that positions lose their high half. Measuring back + // from the newest position recovers them while the two are within four + // gigabytes of each other, which the window guarantees. + const FOUR_GIB: usize = 1 << 32; + let newest = FOUR_GIB + 16; + assert_eq!(resolve(newest, 16), newest); + assert_eq!(resolve(newest, 1), FOUR_GIB + 1); + assert_eq!(resolve(newest, 0), FOUR_GIB); + // Below the boundary the stored half wrapped and `newest` did not, so + // the two are only comparable through the subtraction. + assert_eq!(resolve(newest, u32::MAX - 1), FOUR_GIB - 2); + } +} diff --git a/arca-rar/vendor/rars/src/codec/mod.rs b/arca-rar/vendor/rars/src/codec/mod.rs new file mode 100644 index 00000000..5b326371 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/mod.rs @@ -0,0 +1,145 @@ +//! RAR compression codecs, filters, PPMd, and RARVM components used by `rars`. + +mod canonical; +pub(crate) mod fast; +pub(crate) mod filters; +#[cfg(any(test, feature = "write"))] +mod huffman; +#[cfg(any(test, feature = "write"))] +mod match_finder; +mod ppmd; +pub mod rar13; +pub mod rar20; +pub mod rar29; +pub mod rar50; +pub mod rarvm; +pub(crate) mod workspace; + +pub type Result = std::result::Result; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + InvalidData(&'static str), + NeedMoreInput, + Cancelled, + WorkspaceLimitExceeded(Box), + /// An error carried by a reader or writer, including typed library errors + /// transported through `io::Error`. Successful decoding allocates no + /// diagnostic storage. + Io(Box), +} + +/// A refused codec allocation. Boxed by `Error` so successful codec operations +/// keep the existing compact Result layout. Diagnostic storage is not workspace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkspaceLimitError { + pub limit: u64, + /// Capacity required by this owner, including a simultaneous replacement. + pub required: u64, + pub used: u64, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidData(msg) => write!(f, "{msg}"), + Self::NeedMoreInput => write!(f, "codec input is truncated"), + Self::Cancelled => f.write_str("codec operation was cancelled"), + Self::WorkspaceLimitExceeded(details) => { + write!(f, + "codec workspace limit {} exceeded: owner requires {} bytes with {} bytes in use", + details.limit, details.required, details.used) + } + Self::Io(error) => write!(f, "{error}"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io(error) => Some(error.as_ref()), + _ => None, + } + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + match crate::Error::from(error) { + crate::Error::Cancelled => Self::Cancelled, + crate::Error::Codec(error) => error, + error => Self::Io(Box::new(error)), + } + } +} + +impl Error { + fn from_read_error(error: std::io::Error) -> Self { + if error.kind() == std::io::ErrorKind::UnexpectedEof { + Self::NeedMoreInput + } else { + Self::from(error) + } + } +} + +impl From for Error { + fn from(never: std::convert::Infallible) -> Self { + match never {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error as _; + + #[test] + fn codec_io_errors_expose_the_library_cause_and_leaf_errors_have_no_source() { + let cause = crate::Error::InvalidHeader("bad block").at_archive_offset(42); + let error = Error::Io(Box::new(cause.clone())); + assert_eq!( + error.to_string(), + "at archive offset 0x2a: invalid header: bad block" + ); + let source = error.source().unwrap(); + assert_eq!(source.downcast_ref::(), Some(&cause)); + assert_eq!( + source.source().unwrap().to_string(), + "invalid header: bad block" + ); + for error in [ + Error::InvalidData("bad symbol"), + Error::NeedMoreInput, + Error::Cancelled, + Error::WorkspaceLimitExceeded(Box::new(WorkspaceLimitError { + limit: 1, + required: 2, + used: 1, + })), + ] { + assert!(error.source().is_none()); + } + } + + #[test] + fn codec_diagnostics_propagate_formatter_failure() { + struct RefuseFormatting; + impl std::fmt::Write for RefuseFormatting { + fn write_str(&mut self, _: &str) -> std::fmt::Result { + Err(std::fmt::Error) + } + } + let error = Error::WorkspaceLimitExceeded(Box::new(WorkspaceLimitError { + limit: 1, + required: 2, + used: 1, + })); + assert_eq!( + std::fmt::write(&mut RefuseFormatting, format_args!("{error}")), + Err(std::fmt::Error), + ); + } +} diff --git a/arca-rar/vendor/rars/src/codec/ppmd.rs b/arca-rar/vendor/rars/src/codec/ppmd.rs new file mode 100644 index 00000000..009a5d73 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/ppmd.rs @@ -0,0 +1,1971 @@ +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; + +const MAX_FREQ: u32 = 124; +const MIN_MODEL_CONTEXTS: usize = 1; + +// PPMd-H suballocator parameters, per the format-level spec at +// rar-research/doc/PPMD_ALGORITHM_SPECIFICATION.md §4. +const ALLOC_UNIT_BYTES: usize = 12; +const N_BUCKETS: usize = 38; +const MAX_BUCKET_UNITS: usize = 128; +// Spec §4.3: countdown reset value after a successful glue pass. +const GLUE_RESET: u32 = 255; +// Sentinel offset for "no allocation" (binary contexts carry their state +// inline; their array_offset field is unused but typed `u32`). +const NULL_OFFSET: u32 = u32::MAX; + +// Lookup tables built once from the spec's bucket-construction algorithm +// (§4.2): +// for i in 0..38: step = if i >= 12 { 4 } else { i / 4 + 1 } +// repeat step times: units_to_index[k++] = i +// index_to_units[i] = k +struct AllocTables { + units_to_index: [u8; MAX_BUCKET_UNITS], + index_to_units: [u8; N_BUCKETS], +} + +fn build_alloc_tables() -> AllocTables { + let mut units_to_index = [0u8; MAX_BUCKET_UNITS]; + let mut index_to_units = [0u8; N_BUCKETS]; + let mut k: usize = 0; + for (i, index_to_unit) in index_to_units.iter_mut().enumerate() { + let step = if i >= 12 { 4 } else { i / 4 + 1 }; + for _ in 0..step { + // The bucket construction has exactly 128 entries: the first + // 12 buckets contribute 24 and the remaining 26 contribute 104. + units_to_index[k] = i as u8; + k += 1; + } + *index_to_unit = k.min(u8::MAX as usize) as u8; + } + AllocTables { + units_to_index, + index_to_units, + } +} + +fn alloc_tables() -> &'static AllocTables { + use std::sync::OnceLock; + static TABLES: OnceLock = OnceLock::new(); + TABLES.get_or_init(build_alloc_tables) +} + +// Spec §4.1 layout: contexts grow downward from `hi_unit`, state arrays +// grow upward from `lo_unit`. Tracking the side matters for adjacency +// patterns (gluing) and for which bump pointer advances. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AllocSide { + Lo, // state arrays + Hi, // context headers +} + +#[derive(Debug)] +struct Suballocator { + read_control: crate::read_control::ReadControl, + // Original pool size in bytes (C's p->Size). Preserved across restarts so + // the text/units split — which depends on the exact byte size, not the + // unit-truncated one — stays identical to the reference each restart. + size_bytes: usize, + // Total unit budget (= dict_mb * 1 MiB / 12, capped at u32::MAX). + pool_units: u32, + // Byte slack at the pool top (= Size % UNIT_SIZE). The C reference sets + // HiUnit = Base + Size with Size not necessarily a unit multiple, so all + // its pointers sit at `unit*12 + rem`. We keep unit offsets but fold this + // remainder into byte-domain comparisons (text capacity / fallback bump) + // so restart timing matches the reference exactly. + rem: u32, + // Text region capacity in bytes. text_ptr reaching this triggers a + // restart. Mirrors C's UnitsStart byte pointer (= Size - 84*text_units). + text_capacity_bytes: usize, + // Bump pointers (in units). lo_bump grows up, hi_bump grows down; + // valid bumpable space is [lo_bump, hi_bump). Both start at units_start. + units_start: u32, + lo_bump: u32, + hi_bump: u32, + // Free lists hold released block offsets (in units), one list per bucket. + free_lists: [Buffer; N_BUCKETS], + // Countdown for the glue pass (spec §4.3). Starts at 0, gets refreshed + // to GLUE_RESET after each successful glue. + glue_count: u32, + // Test escape hatch: when reset hasn't been called (pool_units == 0), + // act as unbounded so unit tests bypassing decode_init still work. + unbounded: bool, +} + +impl Suballocator { + fn try_clone(&self) -> Result { + let allowance = self.free_lists[0].allowance(); + let mut free_lists = std::array::from_fn(|_| Buffer::new(&allowance)); + for (copy, original) in free_lists.iter_mut().zip(&self.free_lists) { + *copy = Buffer::copied(original, &allowance)?; + } + Ok(Self { + read_control: self.read_control.clone(), + size_bytes: self.size_bytes, + pool_units: self.pool_units, + rem: self.rem, + text_capacity_bytes: self.text_capacity_bytes, + units_start: self.units_start, + lo_bump: self.lo_bump, + hi_bump: self.hi_bump, + free_lists, + glue_count: self.glue_count, + unbounded: self.unbounded, + }) + } + fn with_allowance(allowance: &B) -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + size_bytes: 0, + pool_units: 0, + rem: 0, + text_capacity_bytes: 0, + units_start: 0, + lo_bump: 0, + hi_bump: 0, + free_lists: std::array::from_fn(|_| Buffer::new(allowance)), + glue_count: 0, + unbounded: true, + } + } +} + +impl Suballocator { + fn reset(&mut self, pool_bytes: usize) { + let pool_units_usize = pool_bytes / ALLOC_UNIT_BYTES; + // Cap at u32::MAX-1 (NULL_OFFSET is u32::MAX). Even 256 MiB / 12 = + // ~22M fits comfortably, so this is theoretical. + let pool_units = pool_units_usize.min(u32::MAX as usize - 1) as u32; + // Ppmd7_RestartModel byte layout: + // text_units = Size / 8 / UNIT_SIZE + // UnitsStart = HiUnit - 7 * text_units * UNIT_SIZE + // i.e. the units region is exactly 7*text_units units carved off the + // top; the leftover slack (Size % 96, plus Size % 12) all stays in the + // text region. The previous code took `pool_units / 8` for text and + // gave the slack to the units region — 5+ extra bump units versus the + // reference, which desynchronised restart timing on large inputs. + let text_units = (pool_bytes / 8 / ALLOC_UNIT_BYTES) as u32; + let units_region = text_units.saturating_mul(7).min(pool_units); + self.size_bytes = pool_bytes; + self.pool_units = pool_units; + self.rem = (pool_bytes % ALLOC_UNIT_BYTES) as u32; + self.units_start = pool_units - units_region; + self.lo_bump = self.units_start; + self.hi_bump = pool_units; + // UnitsStart byte pointer = units_start*12 + rem (= Size - 84*text_units). + self.text_capacity_bytes = self.units_start as usize * ALLOC_UNIT_BYTES + self.rem as usize; + for fl in &mut self.free_lists { + fl.clear(); + } + self.glue_count = 0; + self.unbounded = pool_bytes == 0; + } + + fn bucket_for(units: usize) -> Option { + if units == 0 || units > MAX_BUCKET_UNITS { + return None; + } + Some(alloc_tables().units_to_index[units - 1] as usize) + } + + fn bucket_units(bucket: usize) -> usize { + alloc_tables().index_to_units[bucket] as usize + } + + fn try_alloc( + &mut self, + units: usize, + side: AllocSide, + text_len_bytes: usize, + ) -> Result> { + let Some(bucket) = Self::bucket_for(units) else { + return Ok(None); + }; + // Reference ordering differs by side. Lo-side state arrays go through + // `Ppmd7_AllocUnits` (free_list first, then lo-bump, then rare). Hi-side + // 1-unit context headers go through the inlined sequence in + // `Ppmd7_CreateSuccessors` (hi-bump first, then free_list[0], then + // `Ppmd7_AllocUnitsRare(0)`). The two are not interchangeable: every + // time a free 1-unit block exists alongside hi-side headroom, the two + // orderings consume different memory cells. Adjacency at the next glue + // diverges, and the drift compounds across millions of decode steps. + match side { + AllocSide::Hi => { + if let Some(offset) = self.try_bump(bucket, side) { + return Ok(Some(offset)); + } + if let Some(offset) = self.free_lists[bucket].pop() { + return Ok(Some(offset)); + } + } + AllocSide::Lo => { + if let Some(offset) = self.free_lists[bucket].pop() { + return Ok(Some(offset)); + } + if let Some(offset) = self.try_bump(bucket, side) { + return Ok(Some(offset)); + } + } + } + // Rare path (mirrors Ppmd7_AllocUnitsRare): + // 1. If glue_count == 0, glue and retry the bucket's free list. + // 2. Walk upward through larger buckets; if found, split. + // 3. Otherwise shrink units_start (consume text-reservation). + if self.glue_count == 0 { + self.glue_inner()?; + self.glue_count = GLUE_RESET; + if let Some(offset) = self.free_lists[bucket].pop() { + return Ok(Some(offset)); + } + } + for i in (bucket + 1)..N_BUCKETS { + if let Some(offset) = self.free_lists[i].pop() { + let large_size = Self::bucket_units(i) as u32; + let need = Self::bucket_units(bucket) as u32; + let leftover = large_size - need; + self.emit_run(offset + need, leftover)?; + return Ok(Some(offset)); + } + } + self.glue_count = self.glue_count.saturating_sub(1); + Ok(self.fallback_bump(bucket, text_len_bytes)) + } + + // Spec / ref Ppmd7_AllocUnitsRare bump-fallback: when no free block at + // any bucket size satisfies the request, shrink `units_start` downward + // (into the text-reservation zone). Succeeds only if the new units + // floor would still sit above the current text-write position. + fn fallback_bump(&mut self, bucket: usize, text_len_bytes: usize) -> Option { + // An unbounded allocator always succeeds in try_bump, so only a + // bounded pool reaches this fallback. + let need_units = Self::bucket_units(bucket) as u32; + let need_bytes = need_units as u64 * ALLOC_UNIT_BYTES as u64; + // UnitsStart byte pointer includes the pool-top remainder, matching + // the C reference (all its pointers sit at unit*12 + rem). + let units_start_bytes = self.units_start as u64 * ALLOC_UNIT_BYTES as u64 + self.rem as u64; + // Match C: `(us - Text) > numBytes`. Strict greater-than (not ≥). + if units_start_bytes <= text_len_bytes as u64 + need_bytes { + return None; + } + let new_units_start = self.units_start - need_units; + self.units_start = new_units_start; + // Lower the text capacity so text_has_room reflects the new boundary. + self.text_capacity_bytes = new_units_start as usize * ALLOC_UNIT_BYTES + self.rem as usize; + Some(new_units_start) + } + + fn try_bump(&mut self, bucket: usize, side: AllocSide) -> Option { + let block = Self::bucket_units(bucket) as u32; + if self.unbounded { + // No real pool; hand out monotonic offsets so each alloc is + // distinguishable but never adjacent to another (so gluing is + // a no-op for the tests). + let off = self.lo_bump; + self.lo_bump = self.lo_bump.saturating_add(block); + return Some(off); + } + // hi_bump - lo_bump is the bumpable headroom in units. + if self.hi_bump.saturating_sub(self.lo_bump) < block { + return None; + } + match side { + AllocSide::Lo => { + let off = self.lo_bump; + self.lo_bump += block; + Some(off) + } + AllocSide::Hi => { + self.hi_bump -= block; + Some(self.hi_bump) + } + } + } + + fn try_free(&mut self, offset: u32, units: usize) -> Result<()> { + if offset == NULL_OFFSET { + return Ok(()); + } + if let Some(bucket) = Self::bucket_for(units) { + self.free_lists[bucket].try_push(offset)?; + } + Ok(()) + } + + // Ppmd7_SplitBlock: carve a block of `old_units` (bucket I2U value) down + // to `new_units` (bucket I2U value) in place. The first `new_units` + // stay at `base_offset`; the (old_units - new_units) residue is pushed + // onto the appropriate smaller bucket(s). The "kept" prefix is NOT + // pushed — the caller is still using it as live storage. + fn try_split_in_place( + &mut self, + base_offset: u32, + old_units: u32, + new_units: u32, + ) -> Result<()> { + let nu = old_units - new_units; + let residue_offset = base_offset + new_units; + // Both inputs are distinct bucket sizes; their difference is 1..128. + let mut i = Self::bucket_for(nu as usize).expect("split residue is a valid bucket size"); + if Self::bucket_units(i) as u32 != nu { + // Inexact fit: the C reference pushes the smaller piece first + // (at offset + bucket_units(i-1)) onto bucket index `nu - k - 1`, + // then the i-1-sized prefix onto bucket i-1. Order across + // buckets is irrelevant; per-bucket LIFO is unaffected. + let k = Self::bucket_units(i - 1) as u32; + let small_bucket = (nu - k - 1) as usize; + self.free_lists[small_bucket].try_push(residue_offset + k)?; + i -= 1; + } + self.free_lists[i].try_push(residue_offset)?; + Ok(()) + } + + // Faithful port of Ppmd7_GlueFreeBlocks (§4.4). The earlier + // sort-by-offset version diverged from the reference in three ways that + // all change the resulting free-list state — and therefore the timing of + // the next allocation failure / model restart: + // 1. Threading order: the reference walks free_list[0..38], each list + // head-first (most-recent first), prepending every node to a single + // glue list. We rebuild that exact order so the fill pass re-buckets + // runs in the same sequence (per-bucket LIFO must match). + // 2. Merge cap: a merged run may not reach 0x10000 units (NU is 16-bit). + // 3. Fill split: runs are emitted as repeated 128-unit blocks, then the + // <=128 remainder via Ppmd7_SplitBlock's exact-or-two-piece rule — + // not "largest bucket fitting" greedily. + fn glue_inner(&mut self) -> Result<()> { + let mut poller = self.read_control.poller(); + // Step 1: thread free blocks into the reference's list order. + // list[k] = (offset, nu). The reference's prepend walk ends with + // bucket 37 first, each bucket in its original Vec order. + let allowance = self.free_lists[0].allowance(); + let count = self.free_lists.iter().try_fold(0usize, |total, list| { + total + .checked_add(list.len()) + .ok_or(Error::InvalidData("PPMd free-list size overflows")) + })?; + let mut list = Buffer::with_capacity(count, &allowance)?; + // Reversing the bucket order and visiting each bucket oldest-first + // produces exactly the prior head-first/prepend walk, without moving + // all previous entries for each insertion. + for bucket in (0..N_BUCKETS).rev() { + let nu = Self::bucket_units(bucket) as u32; + for &off in &self.free_lists[bucket] { + poller.check_codec(0)?; + list.push_admitted((off, nu)); + } + self.free_lists[bucket].clear(); + } + if list.is_empty() { + return Ok(()); + } + + // Step 2: glue pass. Absorb the physically-next free block while the + // combined size stays < 0x10000. `nu_at` maps a block's start offset + // to its current size; absorbed blocks are set to 0. + let mut nu_at = GlueIndex::new(&list, &allowance, &self.read_control)?; + for &(off, _) in list.iter() { + poller.check_codec(0)?; + let mut cur = match nu_at.get(&off) { + Some(&n) if n != 0 => n, + _ => continue, + }; + loop { + poller.check_codec(0)?; + let next_off = off + cur; + match nu_at.get(&next_off) { + Some(&n2) => { + let new = cur + n2; + if new >= 0x10000 { + break; + } + cur = new; + nu_at.insert(next_off, 0); + } + None => break, + } + } + nu_at.insert(off, cur); + } + + // Step 3: fill pass in list order. + for &(off, _) in list.iter() { + poller.check_codec(0)?; + let nu = match nu_at.get(&off) { + Some(&n) if n != 0 => n, + _ => continue, + }; + self.emit_run(off, nu)?; + } + Ok(()) + } + + // Re-bucket a single merged run exactly as Ppmd7_GlueFreeBlocks' fill / + // Ppmd7_SplitBlock do: peel 128-unit (bucket 37) blocks, then split the + // <=128 remainder into its exact bucket, or two pieces when inexact. + fn emit_run(&mut self, mut offset: u32, mut remaining: u32) -> Result<()> { + const LAST_BUCKET: usize = N_BUCKETS - 1; // 37 == 128 units + while remaining > MAX_BUCKET_UNITS as u32 { + self.push_free(LAST_BUCKET, offset)?; + offset += MAX_BUCKET_UNITS as u32; + remaining -= MAX_BUCKET_UNITS as u32; + } + // Callers pass positive runs; the loop leaves 1..=128 units. + let mut i = Self::bucket_for(remaining as usize).expect("run remainder fits a bucket"); + if Self::bucket_units(i) as u32 != remaining { + let k = Self::bucket_units(i - 1) as u32; + let small_bucket = (remaining - k - 1) as usize; + self.push_free(small_bucket, offset + k)?; + i -= 1; + } + self.push_free(i, offset)?; + Ok(()) + } + + fn push_free(&mut self, bucket: usize, offset: u32) -> Result<()> { + self.free_lists[bucket].try_push(offset) + } + + fn text_has_room(&self, text_len: usize) -> bool { + // When pool isn't initialised (test fixtures that bypass decode_init), + // text_capacity is 0 — treat that as "unbounded" so we don't break + // out-of-band callers. + self.text_capacity_bytes == 0 || text_len < self.text_capacity_bytes + } +} +const BIN_SCALE: u32 = 1 << 14; +const INT_BITS: u32 = 7; +const PERIOD_BITS: u8 = 7; +const TOP: u32 = 1 << 24; +const BOT: u32 = 1 << 15; +const INIT_BIN_ESC: [u16; 8] = [ + 0x3cdd, 0x1f3f, 0x59bf, 0x48f3, 0x64a1, 0x5abc, 0x6632, 0x6051, +]; +const EXP_ESCAPE: [u8; 16] = [25, 14, 9, 7, 5, 5, 4, 4, 4, 3, 3, 3, 2, 2, 2, 2]; + +pub trait PpmdByteReader { + fn read_ppmd_byte(&mut self) -> Result; +} + +#[derive(Debug, Clone)] +#[cfg(any(test, feature = "write"))] +pub struct PpmdDecoder { + state: PpmdState, +} +impl Clone for PpmdState { + fn clone(&self) -> Self { + self.try_clone().expect("unlimited PPMd model copy") + } +} +#[cfg(any(test, feature = "write"))] +impl PpmdDecoder { + pub fn new() -> Self { + Self { + state: PpmdState::with_allowance(&Allowance::default()), + } + } +} +#[cfg(test)] +impl PpmdState { + fn new() -> Self { + Self::with_allowance(&Allowance::default()) + } +} +#[derive(Debug)] +pub(crate) struct PpmdState { + min_context: usize, + max_context: usize, + found_state: StateRef, + order_fall: usize, + init_esc: u32, + prev_success: u32, + max_order: usize, + hi_bits_flag: u32, + run_length: i32, + init_rl: i32, + ns2bs_indx: [u8; 256], + ns2indx: [u8; 256], + bin_summ: [[u16; 64]; 128], + see: [[See; 16]; 25], + dummy_see: See, + contexts: Buffer, B>, + text: Buffer, + range: RangeDecoder, + allocated: bool, + max_contexts: usize, + suballoc: Suballocator, +} + +#[derive(Debug, Clone)] +#[cfg(any(test, feature = "write"))] +pub struct PpmdEncoder { + model: PpmdDecoder, + range: RangeEncoder, + esc_char: u8, +} + +#[derive(Debug)] +struct Context { + states: Buffer, + summ_freq: u16, + suffix: Option, + // Simulated suballoc offsets so gluing (spec §4.3) can detect adjacency. + // header_offset: from Hi side (1 unit). array_offset: from Lo side, + // set to NULL_OFFSET for binary contexts (states.len() == 1) which + // carry their state inline. + header_offset: u32, + array_offset: u32, +} + +impl Context { + fn try_clone(&self) -> Result { + Ok(Self { + states: Buffer::copied(&self.states, &self.states.allowance())?, + summ_freq: self.summ_freq, + suffix: self.suffix, + header_offset: self.header_offset, + array_offset: self.array_offset, + }) + } +} + +#[derive(Debug, Clone, Copy)] +struct State { + symbol: u8, + freq: u8, + successor: Successor, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Successor { + None, + Raw(usize), + Context(usize), +} + +#[derive(Debug, Clone, Copy)] +struct StateRef { + context: usize, + index: usize, +} + +#[derive(Debug, Clone, Copy)] +struct See { + summ: u16, + shift: u8, + count: u8, +} + +#[derive(Debug, Clone)] +struct RangeDecoder { + range: u32, + code: u32, + low: u32, +} + +impl PpmdState { + pub(crate) fn set_read_control(&mut self, control: crate::read_control::ReadControl) { + self.suballoc.read_control = control; + } + pub(crate) fn with_allowance(allowance: &B) -> Self { + let mut ns2bs_indx = [0u8; 256]; + ns2bs_indx[0] = 0; + ns2bs_indx[1] = 2; + ns2bs_indx[2..11].fill(4); + ns2bs_indx[11..].fill(6); + + let mut ns2indx = [0u8; 256]; + ns2indx[0] = 0; + ns2indx[1] = 1; + ns2indx[2] = 2; + let mut m = 3u8; + let mut k = 1u8; + for item in ns2indx.iter_mut().skip(3) { + *item = m; + k -= 1; + if k == 0 { + m += 1; + k = m - 2; + } + } + + Self { + min_context: 0, + max_context: 0, + found_state: StateRef { + context: 0, + index: 0, + }, + order_fall: 0, + init_esc: 0, + prev_success: 0, + max_order: 0, + hi_bits_flag: 0, + run_length: 0, + init_rl: 0, + ns2bs_indx, + ns2indx, + bin_summ: [[0; 64]; 128], + see: [[See { + summ: 0, + shift: 0, + count: 0, + }; 16]; 25], + dummy_see: See { + summ: 0, + shift: PERIOD_BITS, + count: 64, + }, + contexts: Buffer::new(allowance), + text: Buffer::new(allowance), + range: RangeDecoder::new(), + allocated: false, + max_contexts: MIN_MODEL_CONTEXTS, + suballoc: Suballocator::with_allowance(allowance), + } + } + + // Units occupied by a context's state array. Single-state (binary) + // contexts store the state inline, so the array is 0 units. Otherwise + // ceil(n/2) units fit n states at 2-per-unit (states are 6 bytes each). + fn state_array_units(n: usize) -> usize { + if n < 2 { + 0 + } else { + n.div_ceil(2) + } + } + + pub(crate) fn try_clone(&self) -> Result { + Ok(Self { + min_context: self.min_context, + max_context: self.max_context, + found_state: self.found_state, + order_fall: self.order_fall, + init_esc: self.init_esc, + prev_success: self.prev_success, + max_order: self.max_order, + hi_bits_flag: self.hi_bits_flag, + run_length: self.run_length, + init_rl: self.init_rl, + ns2bs_indx: self.ns2bs_indx, + ns2indx: self.ns2indx, + bin_summ: self.bin_summ, + see: self.see, + dummy_see: self.dummy_see, + contexts: Buffer::try_collect( + self.contexts.iter().map(Context::try_clone), + &self.contexts.allowance(), + )?, + text: Buffer::copied(&self.text, &self.text.allowance())?, + range: self.range.clone(), + allocated: self.allocated, + max_contexts: self.max_contexts, + suballoc: self.suballoc.try_clone()?, + }) + } + pub fn decode_init( + &mut self, + first_byte: u8, + input: &mut impl PpmdByteReader, + esc_char: &mut u8, + ) -> Result<()> { + let reset = first_byte & 0x20 != 0; + let max_mb = if reset { + Some(input.read_ppmd_byte()?) + } else { + None + }; + if first_byte & 0x40 != 0 { + *esc_char = input.read_ppmd_byte()?; + } + self.range.init(input)?; + if reset { + let mut max_order = ((first_byte & 0x1f) as usize) + 1; + if max_order > 16 { + max_order = 16 + (max_order - 16) * 3; + } + if max_order == 1 { + return Err(Error::InvalidData("RAR PPMd order is invalid")); + } + let dictionary_mb = max_mb.unwrap_or(0) as usize + 1; + self.max_contexts = model_context_limit(dictionary_mb); + self.suballoc + .reset(dictionary_mb.saturating_mul(1024 * 1024)); + self.init_model(max_order)?; + self.allocated = true; + } else if !self.allocated { + return Err(Error::InvalidData("RAR PPMd block reuses missing model")); + } + Ok(()) + } + + pub fn decode_symbol(&mut self, input: &mut impl PpmdByteReader) -> Result> { + // Ordinary symbol/context work is bounded by the model order and the + // 256-symbol alphabet; the enclosing decoder polls between symbols. + // Free-list maintenance can traverse the whole model and polls itself. + self.decode_symbol_inner(input) + } + + fn decode_symbol_inner(&mut self, input: &mut impl PpmdByteReader) -> Result> { + let mut mask = [true; 256]; + let min = self.min_context; + if self.contexts[min].states.len() != 1 { + let summ_freq = self.contexts[min].summ_freq as u32; + if summ_freq > self.range.range { + return Err(Error::InvalidData("RAR PPMd range is invalid")); + } + let mut count = self.range.get_threshold(summ_freq)?; + let mut hi_cnt = 0u32; + let mut found = None; + for (index, state) in self.contexts[min].states.iter().enumerate() { + if count < state.freq as u32 { + found = Some((index, hi_cnt, state.freq as u32, state.symbol)); + break; + } + count -= state.freq as u32; + hi_cnt += state.freq as u32; + } + if let Some((index, start, size, symbol)) = found { + self.range.decode(start, size); + self.found_state = StateRef { + context: min, + index, + }; + if index == 0 { + self.update1_0()?; + } else { + self.prev_success = 0; + self.update1()?; + } + self.range.normalize(input)?; + return Ok(Some(symbol)); + } + if hi_cnt >= summ_freq { + return Err(Error::InvalidData("RAR PPMd frequency sum is invalid")); + } + self.prev_success = 0; + self.range.decode(hi_cnt, summ_freq - hi_cnt); + self.hi_bits_flag = hi_bits_flag(self.state(self.found_state)?.symbol, 3); + for state in &self.contexts[min].states { + mask[state.symbol as usize] = false; + } + } else { + let state = self.contexts[min].states[0]; + let idx = self.bin_summ_index(state)?; + let prob = self.bin_summ[idx.0][idx.1] as u32; + let size0 = (self.range.range >> 14).wrapping_mul(prob); + let next_prob = update_prob_1(prob); + if self.range.code.wrapping_sub(self.range.low) < size0 { + self.bin_summ[idx.0][idx.1] = (next_prob + (1 << INT_BITS)) as u16; + self.range.decode_bit0(size0); + self.found_state = StateRef { + context: min, + index: 0, + }; + self.update_bin()?; + self.range.normalize(input)?; + return Ok(Some(state.symbol)); + } + self.bin_summ[idx.0][idx.1] = next_prob as u16; + self.init_esc = EXP_ESCAPE[(next_prob >> 10) as usize] as u32; + self.range.decode_bit1(size0); + mask[state.symbol as usize] = false; + self.prev_success = 0; + } + + loop { + self.range.normalize(input)?; + let mut mc = self.min_context; + let num_masked = self.contexts[mc].states.len(); + loop { + self.order_fall += 1; + let Some(suffix) = self.contexts[mc].suffix else { + return Ok(None); + }; + mc = suffix; + if self.contexts[mc].states.len() != num_masked { + break; + } + } + self.min_context = mc; + + let hi_cnt = self.contexts[mc] + .states + .iter() + .filter(|state| mask[state.symbol as usize]) + .map(|state| state.freq as u32) + .sum::(); + let (see_ref, esc_freq) = self.make_esc_freq(num_masked)?; + let freq_sum = hi_cnt + esc_freq; + if freq_sum > self.range.range { + return Err(Error::InvalidData("RAR PPMd escape range is invalid")); + } + let mut count = self.range.get_threshold(freq_sum)?; + if count < hi_cnt { + let mut start = 0u32; + let mut selected = None; + for (index, state) in self.contexts[mc].states.iter().enumerate() { + if !mask[state.symbol as usize] { + continue; + } + let freq = state.freq as u32; + if count < freq { + selected = Some((index, start, freq, state.symbol)); + break; + } + count -= freq; + start += freq; + } + // hi_cnt was summed from these same unmodified, unmasked + // frequencies, so count < hi_cnt must select a state. + let (index, start, freq, symbol) = + selected.expect("masked threshold selects a state"); + self.range.decode(start, freq); + self.update_see(see_ref); + self.found_state = StateRef { context: mc, index }; + self.update2()?; + self.range.normalize(input)?; + return Ok(Some(symbol)); + } + if count >= freq_sum { + return Err(Error::InvalidData("RAR PPMd escape symbol is invalid")); + } + self.range.decode(hi_cnt, freq_sum - hi_cnt); + self.add_see_summ(see_ref, freq_sum); + for state in &self.contexts[mc].states { + mask[state.symbol as usize] = false; + } + } + } + + #[cfg(any(test, feature = "write"))] + fn encode_symbol(&mut self, symbol: u8, output: &mut RangeEncoder) -> Result<()> { + let mut mask = [true; 256]; + let min = self.min_context; + if self.contexts[min].states.len() != 1 { + let summ_freq = self.contexts[min].summ_freq as u32; + let mut start = 0u32; + let mut found = None; + for (index, state) in self.contexts[min].states.iter().enumerate() { + if state.symbol == symbol { + found = Some((index, start, state.freq as u32)); + break; + } + start += state.freq as u32; + } + if let Some((index, start, size)) = found { + output.encode(start, size, summ_freq); + self.found_state = StateRef { + context: min, + index, + }; + if index == 0 { + self.update1_0()?; + } else { + self.prev_success = 0; + self.update1()?; + } + output.normalize(); + return Ok(()); + } + if start >= summ_freq { + return Err(Error::InvalidData("RAR PPMd frequency sum is invalid")); + } + self.prev_success = 0; + output.encode(start, summ_freq - start, summ_freq); + self.hi_bits_flag = hi_bits_flag(self.state(self.found_state)?.symbol, 3); + for state in &self.contexts[min].states { + mask[state.symbol as usize] = false; + } + } else { + let state = self.contexts[min].states[0]; + let idx = self.bin_summ_index(state)?; + let prob = self.bin_summ[idx.0][idx.1] as u32; + let size0 = (output.range >> 14).wrapping_mul(prob); + let next_prob = update_prob_1(prob); + if state.symbol == symbol { + self.bin_summ[idx.0][idx.1] = (next_prob + (1 << INT_BITS)) as u16; + output.encode_bit0(size0); + self.found_state = StateRef { + context: min, + index: 0, + }; + self.update_bin()?; + output.normalize(); + return Ok(()); + } + self.bin_summ[idx.0][idx.1] = next_prob as u16; + self.init_esc = EXP_ESCAPE[(next_prob >> 10) as usize] as u32; + output.encode_bit1(size0); + mask[state.symbol as usize] = false; + self.prev_success = 0; + } + + loop { + output.normalize(); + let mut mc = self.min_context; + let num_masked = self.contexts[mc].states.len(); + loop { + self.order_fall += 1; + let Some(suffix) = self.contexts[mc].suffix else { + return Err(Error::InvalidData("RAR PPMd symbol is not encodable")); + }; + mc = suffix; + if self.contexts[mc].states.len() != num_masked { + break; + } + } + self.min_context = mc; + + let hi_cnt = self.contexts[mc] + .states + .iter() + .filter(|state| mask[state.symbol as usize]) + .map(|state| state.freq as u32) + .sum::(); + let (see_ref, esc_freq) = self.make_esc_freq(num_masked)?; + let freq_sum = hi_cnt + esc_freq; + let mut start = 0u32; + let mut found = None; + for (index, state) in self.contexts[mc].states.iter().enumerate() { + if !mask[state.symbol as usize] { + continue; + } + let freq = state.freq as u32; + if state.symbol == symbol { + found = Some((index, start, freq)); + break; + } + start += freq; + } + if let Some((index, start, freq)) = found { + output.encode(start, freq, freq_sum); + self.update_see(see_ref); + self.found_state = StateRef { context: mc, index }; + self.update2()?; + output.normalize(); + return Ok(()); + } + output.encode(hi_cnt, freq_sum - hi_cnt, freq_sum); + self.add_see_summ(see_ref, freq_sum); + for state in &self.contexts[mc].states { + mask[state.symbol as usize] = false; + } + } + } + + fn init_model(&mut self, max_order: usize) -> Result<()> { + self.suballoc.read_control.check_codec()?; + self.contexts.clear(); + self.text.clear(); + // Spec §5.2 RestartModel: clear free lists, reserve root context + // (1 unit from Hi) + its 256-state array (128 units from Lo). + // Reuse the exact original byte size (C's p->Size) so the text/units + // split and the pool-top remainder survive the restart unchanged. + let unbounded_before = self.suballoc.unbounded; + let reset_arg = if unbounded_before { + 0 + } else { + self.suballoc.size_bytes + }; + self.suballoc.reset(reset_arg); + // At init time text is empty. + let header_offset = self + .suballoc + .try_alloc(1, AllocSide::Hi, 0)? + .unwrap_or(NULL_OFFSET); + let array_offset = self + .suballoc + .try_alloc(128, AllocSide::Lo, 0)? + .unwrap_or(NULL_OFFSET); + self.max_order = max_order; + self.order_fall = max_order; + self.init_rl = -(max_order.min(12) as i32) - 1; + self.run_length = self.init_rl; + self.prev_success = 0; + + let states = Buffer::collect( + (0..=255).map(|symbol| State { + symbol, + freq: 1, + successor: Successor::None, + }), + &self.contexts.allowance(), + )?; + self.contexts.try_push(Context { + states, + summ_freq: 257, + suffix: None, + header_offset, + array_offset, + })?; + self.min_context = 0; + self.max_context = 0; + self.found_state = StateRef { + context: 0, + index: 0, + }; + + for i in 0..128 { + for (k, &init_bin_esc) in INIT_BIN_ESC.iter().enumerate() { + let value = BIN_SCALE - u32::from(init_bin_esc) / (i as u32 + 2); + for m in (0..64).step_by(8) { + self.bin_summ[i][k + m] = value as u16; + } + } + } + for i in 0..25 { + let summ = ((5 * i + 10) << (PERIOD_BITS - 4)) as u16; + for k in 0..16 { + self.see[i][k] = See { + summ, + shift: PERIOD_BITS - 4, + count: 4, + }; + } + } + self.dummy_see = See { + summ: 0, + shift: PERIOD_BITS, + count: 64, + }; + Ok(()) + } + + fn bin_summ_index(&mut self, state: State) -> Result<(usize, usize)> { + let suffix = self.contexts[self.min_context] + .suffix + .ok_or(Error::InvalidData("RAR PPMd binary context has no suffix"))?; + let suffix_stats = self.contexts[suffix].states.len(); + self.hi_bits_flag = hi_bits_flag(self.state(self.found_state)?.symbol, 3); + let row = state.freq as usize - 1; + let col = self.prev_success as usize + + ((self.run_length >> 26) as usize & 0x20) + + self.ns2bs_indx[suffix_stats - 1] as usize + + hi_bits_flag(state.symbol, 4) as usize + + self.hi_bits_flag as usize; + Ok((row, col)) + } + + fn make_esc_freq(&mut self, num_masked: usize) -> Result<(SeeRef, u32)> { + let mc = self.min_context; + let num_stats = self.contexts[mc].states.len(); + if num_stats == 256 { + return Ok((SeeRef::Dummy, 1)); + } + if num_masked >= num_stats { + return Err(Error::InvalidData("RAR PPMd masked-state count is invalid")); + } + // The guard above establishes num_masked < num_stats. + let non_masked = num_stats - num_masked; + let suffix = self.contexts[mc].suffix.unwrap_or(mc); + let suffix_stats = self.contexts[suffix].states.len(); + // Spec §9.1: the subtraction is unsigned C wraparound — when + // `suffix.num_stats < min_context.num_stats`, the underflow yields a + // very large value and the `non_masked < suffix_delta` comparison + // evaluates to TRUE. Replicate the wraparound here. + let suffix_delta = suffix_stats.wrapping_sub(num_stats); + let col = (non_masked < suffix_delta) as usize + + 2 * ((self.contexts[mc].summ_freq as usize) < 11 * num_stats) as usize + + 4 * (num_masked > non_masked) as usize + + self.hi_bits_flag as usize; + let row = self.ns2indx[non_masked - 1] as usize; + let see = &mut self.see[row][col]; + let summ = see.summ; + let r = (summ >> see.shift) as u32; + see.summ = summ.wrapping_sub(r as u16); + Ok((SeeRef::Table(row, col), r + u32::from(r == 0))) + } + + fn update_see(&mut self, see_ref: SeeRef) { + let see = match see_ref { + SeeRef::Dummy => &mut self.dummy_see, + SeeRef::Table(row, col) => &mut self.see[row][col], + }; + if see.shift < PERIOD_BITS { + see.count = see.count.wrapping_sub(1); + if see.count == 0 { + see.summ = see.summ.wrapping_shl(1); + see.count = 3 << see.shift; + see.shift += 1; + } + } + } + + fn add_see_summ(&mut self, see_ref: SeeRef, value: u32) { + let see = match see_ref { + SeeRef::Dummy => &mut self.dummy_see, + SeeRef::Table(row, col) => &mut self.see[row][col], + }; + see.summ = see.summ.wrapping_add(value as u16); + } + + fn update1_0(&mut self) -> Result<()> { + let fs = self.found_state; + let freq = self.state(fs)?.freq as u32; + let summ_freq = self.contexts[fs.context].summ_freq as u32; + self.prev_success = u32::from(2 * freq > summ_freq); + self.run_length += self.prev_success as i32; + self.contexts[fs.context].summ_freq = self.contexts[fs.context].summ_freq.wrapping_add(4); + self.state_mut(fs)?.freq = (freq + 4) as u8; + if freq + 4 > MAX_FREQ { + self.rescale()?; + } + self.next_context() + } + + fn update1(&mut self) -> Result<()> { + let fs = self.found_state; + let freq = self.state(fs)?.freq as u32 + 4; + self.contexts[fs.context].summ_freq = self.contexts[fs.context].summ_freq.wrapping_add(4); + self.state_mut(fs)?.freq = freq as u8; + // Both decode and encode call update1 only for a non-first state. + if self.contexts[fs.context].states[fs.index].freq + > self.contexts[fs.context].states[fs.index - 1].freq + { + self.contexts[fs.context] + .states + .swap(fs.index, fs.index - 1); + self.found_state.index -= 1; + if freq > MAX_FREQ { + self.rescale()?; + } + } + self.next_context() + } + + fn update2(&mut self) -> Result<()> { + let fs = self.found_state; + let freq = self.state(fs)?.freq as u32 + 4; + self.run_length = self.init_rl; + self.contexts[fs.context].summ_freq = self.contexts[fs.context].summ_freq.wrapping_add(4); + self.state_mut(fs)?.freq = freq as u8; + if freq > MAX_FREQ { + self.rescale()?; + } + self.update_model() + } + + fn update_bin(&mut self) -> Result<()> { + let fs = self.found_state; + let freq = self.state(fs)?.freq; + self.state_mut(fs)?.freq = freq.wrapping_add(u8::from(freq < 128)); + self.prev_success = 1; + self.run_length += 1; + self.next_context() + } + + fn next_context(&mut self) -> Result<()> { + let successor = self.state(self.found_state)?.successor; + if let Successor::Context(context) = successor { + if self.order_fall == 0 { + self.max_context = context; + self.min_context = context; + return Ok(()); + } + } + self.update_model() + } + + fn update_model(&mut self) -> Result<()> { + let fs = self.state(self.found_state)?; + let found_symbol = fs.symbol; + if fs.freq < (MAX_FREQ / 4) as u8 && self.contexts[self.min_context].suffix.is_some() { + let suffix = self.contexts[self.min_context].suffix.unwrap(); + if self.contexts[suffix].states.len() == 1 { + let freq = self.contexts[suffix].states[0].freq; + if freq < 32 { + self.contexts[suffix].states[0].freq += 1; + } + } else if let Some(mut index) = self.contexts[suffix] + .states + .iter() + .position(|state| state.symbol == found_symbol) + { + if index > 0 + && self.contexts[suffix].states[index].freq + >= self.contexts[suffix].states[index - 1].freq + { + self.contexts[suffix].states.swap(index, index - 1); + index -= 1; + } + if self.contexts[suffix].states[index].freq < (MAX_FREQ - 9) as u8 { + self.contexts[suffix].states[index].freq += 2; + self.contexts[suffix].summ_freq = + self.contexts[suffix].summ_freq.wrapping_add(2); + } + } + } + + if self.order_fall == 0 { + let Some(context) = self.create_successors()? else { + self.init_model(self.max_order)?; + return Ok(()); + }; + self.max_context = context; + self.min_context = context; + self.state_mut(self.found_state)?.successor = Successor::Context(context); + return Ok(()); + } + + // Match Ppmd7_UpdateModel exactly: advance text first, then check + // `text >= UnitsStart`. The previous "check then push" version + // restarted one symbol later than the reference, advancing the range + // coder by one extra symbol before resetting — which left the + // post-restart state out of sync. + self.text.try_push(found_symbol)?; + let max_successor = Successor::Raw(self.text.len()); + if !self.suballoc.text_has_room(self.text.len()) { + self.init_model(self.max_order)?; + return Ok(()); + } + let (min_context, had_successor) = match fs.successor { + Successor::None => { + self.state_mut(self.found_state)?.successor = max_successor; + (self.min_context, false) + } + Successor::Context(context) => (context, true), + Successor::Raw(_) => { + let Some(context) = self.create_successors()? else { + self.init_model(self.max_order)?; + return Ok(()); + }; + (context, true) + } + }; + if had_successor { + self.order_fall -= 1; + if self.order_fall == 0 && self.max_context != self.min_context { + self.text.pop(); + } + } + let min_successor = Successor::Context(min_context); + + let mc = self.min_context; + let mut c = self.max_context; + self.min_context = min_context; + self.max_context = min_context; + if c == mc { + return Ok(()); + } + + let ns = self.contexts[mc].states.len() as u32; + let s0 = self.contexts[mc] + .summ_freq + .checked_sub(ns as u16) + .map(u32::from) + .and_then(|value| value.checked_sub(fs.freq as u32 - 1)) + .ok_or(Error::InvalidData("RAR PPMd model frequency is invalid"))?; + while c != mc { + let ns1 = self.contexts[c].states.len() as u32; + let mut sum; + if ns1 != 1 { + sum = self.contexts[c].summ_freq as u32; + sum += u32::from(2 * ns1 < ns) + 2 * u32::from(4 * ns1 <= ns && sum <= 8 * ns1); + } else { + let old = self.contexts[c].states[0]; + let freq = if old.freq < (MAX_FREQ / 4 - 1) as u8 { + old.freq * 2 + } else { + (MAX_FREQ - 4) as u8 + }; + self.contexts[c].states[0].freq = freq; + sum = freq as u32 + self.init_esc + u32::from(ns > 3); + } + + // sum is at most u16::MAX + 3 here, s0 at most u16::MAX, + // and fs.freq at most u8::MAX. Even the largest product is + // below 34 million, so u32 arithmetic cannot overflow. + let mut cf = (sum + 6) * 2 * fs.freq as u32; + let sf = s0 + sum; + if sf == 0 { + return Err(Error::InvalidData("RAR PPMd model frequency is invalid")); + } + if cf < 6 * sf { + cf = 1 + u32::from(cf > sf) + u32::from(cf >= 4 * sf); + sum += 3; + } else { + cf = 4 + + u32::from(cf >= 9 * sf) + + u32::from(cf >= 12 * sf) + + u32::from(cf >= 15 * sf); + sum += cf; + } + let old_n = self.contexts[c].states.len(); + if !self.grow_state_array(c, old_n + 1)? { + self.init_model(self.max_order)?; + return Ok(()); + } + self.contexts[c].states.try_push(State { + symbol: found_symbol, + freq: cf as u8, + successor: if self.order_fall == 0 { + min_successor + } else { + max_successor + }, + })?; + self.contexts[c].summ_freq = u16::try_from(sum) + .map_err(|_| Error::InvalidData("RAR PPMd model frequency overflows"))?; + c = self.contexts[c].suffix.unwrap_or(mc); + } + Ok(()) + } + + fn create_successors(&mut self) -> Result> { + // Missing links retain the reference model's restart path. Workspace + // refusal is propagated separately and must never restart the model. + macro_rules! pressure { + ($value:expr) => { + match $value { + Some(value) => value, + None => return Ok(None), + } + }; + } + let up_branch = match pressure!(self.state(self.found_state).ok()).successor { + Successor::Raw(pos) => pos, + Successor::Context(context) if self.order_fall == 0 => return Ok(Some(context)), + _ => return Ok(None), + }; + let mut c = self.min_context; + let allowance = self.contexts.allowance(); + let mut ps = Buffer::new(&allowance); + if self.order_fall != 0 { + ps.try_push(self.found_state)?; + } + while let Some(suffix) = self.contexts[c].suffix { + c = suffix; + let found_symbol = pressure!(self.state(self.found_state).ok()).symbol; + let index = pressure!(self.contexts[c] + .states + .iter() + .position(|state| state.symbol == found_symbol)); + let successor = self.contexts[c].states[index].successor; + if successor != Successor::Raw(up_branch) { + if let Successor::Context(context) = successor { + c = context; + if ps.is_empty() { + return Ok(Some(c)); + } + break; + } + return Ok(None); + } + ps.try_push(StateRef { context: c, index })?; + } + if ps.is_empty() { + return Ok(Some(c)); + } + let new_sym = *pressure!(self.text.get(up_branch)); + let up_successor = Successor::Raw(up_branch + 1); + let new_freq = if self.contexts[c].states.len() == 1 { + self.contexts[c].states[0].freq + } else { + let state = pressure!(self.contexts[c] + .states + .iter() + .find(|state| state.symbol == new_sym)); + let cf = state.freq as u32 - 1; + let s0 = self.contexts[c].summ_freq as u32 - self.contexts[c].states.len() as u32 - cf; + (1 + if 2 * cf <= s0 { + u32::from(5 * cf > s0) + } else { + (2 * cf + 3 * s0 - 1) / (2 * s0) + }) as u8 + }; + while let Some(state_ref) = ps.pop() { + let context = pressure!(self.push_context(Context { + states: Buffer::filled( + 1, + State { + symbol: new_sym, + freq: new_freq, + successor: up_successor + }, + &allowance + )?, + summ_freq: 0, + suffix: Some(c), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + })?); + pressure!(self.state_mut(state_ref).ok()).successor = Successor::Context(context); + c = context; + } + Ok(Some(c)) + } + + fn push_context(&mut self, mut context: Context) -> Result> { + if self.contexts.len() >= self.max_contexts { + return Ok(None); + } + // 1 unit for the context header (Hi side); binary contexts (1 state) + // carry their state inline, multi-state contexts also need a state + // array (Lo side). + let text_len = self.text.len(); + let Some(header_offset) = self.suballoc.try_alloc(1, AllocSide::Hi, text_len)? else { + return Ok(None); + }; + let array_units = Self::state_array_units(context.states.len()); + let array_offset = if array_units > 0 { + match self + .suballoc + .try_alloc(array_units, AllocSide::Lo, text_len)? + { + Some(off) => off, + None => { + self.suballoc.try_free(header_offset, 1)?; + return Ok(None); + } + } + } else { + NULL_OFFSET + }; + context.header_offset = header_offset; + context.array_offset = array_offset; + let index = self.contexts.len(); + self.contexts.try_push(context)?; + Ok(Some(index)) + } + + // Grow a context's state array from its current size → `new_n` states. + // Models ExpandUnits: if the same bucket still fits, no realloc happens; + // otherwise allocate the new size first, then release the old slot. The + // context's array_offset is updated to point at the new block when + // moved. + fn grow_state_array(&mut self, ctx_idx: usize, new_n: usize) -> Result { + let old_n = self.contexts[ctx_idx].states.len(); + let old_units = Self::state_array_units(old_n); + let new_units = Self::state_array_units(new_n); + // Called only for old_n + 1 during UpdateModel. The new array has + // positive size; unchanged bucket sizes are handled below. + if old_units > 0 { + let old_b = Suballocator::::bucket_for(old_units); + let new_b = Suballocator::::bucket_for(new_units); + if old_b == new_b { + return Ok(true); + } + } + let text_len = self.text.len(); + let new_offset = match self + .suballoc + .try_alloc(new_units, AllocSide::Lo, text_len)? + { + Some(off) => off, + None => return Ok(false), + }; + if old_units > 0 { + let old_offset = self.contexts[ctx_idx].array_offset; + self.suballoc.try_free(old_offset, old_units)?; + } + self.contexts[ctx_idx].array_offset = new_offset; + Ok(true) + } + + // Rescale shrinks a state array. Mirrors Ppmd7_Rescale's branch at + // Ppmd7.c:901-919: if the target bucket has a free block, swap to it + // (pop i1, push old to i0); otherwise SplitBlock the existing block in + // place — the first i1 units stay at the current address as the new + // array, the (i0 - i1)-unit residue is bucketed onto smaller free lists. + // No bump is consumed on the in-place path, which matters because over + // many rescales the reference never grows lo_bump from shrinks. + // + // Collapse-to-unary (new_n == 1, new_units == 0) is handled by Rescale's + // earlier branch (Ppmd7.c:881-898), which frees `stats` at bucket + // U2I(n0) and copies the surviving state into the inline OneState slot. + fn shrink_state_array(&mut self, ctx_idx: usize, old_n: usize, new_n: usize) -> Result<()> { + let old_units = Self::state_array_units(old_n); + let new_units = Self::state_array_units(new_n); + // Rescale starts with a multi-state context, so old_units is positive. + if new_units >= old_units { + return Ok(()); + } + if new_units == 0 { + // Collapse to unary: free the whole array, clear the pointer. + let old_offset = self.contexts[ctx_idx].array_offset; + self.suballoc.try_free(old_offset, old_units)?; + self.contexts[ctx_idx].array_offset = NULL_OFFSET; + return Ok(()); + } + // Contexts contain at most 256 states, so both nonzero array sizes + // are in the allocator's 1..=128-unit range. + let i0 = Suballocator::::bucket_for(old_units).expect("old array fits a bucket"); + let i1 = Suballocator::::bucket_for(new_units).expect("new array fits a bucket"); + if i0 == i1 { + return Ok(()); + } + let old_offset = self.contexts[ctx_idx].array_offset; + if let Some(swap_offset) = self.suballoc.free_lists[i1].pop() { + // Swap path: take a same-sized block from the target bucket, + // return the oversized block to its bucket. Data lives in + // self.contexts[ctx_idx].states (Vec), so no MEM_12_CPY needed. + self.suballoc.try_free(old_offset, old_units)?; + self.contexts[ctx_idx].array_offset = swap_offset; + } else { + // SplitBlock in place: keep the first I2U(i1) units at the same + // address, bucket the residue. Matches Ppmd7_SplitBlock. + let i0_units = Suballocator::::bucket_units(i0) as u32; + let i1_units = Suballocator::::bucket_units(i1) as u32; + self.suballoc + .try_split_in_place(old_offset, i0_units, i1_units)?; + // array_offset is unchanged. + } + Ok(()) + } + + fn rescale(&mut self) -> Result<()> { + let ctx = self.min_context; + let original_state_count = self.contexts[ctx].states.len(); + let mut states = Buffer::copied(&self.contexts[ctx].states, &self.contexts.allowance())?; + let found = self.found_state.index; + if found != 0 { + let state = states.remove(found); + states.insert(0, state)?; + self.found_state.index = 0; + } + let mut sum_freq = states[0].freq as u32; + let mut esc_freq = self.contexts[ctx].summ_freq as u32 - sum_freq; + let adder = u32::from(self.order_fall != 0); + sum_freq = (sum_freq + 4 + adder) >> 1; + states[0].freq = sum_freq as u8; + for index in 1..states.len() { + let freq = states[index].freq as u32; + esc_freq -= freq; + let freq = (freq + adder) >> 1; + sum_freq += freq; + states[index].freq = freq as u8; + let mut j = index; + while j > 0 && states[j].freq > states[j - 1].freq { + states.swap(j, j - 1); + j -= 1; + } + } + while states.last().is_some_and(|state| state.freq == 0) { + states.pop(); + esc_freq += 1; + } + if states.len() == 1 { + let mut freq = states[0].freq as u32; + while esc_freq > 1 { + esc_freq >>= 1; + freq = (freq + 1) >> 1; + } + states[0].freq = freq as u8; + self.shrink_state_array(ctx, original_state_count, 1)?; + self.contexts[ctx].states = states; + self.found_state.index = 0; + return Ok(()); + } + self.contexts[ctx].summ_freq = (sum_freq + esc_freq - (esc_freq >> 1)) as u16; + self.shrink_state_array(ctx, original_state_count, states.len())?; + self.contexts[ctx].states = states; + self.found_state.index = 0; + Ok(()) + } + + fn state(&self, state: StateRef) -> Result { + self.contexts + .get(state.context) + .and_then(|context| context.states.get(state.index)) + .copied() + .ok_or(Error::InvalidData("RAR PPMd state reference is invalid")) + } + + fn state_mut(&mut self, state: StateRef) -> Result<&mut State> { + self.contexts + .get_mut(state.context) + .and_then(|context| context.states.get_mut(state.index)) + .ok_or(Error::InvalidData("RAR PPMd state reference is invalid")) + } +} + +#[cfg(any(test, feature = "write"))] +impl PpmdEncoder { + pub fn new(max_order: usize, esc_char: u8, dictionary_mb: usize) -> Result { + if !(2..=64).contains(&max_order) { + return Err(Error::InvalidData("RAR PPMd order is invalid")); + } + if dictionary_mb == 0 { + return Err(Error::InvalidData("RAR PPMd dictionary size is invalid")); + } + let mut model = PpmdDecoder::new(); + model.state.max_contexts = model_context_limit(dictionary_mb); + model + .state + .suballoc + .reset(dictionary_mb.saturating_mul(1024 * 1024)); + model.state.init_model(max_order)?; + model.state.allocated = true; + Ok(Self { + model, + range: RangeEncoder::new(), + esc_char, + }) + } + + /// Carries a model from one block into the next. + /// + /// A block that clears the reset bit in its header leaves the reader's + /// model alone and only re-reads the range coder, so a solid chain can keep + /// one model across every member in it. Each block still gets a fresh range + /// coder, which is what lets a member's packed bytes start on a byte + /// boundary of their own. + #[cfg(feature = "write")] + pub fn continuing(model: PpmdDecoder, esc_char: u8) -> Self { + Self { + model, + range: RangeEncoder::new(), + esc_char, + } + } + + /// Ends the block and hands the model back for the next one to continue. + pub fn finish_keeping_model(mut self) -> Result<(Vec, PpmdDecoder)> { + self.model + .state + .encode_symbol(self.esc_char, &mut self.range)?; + self.model.state.encode_symbol(2, &mut self.range)?; + Ok((self.range.finish(), self.model)) + } + + /// Ends this PPMd block while keeping the member open for another block. + #[cfg(all(test, feature = "write"))] + pub(crate) fn finish_block_keeping_model(mut self) -> Result<(Vec, PpmdDecoder)> { + self.model + .state + .encode_symbol(self.esc_char, &mut self.range)?; + self.model.state.encode_symbol(0, &mut self.range)?; + Ok((self.range.finish(), self.model)) + } + + #[cfg(all(test, feature = "write"))] + pub(crate) fn finish_with_command(self, command: u8) -> Result> { + self.finish_with_command_prefix(command, &[]) + } + + #[cfg(all(test, feature = "write"))] + pub(crate) fn finish_with_command_prefix( + mut self, + command: u8, + parameters: &[u8], + ) -> Result> { + self.model + .state + .encode_symbol(self.esc_char, &mut self.range)?; + self.model.state.encode_symbol(command, &mut self.range)?; + for ¶meter in parameters { + self.model.state.encode_symbol(parameter, &mut self.range)?; + } + Ok(self.range.finish()) + } + + pub fn encode_literal(&mut self, symbol: u8) -> Result<()> { + self.model.state.encode_symbol(symbol, &mut self.range)?; + if symbol == self.esc_char { + self.model.state.encode_symbol(1, &mut self.range)?; + } + Ok(()) + } + + pub fn encode_repeat_offset_one(&mut self, length: usize) -> Result<()> { + if !(4..=259).contains(&length) { + return Err(Error::InvalidData( + "RAR PPMd offset-one repeat length is invalid", + )); + } + self.model + .state + .encode_symbol(self.esc_char, &mut self.range)?; + self.model.state.encode_symbol(5, &mut self.range)?; + self.model + .state + .encode_symbol((length - 4) as u8, &mut self.range)?; + Ok(()) + } + + pub fn encode_match(&mut self, offset: usize, length: usize) -> Result<()> { + if !(2..=0x1000001).contains(&offset) || !(32..=287).contains(&length) { + return Err(Error::InvalidData("RAR PPMd match is invalid")); + } + let encoded_offset = offset - 2; + self.model + .state + .encode_symbol(self.esc_char, &mut self.range)?; + self.model.state.encode_symbol(4, &mut self.range)?; + self.model + .state + .encode_symbol(((encoded_offset >> 16) & 0xff) as u8, &mut self.range)?; + self.model + .state + .encode_symbol(((encoded_offset >> 8) & 0xff) as u8, &mut self.range)?; + self.model + .state + .encode_symbol((encoded_offset & 0xff) as u8, &mut self.range)?; + self.model + .state + .encode_symbol((length - 32) as u8, &mut self.range)?; + Ok(()) + } + + #[cfg(feature = "write")] + pub fn encode_vm_filter_record(&mut self, record: &[u8]) -> Result<()> { + self.model + .state + .encode_symbol(self.esc_char, &mut self.range)?; + self.model.state.encode_symbol(3, &mut self.range)?; + for &byte in record { + self.model.state.encode_symbol(byte, &mut self.range)?; + } + Ok(()) + } + + /// Bits spent so far, to fractional precision. log2 of the range register + /// falls as symbols narrow the interval and rises by eight for every byte + /// normalisation flushes, so the difference between two readings is the + /// cost of whatever was coded between them. The RAR 2.9 tokeniser reads + /// this to price escape tokens against the literals they would replace. + #[cfg(feature = "write")] + pub(crate) fn spent_bits(&self) -> f64 { + 8.0 * self.range.out.len() as f64 - f64::from(self.range.range.max(1)).log2() + } +} + +#[derive(Debug, Clone, Copy)] +enum SeeRef { + Dummy, + Table(usize, usize), +} + +impl RangeDecoder { + fn new() -> Self { + Self { + range: 0xffff_ffff, + code: 0, + low: 0, + } + } + + fn init(&mut self, input: &mut impl PpmdByteReader) -> Result<()> { + self.code = 0; + self.range = 0xffff_ffff; + self.low = 0; + for _ in 0..4 { + self.code = (self.code << 8) | input.read_ppmd_byte()? as u32; + } + if self.code == 0xffff_ffff { + return Err(Error::InvalidData("RAR PPMd range code is invalid")); + } + Ok(()) + } + + fn get_threshold(&mut self, total: u32) -> Result { + if total == 0 { + return Err(Error::InvalidData("RAR PPMd frequency sum is zero")); + } + self.range /= total; + Ok(self.code.wrapping_sub(self.low) / self.range) + } + + fn decode(&mut self, start: u32, size: u32) { + let start = start.wrapping_mul(self.range); + self.low = self.low.wrapping_add(start); + self.range = self.range.wrapping_mul(size); + } + + fn decode_bit0(&mut self, size0: u32) { + self.range = size0; + } + + fn decode_bit1(&mut self, size0: u32) { + self.low = self.low.wrapping_add(size0); + self.range = (self.range & !(BIN_SCALE - 1)).wrapping_sub(size0); + } + + fn normalize(&mut self, input: &mut impl PpmdByteReader) -> Result<()> { + while normalize_range(self.low, &mut self.range) { + self.code = (self.code << 8) | input.read_ppmd_byte()? as u32; + self.range <<= 8; + self.low <<= 8; + } + Ok(()) + } +} + +#[derive(Debug, Clone)] +#[cfg(any(test, feature = "write"))] +struct RangeEncoder { + low: u32, + range: u32, + out: Vec, +} + +#[cfg(any(test, feature = "write"))] +impl RangeEncoder { + fn new() -> Self { + Self { + low: 0, + range: 0xffff_ffff, + out: Vec::new(), + } + } + + fn encode(&mut self, start: u32, size: u32, total: u32) { + self.range /= total; + self.low = self.low.wrapping_add(start.wrapping_mul(self.range)); + self.range = self.range.wrapping_mul(size); + } + + fn encode_bit0(&mut self, size0: u32) { + self.range = size0; + } + + fn encode_bit1(&mut self, size0: u32) { + self.low = self.low.wrapping_add(size0); + self.range = (self.range & !(BIN_SCALE - 1)).wrapping_sub(size0); + } + + fn normalize(&mut self) { + while normalize_range(self.low, &mut self.range) { + self.out.push((self.low >> 24) as u8); + self.range <<= 8; + self.low <<= 8; + } + } + + fn finish(mut self) -> Vec { + for _ in 0..4 { + self.out.push((self.low >> 24) as u8); + self.low <<= 8; + } + self.out + } +} + +fn normalize_range(low: u32, range: &mut u32) -> bool { + if (low ^ low.wrapping_add(*range)) < TOP { + return true; + } + if *range >= BOT { + return false; + } + *range = low.wrapping_neg() & (BOT - 1); + true +} + +fn update_prob_1(prob: u32) -> u32 { + prob - ((prob + (1 << 5)) >> INT_BITS) +} + +fn hi_bits_flag(symbol: u8, bits: u32) -> u32 { + ((symbol as u32 + 0xc0) >> (8 - bits)) & (1 << bits) +} + +#[cfg(test)] +impl Default for Suballocator { + fn default() -> Self { + Self::with_allowance(&Allowance::default()) + } +} +#[cfg(test)] +impl Suballocator { + fn alloc(&mut self, units: usize, side: AllocSide, text_len_bytes: usize) -> Option { + self.try_alloc(units, side, text_len_bytes) + .expect("unlimited arena allocation") + } + fn free(&mut self, offset: u32, units: usize) { + self.try_free(offset, units).unwrap(); + } + fn split_in_place(&mut self, base: u32, old: u32, new: u32) { + self.try_split_in_place(base, old, new).unwrap(); + } + fn glue(&mut self) { + self.glue_inner().expect("unlimited glue maintenance"); + } +} +/// Keep the glue walk's original order while admitting an exact allocation +/// for its offset lookup. The last original entry wins for duplicate offsets, +/// matching the previous map's insert semantics without opaque hash-table RAM. +struct GlueIndex { + entries: Buffer, B>, +} +impl GlueIndex { + fn new( + list: &[(u32, u32)], + allowance: &B, + control: &crate::read_control::ReadControl, + ) -> Result { + control.check_codec()?; + if list.is_empty() { + return Ok(Self { + entries: Buffer::new(allowance), + }); + } + let count = list + .len() + .checked_mul(2) + .and_then(usize::checked_next_power_of_two) + .ok_or(Error::InvalidData("PPMd glue lookup size overflows"))?; + let mut out = Self { + entries: Buffer::filled(count, None, allowance)?, + }; + let mut poller = control.poller(); + for &(offset, units) in list { + poller.check_codec(0)?; + out.insert(offset, units); + } + Ok(out) + } + fn hash(mut value: u32) -> usize { + value ^= value >> 16; + value = value.wrapping_mul(0x85eb_ca6b); + value ^= value >> 13; + value = value.wrapping_mul(0xc2b2_ae35); + (value ^ (value >> 16)) as usize + } + fn slot(&self, offset: u32) -> Option { + if self.entries.is_empty() { + return None; + } + let mask = self.entries.len() - 1; + let mut index = Self::hash(offset) & mask; + loop { + match self.entries[index] { + Some((key, _)) if key != offset => index = (index + 1) & mask, + _ => return Some(index), + } + } + } + fn get(&self, offset: &u32) -> Option<&u32> { + self.entries[self.slot(*offset)?] + .as_ref() + .map(|(_, units)| units) + } + fn insert(&mut self, offset: u32, units: u32) { + let index = self + .slot(offset) + .expect("glue inserts into a nonempty lookup"); + self.entries[index] = Some((offset, units)); + } +} + +// Upper bound on the number of live context records. The C reference +// (Ppmd7.c) has NO independent context cap — model restart is driven solely +// by suballocator exhaustion (`Ppmd7_AllocUnits`/`AllocUnitsRare` returning +// NULL). A context header is one 12-byte unit, and headers + state arrays +// share the pool, so the live context count can never exceed `pool_units` +// (= Size / UNIT_SIZE). Sizing the cap to that makes the suballocator the +// binding constraint, exactly as in the reference, while still bounding Vec +// growth defensively. A smaller cap (the old Size/16) forced a premature +// RestartModel that desynchronised the decoder from the reference. +fn model_context_limit(dictionary_mb: usize) -> usize { + dictionary_mb + .saturating_mul(1024 * 1024) + .checked_div(ALLOC_UNIT_BYTES) + .unwrap_or(usize::MAX) + .max(MIN_MODEL_CONTEXTS) +} + +#[cfg(test)] +mod tests; diff --git a/arca-rar/vendor/rars/src/codec/ppmd/tests.rs b/arca-rar/vendor/rars/src/codec/ppmd/tests.rs new file mode 100644 index 00000000..e527cea2 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/ppmd/tests.rs @@ -0,0 +1,1545 @@ +fn refuse_each_ppmd_allocation( + mut run: impl FnMut(&crate::codec::workspace::RefusingBudget) -> Result<()>, +) { + use crate::codec::workspace::RefusingBudget; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + let attempts = baseline.attempts(); + assert!(attempts > 0); + assert_eq!(baseline.used(), 0); + for index in 0..attempts { + let budget = RefusingBudget::new(index); + assert!( + matches!(run(&budget), Err(Error::Cancelled)), + "allocation {index}" + ); + assert_eq!(budget.used(), 0); + } +} + +#[test] +fn reader_ppmd_workspace_refusals_release_contexts_states_text_and_checkpoints() { + let data = b"abracadabra and evolving contexts abracadabra"; + let mut encoder = PpmdEncoder::new(4, 2, 1).unwrap(); + for &byte in data { + encoder.encode_literal(byte).unwrap(); + } + let (packed, _) = encoder.finish_keeping_model().unwrap(); + refuse_each_ppmd_allocation(|budget| { + let mut model = PpmdState::with_allowance(budget); + model.max_contexts = model_context_limit(1); + model.suballoc.reset(1024 * 1024); + model.init_model(4)?; + let mut input = Bytes { input: &packed }; + model.range.init(&mut input)?; + for &byte in data { + assert_eq!(model.decode_symbol(&mut input)?, Some(byte)); + } + let checkpoint = model.try_clone()?; + assert_eq!(checkpoint.contexts.len(), model.contexts.len()); + assert_eq!(checkpoint.text, model.text); + assert_eq!( + checkpoint.contexts[0].states[0].freq, + model.contexts[0].states[0].freq + ); + Ok(()) + }); +} + +#[test] +fn reader_ppmd_workspace_refusals_release_free_lists_and_glue_lookup() { + refuse_each_ppmd_allocation(|budget| { + let mut allocator = super::Suballocator::with_allowance(budget); + allocator.reset(32 * ALLOC_UNIT_BYTES); + for offset in [10, 11, 20, 21] { + allocator.try_free(offset, 1)?; + } + allocator.glue_inner()?; + assert_eq!(&*allocator.free_lists[1], &[10, 20]); + let checkpoint = allocator.try_clone()?; + assert_eq!(checkpoint.free_lists[1], allocator.free_lists[1]); + Ok(()) + }); +} + +#[test] +fn reader_ppmd_workspace_refusal_is_not_an_arena_restart() { + let ledger = Allowance::limited(1); + let mut model = PpmdState::with_allowance(&ledger); + let mut input = Bytes { input: &[0; 5] }; + let mut escape = 2; + assert!(matches!( + model.decode_init(0x20 | 3, &mut input, &mut escape), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert!(!model.allocated); + assert!(model.contexts.is_empty()); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn reader_ppmd_glue_index_probes_collisions_wraps_and_accepts_zero_offsets() { + let keys: Vec<_> = (0..1000u32) + .filter(|&key| GlueIndex::::hash(key) & 7 == 7) + .take(3) + .collect(); + let list: Vec<_> = keys + .iter() + .enumerate() + .map(|(index, &key)| (key, index as u32 + 1)) + .collect(); + let ledger = Allowance::limited(1024); + let control = crate::read_control::ReadControl::default(); + let mut index = GlueIndex::new(&list, &ledger, &control).unwrap(); + assert_eq!(index.entries.len(), 8); + assert!(index.entries[7].is_some() && index.entries[0].is_some() && index.entries[1].is_some()); + for &(key, units) in &list { + assert_eq!(index.get(&key), Some(&units)); + } + index.insert(keys[1], 0); + assert_eq!(index.get(&keys[1]), Some(&0)); + assert_eq!(index.get(&u32::MAX), None); + drop(index); + let zero = GlueIndex::new(&[(0, 1), (u32::MAX, 2)], &ledger, &control).unwrap(); + assert_eq!(zero.get(&0), Some(&1)); + assert_eq!(zero.get(&u32::MAX), Some(&2)); + drop(zero); + let empty = GlueIndex::new(&[], &ledger, &control).unwrap(); + assert_eq!(empty.get(&0), None); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn reader_ppmd_glue_index_matches_map_lookup_and_duplicate_overwrites() { + let list = [(10, 1), (4, 3), (10, 8), (20, 2), (4, 5)]; + let ledger = Allowance::limited(1024); + let mut index = + GlueIndex::new(&list, &ledger, &crate::read_control::ReadControl::default()).unwrap(); + let mut reference: std::collections::HashMap<_, _> = list.into_iter().collect(); + for key in [0, 4, 10, 20, 21, u32::MAX] { + assert_eq!(index.get(&key), reference.get(&key)); + } + for (key, value) in [(10, 0), (4, 7), (20, 0)] { + index.insert(key, value); + reference.insert(key, value); + assert_eq!(index.get(&key), reference.get(&key)); + } + assert_eq!( + ledger.used(), + index.entries.capacity() as u64 * std::mem::size_of::>() as u64 + ); + drop(index); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn cancellation_interrupts_suballocator_maintenance() { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(1); + let mut allocator = Suballocator { + read_control: control, + ..Suballocator::default() + }; + allocator.free_lists[0] = (0..5000).collect::>().into(); + assert_eq!(allocator.glue_inner(), Err(Error::Cancelled)); + assert!(token.is_cancelled()); +} +use super::*; +type PpmdDecoder = PpmdState; +type Suballocator = super::Suballocator; + +#[test] +fn alloc_tables_match_spec_pattern() { + let t = alloc_tables(); + // Spec §4.2: bucket sizes are 1,2,3,4 then groups of 4 at strides 1,2,3,4. + // First 12 buckets cover 1..4, then 6,8,10,12, then 15,18,21,24. + assert_eq!(t.index_to_units[0], 1); + assert_eq!(t.index_to_units[1], 2); + assert_eq!(t.index_to_units[2], 3); + assert_eq!(t.index_to_units[3], 4); + assert_eq!(t.index_to_units[4], 6); + assert_eq!(t.index_to_units[5], 8); + assert_eq!(t.index_to_units[6], 10); + assert_eq!(t.index_to_units[7], 12); + assert_eq!(t.index_to_units[8], 15); + assert_eq!(t.index_to_units[9], 18); + assert_eq!(t.index_to_units[10], 21); + assert_eq!(t.index_to_units[11], 24); + // From bucket 12 onward, stride is 4: 28, 32, 36, 40 ... up to 128. + assert_eq!(t.index_to_units[12], 28); + assert_eq!(t.index_to_units[N_BUCKETS - 1], 128); + + // 1-unit requests resolve to bucket 0; 2 → 1; 5,6 → 4 (the 6-unit bucket). + assert_eq!(t.units_to_index[0], 0); + assert_eq!(t.units_to_index[1], 1); + assert_eq!(t.units_to_index[4], 4); // 5 units + assert_eq!(t.units_to_index[5], 4); // 6 units (same bucket as 5) +} + +#[test] +fn suballoc_alloc_until_exhaustion() { + // Pool needs the text reservation (1/8) plus units area. 16 units + // total gives 2-unit text region + 14 units of headroom for bumping. + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + // 14 units between lo_bump and hi_bump → can hand out 7 × 2-unit blocks. + for _ in 0..7 { + assert!(s.alloc(2, AllocSide::Lo, 0).is_some()); + } + assert!(s.alloc(2, AllocSide::Lo, 0).is_none()); // pool full +} + +#[test] +fn suballoc_reuses_freed_bucket() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + let offsets: Vec = (0..7) + .map(|_| s.alloc(2, AllocSide::Lo, 0).expect("alloc")) + .collect(); + assert!(s.alloc(2, AllocSide::Lo, 0).is_none()); // full + s.free(offsets[0], 2); // freed → bucket 1 has 1 slot + assert!(s.alloc(2, AllocSide::Lo, 0).is_some()); // reuses freed slot + assert!(s.alloc(2, AllocSide::Lo, 0).is_none()); // truly full again +} + +#[test] +fn suballoc_uninitialised_pool_acts_unbounded() { + let mut s = Suballocator::default(); + // pool_bytes=0; the suballoc shouldn't reject any allocation since the + // model hasn't been told its budget yet. + for _ in 0..1000 { + assert!(s.alloc(2, AllocSide::Lo, 0).is_some()); + } + assert!(s.text_has_room(usize::MAX / 2)); +} + +#[test] +fn suballoc_rejects_invalid_sizes_and_ignores_null_free() { + let mut s = Suballocator::default(); + assert_eq!(s.alloc(0, AllocSide::Lo, 0), None); + assert_eq!(s.alloc(MAX_BUCKET_UNITS + 1, AllocSide::Lo, 0), None); + s.free(NULL_OFFSET, 1); + s.free(1, 0); + s.free(1, MAX_BUCKET_UNITS + 1); + assert!(s.free_lists.iter().all(|list| list.is_empty())); +} + +#[test] +fn hi_side_reuses_a_header_only_after_bump_space_is_exhausted() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + let old = s.alloc(1, AllocSide::Hi, 0).unwrap(); + s.free(old, 1); + assert_ne!(s.alloc(1, AllocSide::Hi, 0), Some(old)); + while s.hi_bump > s.lo_bump { + s.alloc(1, AllocSide::Hi, 0).unwrap(); + } + assert_eq!(s.alloc(1, AllocSide::Hi, 0), Some(old)); +} + +#[test] +fn hi_side_exhaustion_without_a_free_header_uses_rare_allocation() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + while s.hi_bump > s.lo_bump { + s.alloc(1, AllocSide::Hi, 0).unwrap(); + } + let before = s.units_start; + assert_eq!(s.alloc(1, AllocSide::Hi, 0), Some(before - 1)); +} + +#[test] +fn rare_allocation_splits_larger_free_block() { + let mut s = Suballocator::default(); + s.reset(32 * ALLOC_UNIT_BYTES); + // Exhaust the ordinary unit interval. Keep one six-unit block free + // and suppress glue so the rare path must split that exact block. + let large = s.alloc(6, AllocSide::Lo, 0).unwrap(); + while s.hi_bump > s.lo_bump { + s.alloc(1, AllocSide::Lo, 0).unwrap(); + } + s.free(large, 6); + s.glue_count = 1; + assert_eq!(s.alloc(1, AllocSide::Lo, 0), Some(large)); + // A six-unit block minus one leaves five, represented as 4 + 1. + assert!(s.free_lists[3].contains(&(large + 1))); + assert!(s.free_lists[0].contains(&(large + 5))); +} + +#[test] +fn rare_allocation_borrows_text_space() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + while s.hi_bump > s.lo_bump { + s.alloc(1, AllocSide::Lo, 0).unwrap(); + } + // With no free block, the allocator takes one unit from the text + // reservation while leaving room for the text pointer. + let before = s.units_start; + assert_eq!(s.alloc(1, AllocSide::Lo, 0), Some(before - 1)); + assert_eq!( + s.text_capacity_bytes, + (before - 1) as usize * ALLOC_UNIT_BYTES + ); +} + +#[test] +fn rare_allocation_stops_when_text_reservation_is_full() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + while s.hi_bump > s.lo_bump { + s.alloc(1, AllocSide::Lo, 0).unwrap(); + } + let text_end = s.text_capacity_bytes; + assert_eq!(s.alloc(1, AllocSide::Lo, text_end), None); +} + +#[test] +fn cancelled_glue_aborts_rare_allocation() { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(1); + let mut s = Suballocator { + read_control: control, + ..Suballocator::default() + }; + s.reset(16 * ALLOC_UNIT_BYTES); + while s.hi_bump > s.lo_bump { + s.alloc(1, AllocSide::Lo, 0).unwrap(); + } + s.free_lists[0] = (0..5000).collect::>().into(); + assert_eq!(s.try_alloc(2, AllocSide::Lo, 0), Err(Error::Cancelled)); + assert!(token.is_cancelled()); +} + +// Spec §4.3: free blocks that are address-adjacent should merge into a +// larger run during glue, and that merged run should be visible in the +// bucket whose size matches. +#[test] +fn glue_merges_adjacent_free_blocks() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + // Allocate 4 consecutive 1-unit blocks from the Lo side. They will + // be at offsets [units_start, units_start+1, units_start+2, + // units_start+3], i.e. adjacent. + let a = s.alloc(1, AllocSide::Lo, 0).unwrap(); + let b = s.alloc(1, AllocSide::Lo, 0).unwrap(); + let c = s.alloc(1, AllocSide::Lo, 0).unwrap(); + let d = s.alloc(1, AllocSide::Lo, 0).unwrap(); + assert_eq!(b, a + 1); + assert_eq!(c, a + 2); + assert_eq!(d, a + 3); + // Free them — bucket 0 now has 4 entries, bucket 3 (size 4) has none. + s.free(a, 1); + s.free(b, 1); + s.free(c, 1); + s.free(d, 1); + assert_eq!(s.free_lists[0].len(), 4); + assert!(s.free_lists[3].is_empty()); + s.glue(); + // After gluing, the 4-adjacent run becomes one 4-unit block in bucket 3. + assert!(s.free_lists[0].is_empty()); + assert_eq!(s.free_lists[3].len(), 1); + // And we can satisfy a 4-unit request from the merged block. + assert_eq!(s.alloc(4, AllocSide::Lo, 0), Some(a)); +} + +// Non-adjacent free blocks shouldn't merge even though their bucket sizes +// are compatible. +#[test] +fn glue_keeps_non_adjacent_blocks_separate() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + // Two 1-unit blocks separated by a still-live 2-unit allocation. + let a = s.alloc(1, AllocSide::Lo, 0).unwrap(); + let _live = s.alloc(2, AllocSide::Lo, 0).unwrap(); + let b = s.alloc(1, AllocSide::Lo, 0).unwrap(); + s.free(a, 1); + s.free(b, 1); + s.glue(); + // Still two separate 1-unit free entries; no merged 2-unit block. + assert_eq!(s.free_lists[0].len(), 2); +} + +// A merged run larger than the biggest bucket (128 units) should be +// chunked into multiple bucket-sized pieces during redistribution. +#[test] +fn glue_splits_oversized_run_into_buckets() { + // 256-unit pool (text=32, bumpable=224). Adjacent 64-unit + 64-unit + // blocks combine to 128, which is the biggest bucket — exactly one + // entry. Push it to 192 and we get a 128 + a 64 (closest bucket). + let mut s = Suballocator::default(); + s.reset(256 * ALLOC_UNIT_BYTES); + // Three adjacent 64-unit allocs → run of 192 units after freeing. + // 64 is bucket 18 in the spec table. + let bucket_64 = Suballocator::bucket_for(64).unwrap(); + assert_eq!(Suballocator::bucket_units(bucket_64), 64); + let a = s.alloc(64, AllocSide::Lo, 0).unwrap(); + let b = s.alloc(64, AllocSide::Lo, 0).unwrap(); + let c = s.alloc(64, AllocSide::Lo, 0).unwrap(); + assert_eq!(b, a + 64); + assert_eq!(c, a + 128); + s.free(a, 64); + s.free(b, 64); + s.free(c, 64); + s.glue(); + // Should emit one 128-unit block + one 64-unit block. + let bucket_128 = Suballocator::bucket_for(128).unwrap(); + assert_eq!(s.free_lists[bucket_128].len(), 1); + assert_eq!(s.free_lists[bucket_64].len(), 1); + // The 128 block starts at `a`, the 64 follows. + assert_eq!(s.free_lists[bucket_128][0], a); + assert_eq!(s.free_lists[bucket_64][0], a + 128); +} + +#[test] +fn glue_does_not_merge_a_run_to_65536_units() { + let mut s = Suballocator::default(); + // A 16-bit NU cannot hold 65536 units. Populate the same 512 adjacent + // 128-unit free blocks without allocating a huge backing pool. + let bucket = Suballocator::bucket_for(128).unwrap(); + s.free_lists[bucket] = (0..512).map(|i| i * 128).collect::>().into(); + s.glue(); + assert_eq!(s.free_lists[bucket].len(), 512); + assert_eq!(s.free_lists[bucket].iter().copied().min(), Some(0)); + assert_eq!(s.free_lists[bucket].iter().copied().max(), Some(511 * 128)); +} + +#[test] +fn glue_emits_exact_128_unit_chunks_and_inexact_remainder() { + let mut s = Suballocator::default(); + s.emit_run(10, 256).unwrap(); + let full = Suballocator::bucket_for(128).unwrap(); + assert_eq!(s.free_lists[full], vec![10, 138]); + + // Five units have no dedicated bucket: split into four plus one. + s.emit_run(300, 5).unwrap(); + assert_eq!(s.free_lists[3], vec![300]); + assert_eq!(s.free_lists[0], vec![304]); +} + +#[test] +fn in_place_split_buckets_an_inexact_five_unit_residue() { + let mut s = Suballocator::default(); + s.split_in_place(100, 6, 1); + assert_eq!(s.free_lists[3], vec![101]); + assert_eq!(s.free_lists[0], vec![105]); +} + +// glue_count debounce: a single bump failure runs the glue pass exactly +// once, then subsequent failures decrement instead of re-gluing. +#[test] +fn glue_count_debounces_subsequent_failures() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + // Fill the pool to capacity (14 bumpable units in a 16-unit pool; + // text region is 2 units). All 14 calls succeed — glue_count stays 0. + // Pass a text_len that occupies the entire 2-unit text region so + // the fallback_bump path can't grab any more. + let text_len_full = 2 * ALLOC_UNIT_BYTES; + for _ in 0..14 { + assert!(s.alloc(1, AllocSide::Lo, text_len_full).is_some()); + } + assert_eq!(s.glue_count, 0); + // 15th call fails: triggers glue (no-op, no free blocks) then + // walks up larger buckets (none), then fallback_bump fails (text + // already fills text region), and on the fallback path we + // decrement glue_count once. Mirrors C's `p->GlueCount--` inside + // the bump-fallback branch of AllocUnitsRare. + assert!(s.alloc(1, AllocSide::Lo, text_len_full).is_none()); + assert_eq!(s.glue_count, GLUE_RESET - 1); + // Subsequent failure decrements again (no re-glue because count > 0). + assert!(s.alloc(1, AllocSide::Lo, text_len_full).is_none()); + assert_eq!(s.glue_count, GLUE_RESET - 2); +} + +// After exhaustion, freeing two adjacent blocks lets a larger request +// succeed via glue. +#[test] +fn glue_recovers_capacity_for_larger_request() { + let mut s = Suballocator::default(); + s.reset(16 * ALLOC_UNIT_BYTES); + // Fill the pool to capacity (14 × 1-unit) without triggering a + // failure, so glue_count remains 0. + let mut held = Vec::new(); + for _ in 0..14 { + held.push(s.alloc(1, AllocSide::Lo, 0).unwrap()); + } + // Free two adjacent 1-unit blocks at the bottom of the units region. + s.free(held[0], 1); + s.free(held[1], 1); + // alloc(2) needs bucket 1 (size 2). Empty, bump exhausted, but + // glue_count == 0 so glue fires, merging into one 2-unit block at + // held[0]. Retry pop succeeds. + assert_eq!(s.alloc(2, AllocSide::Lo, 0), Some(held[0])); +} + +#[test] +fn state_array_units_handles_boundaries() { + assert_eq!(PpmdDecoder::state_array_units(0), 0); + assert_eq!(PpmdDecoder::state_array_units(1), 0); // binary: inline + assert_eq!(PpmdDecoder::state_array_units(2), 1); + assert_eq!(PpmdDecoder::state_array_units(3), 2); + assert_eq!(PpmdDecoder::state_array_units(4), 2); + assert_eq!(PpmdDecoder::state_array_units(255), 128); + assert_eq!(PpmdDecoder::state_array_units(256), 128); +} + +struct Bytes<'a> { + input: &'a [u8], +} + +impl PpmdByteReader for Bytes<'_> { + fn read_ppmd_byte(&mut self) -> Result { + let Some((&byte, rest)) = self.input.split_first() else { + return Err(Error::NeedMoreInput); + }; + self.input = rest; + Ok(byte) + } +} + +#[test] +fn decode_init_rejects_truncated_range_header_without_panic() { + let mut decoder = PpmdDecoder::new(); + let mut input = Bytes { input: &[0, 0] }; + let mut esc = 0; + + assert_eq!( + decoder.decode_init(0x20 | 1, &mut input, &mut esc), + Err(Error::NeedMoreInput) + ); +} + +#[test] +fn decode_init_rejects_reuse_before_model_allocation() { + let mut decoder = PpmdDecoder::new(); + let mut input = Bytes { + input: &[0, 0, 0, 0], + }; + let mut esc = 0; + + assert_eq!( + decoder.decode_init(0, &mut input, &mut esc), + Err(Error::InvalidData("RAR PPMd block reuses missing model")) + ); +} + +#[test] +fn decode_init_accepts_max_wire_order_without_growing_unbounded_model() { + let mut decoder = PpmdDecoder::new(); + let mut input = Bytes { + input: &[0, 0, 0, 0, 0], + }; + let mut esc = 0; + + decoder + .decode_init(0x20 | 0x1f, &mut input, &mut esc) + .unwrap(); + + assert_eq!(decoder.max_order, 64); + assert_eq!(decoder.contexts.len(), 1); + assert_eq!(decoder.contexts[0].states.len(), 256); + assert_eq!(decoder.max_contexts, model_context_limit(1)); +} + +#[test] +fn decode_init_rejects_wire_order_one() { + let mut decoder = PpmdDecoder::new(); + let mut input = Bytes { + input: &[0, 0, 0, 0, 0], + }; + let mut esc = 0; + assert_eq!( + decoder.decode_init(0x20, &mut input, &mut esc), + Err(Error::InvalidData("RAR PPMd order is invalid")) + ); +} + +#[test] +fn decode_init_reads_explicit_escape_character() { + let mut decoder = PpmdDecoder::new(); + let mut input = Bytes { + input: &[0, b'!', 0, 0, 0, 0], + }; + let mut esc = 2; + decoder + .decode_init(0x20 | 0x40 | 3, &mut input, &mut esc) + .unwrap(); + assert_eq!(esc, b'!'); +} + +#[test] +fn decoder_reports_invalid_range_and_frequency_sum() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + let mut input = Bytes { input: &[] }; + decoder.range.range = 1; + assert_eq!( + decoder.decode_symbol(&mut input), + Err(Error::InvalidData("RAR PPMd range is invalid")) + ); + + decoder.range.range = 257; + decoder.contexts[0].summ_freq = 1; + decoder.range.code = 256 * 257; + assert_eq!( + decoder.decode_symbol(&mut input), + Err(Error::InvalidData("RAR PPMd frequency sum is invalid")) + ); +} + +#[test] +fn decoder_rejects_invalid_escape_range_and_symbol() { + fn escaping_model(code_delta: u32) -> PpmdDecoder { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + let state = |symbol| State { + symbol, + freq: 1, + successor: Successor::None, + }; + decoder + .contexts + .push(Context { + states: vec![state(b'a'), state(b'b'), state(b'c')].into(), + summ_freq: 4, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap(); + decoder + .contexts + .push(Context { + states: vec![state(b'a'), state(b'b')].into(), + summ_freq: 3, + suffix: Some(1), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap(); + decoder.min_context = 2; + decoder.range.range = 100_000; + // The first escape leaves a 33_333-unit range straddling TOP. + // That is above BOT, so normalization consumes no input. + decoder.range.low = TOP - 20_000 - 2 * (100_000 / 3); + decoder.range.code = decoder.range.low + code_delta; + decoder + } + + let mut decoder = escaping_model(2 * (100_000 / 3)); + decoder.see[0][7] = See { + summ: u16::MAX, + shift: 0, + count: 1, + }; + assert_eq!( + decoder.decode_symbol(&mut Bytes { input: &[] }), + Err(Error::InvalidData("RAR PPMd escape range is invalid")) + ); + + let mut decoder = escaping_model(110_000); + assert_eq!( + decoder.decode_symbol(&mut Bytes { input: &[] }), + Err(Error::InvalidData("RAR PPMd escape symbol is invalid")) + ); +} + +#[test] +fn root_escape_ends_ppmd_stream() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + let total = decoder.contexts[0].summ_freq as u32; + decoder.range.code = 256 * (decoder.range.range / total); + let mut input = Bytes { input: &[0; 16] }; + assert_eq!(decoder.decode_symbol(&mut input), Ok(None)); +} + +#[test] +fn range_decoder_rejects_reserved_initial_code() { + let mut range = RangeDecoder::new(); + let mut input = Bytes { input: &[0xff; 4] }; + assert_eq!( + range.init(&mut input), + Err(Error::InvalidData("RAR PPMd range code is invalid")) + ); +} + +#[test] +fn range_coders_normalize_across_the_bottom_boundary() { + let mut decoder = RangeDecoder::new(); + decoder.low = TOP - 50; + decoder.range = 100; + decoder.normalize(&mut Bytes { input: &[0; 8] }).unwrap(); + assert_ne!(decoder.range, 100); + + let mut encoder = RangeEncoder::new(); + encoder.low = TOP - 50; + encoder.range = 100; + encoder.normalize(); + assert!(!encoder.out.is_empty()); + + // Straddling TOP alone does not normalize while enough range remains. + let mut decoder = RangeDecoder::new(); + decoder.low = TOP - 20_000; + decoder.range = 40_000; + decoder.normalize(&mut Bytes { input: &[] }).unwrap(); + assert_eq!(decoder.range, 40_000); + + let mut encoder = RangeEncoder::new(); + encoder.low = TOP - 20_000; + encoder.range = 40_000; + encoder.normalize(); + assert!(encoder.out.is_empty()); +} + +#[test] +fn see_counter_ages_and_dummy_accumulates_without_aging() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + decoder.see[0][0] = See { + summ: 7, + shift: PERIOD_BITS - 1, + count: 1, + }; + decoder.update_see(SeeRef::Table(0, 0)); + assert_eq!(decoder.see[0][0].summ, 14); + assert_eq!(decoder.see[0][0].shift, PERIOD_BITS); + assert_eq!(decoder.see[0][0].count, 3 << (PERIOD_BITS - 1)); + decoder.update_see(SeeRef::Table(0, 0)); + assert_eq!(decoder.see[0][0].summ, 14); + + let before = decoder.dummy_see; + decoder.update_see(SeeRef::Dummy); + assert_eq!(decoder.dummy_see.summ, before.summ); + assert_eq!(decoder.dummy_see.count, before.count); + decoder.add_see_summ(SeeRef::Dummy, 9); + assert_eq!(decoder.dummy_see.summ, before.summ + 9); +} + +#[test] +fn encoder_rejects_invalid_match_and_repeat_bounds() { + let mut encoder = PpmdEncoder::new(4, 2, 1).unwrap(); + for length in [3, 260] { + assert_eq!( + encoder.encode_repeat_offset_one(length), + Err(Error::InvalidData( + "RAR PPMd offset-one repeat length is invalid" + )) + ); + } + for (offset, length) in [(1, 32), (0x1000002, 32), (2, 31), (2, 288)] { + assert_eq!( + encoder.encode_match(offset, length), + Err(Error::InvalidData("RAR PPMd match is invalid")) + ); + } +} + +#[test] +fn encoder_rejects_corrupt_frequency_sum_and_missing_root_symbol() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + decoder.contexts[0].states.truncate(2); + decoder.contexts[0].summ_freq = 2; + assert_eq!( + decoder.encode_symbol(2, &mut RangeEncoder::new()), + Err(Error::InvalidData("RAR PPMd frequency sum is invalid")) + ); + + decoder.init_model(4).unwrap(); + decoder.contexts[0].states.truncate(2); + decoder.contexts[0].summ_freq = 3; + assert_eq!( + decoder.encode_symbol(2, &mut RangeEncoder::new()), + Err(Error::InvalidData("RAR PPMd symbol is not encodable")) + ); +} + +#[test] +fn encoder_rejects_orders_outside_model_bounds() { + assert!(matches!( + PpmdEncoder::new(1, 2, 1), + Err(Error::InvalidData("RAR PPMd order is invalid")) + )); + assert!(matches!( + PpmdEncoder::new(65, 2, 1), + Err(Error::InvalidData("RAR PPMd order is invalid")) + )); +} + +#[test] +fn encoder_rejects_zero_dictionary_size() { + assert!(matches!( + PpmdEncoder::new(4, 2, 0), + Err(Error::InvalidData("RAR PPMd dictionary size is invalid")) + )); +} + +#[test] +fn range_decoder_rejects_zero_total_without_panic() { + let mut decoder = RangeDecoder::new(); + let mut input = Bytes { + input: &[0, 0, 0, 0], + }; + decoder.init(&mut input).unwrap(); + + assert_eq!( + decoder.get_threshold(0), + Err(Error::InvalidData("RAR PPMd frequency sum is zero")) + ); +} + +#[test] +fn context_allocation_respects_dictionary_limit() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 1; + decoder.init_model(4).unwrap(); + + assert_eq!( + decoder + .push_context(Context { + states: Buffer::new(&Allowance::default()), + summ_freq: 0, + suffix: None, + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap(), + None + ); +} + +#[test] +fn context_header_pressure_is_distinct_from_the_context_count_limit() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = model_context_limit(1); + decoder.suballoc.reset(1024 * 1024); + decoder.init_model(4).unwrap(); + + // Occupy the pool through the allocator's normal state-array and + // header operations, including its rare text-reservation fallback. + // The root's 128-unit array alone makes memory bind before the + // defensive one-header-per-unit context-count cap. + while decoder.suballoc.alloc(128, AllocSide::Lo, 0).is_some() {} + while decoder.suballoc.alloc(1, AllocSide::Hi, 0).is_some() {} + assert!(decoder.contexts.len() < decoder.max_contexts); + let root_header = decoder.contexts[0].header_offset; + let root_array = decoder.contexts[0].array_offset; + let new_context = || Context { + states: vec![State { + symbol: b'a', + freq: 1, + successor: Successor::None, + }] + .into(), + summ_freq: 0, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }; + assert_eq!(decoder.push_context(new_context()).unwrap(), None); + assert_eq!(decoder.contexts.len(), 1); + assert_eq!(decoder.contexts[0].header_offset, root_header); + assert_eq!(decoder.contexts[0].array_offset, root_array); + assert_eq!(decoder.contexts[0].states.len(), 256); + + decoder.init_model(4).unwrap(); + assert_eq!(decoder.push_context(new_context()).unwrap(), Some(1)); +} + +#[test] +fn context_allocation_releases_header_when_state_array_does_not_fit() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 10; + decoder.suballoc.reset(16 * ALLOC_UNIT_BYTES); + let state = State { + symbol: 0, + freq: 1, + successor: Successor::None, + }; + let context = Context { + states: vec![state; 256].into(), + summ_freq: 257, + suffix: None, + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }; + assert_eq!(decoder.push_context(context).unwrap(), None); + assert_eq!(decoder.suballoc.free_lists[0].len(), 1); + assert!(decoder.contexts.is_empty()); +} + +#[test] +fn model_restarts_at_text_boundary_and_failed_successor_creation() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + decoder.suballoc.reset(16 * ALLOC_UNIT_BYTES); + decoder + .text + .resize(decoder.suballoc.text_capacity_bytes - 1, 0) + .unwrap(); + decoder.update_model().unwrap(); + assert!(decoder.text.is_empty()); + assert_eq!(decoder.contexts.len(), 1); + + decoder.order_fall = 0; + decoder.update_model().unwrap(); + assert_eq!(decoder.order_fall, 4); + + decoder.contexts[0].states[0].successor = Successor::Raw(usize::MAX); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[0].states[0].successor, Successor::None); +} + +#[test] +fn model_restarts_when_an_ancestor_state_array_cannot_grow() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 10; + decoder.init_model(4).unwrap(); + let state = |symbol| State { + symbol, + freq: 1, + successor: Successor::None, + }; + let ancestor = decoder + .push_context(Context { + states: vec![state(b'a'), state(b'b')].into(), + summ_freq: 3, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + let selected = decoder + .push_context(Context { + states: vec![state(b'c'), state(b'd')].into(), + summ_freq: 3, + suffix: Some(ancestor), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + let expanded = decoder + .push_context(Context { + states: vec![state(b'a'), state(b'b')].into(), + summ_freq: 3, + suffix: Some(selected), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.min_context = selected; + decoder.max_context = expanded; + decoder.found_state = StateRef { + context: selected, + index: 0, + }; + decoder.order_fall = 1; + + decoder.suballoc.reset(16 * ALLOC_UNIT_BYTES); + decoder + .text + .resize(decoder.suballoc.text_capacity_bytes - 2, 0) + .unwrap(); + while decoder.suballoc.hi_bump > decoder.suballoc.lo_bump { + decoder.suballoc.alloc(1, AllocSide::Lo, 0).unwrap(); + } + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts.len(), 1); + assert_eq!(decoder.order_fall, 4); + assert!(decoder.text.is_empty()); +} + +#[test] +fn model_adds_ancestor_state_with_raw_or_context_successor() { + fn chain(existing_successor: Successor) -> (PpmdDecoder, usize) { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 10; + decoder.init_model(4).unwrap(); + let state = |symbol, successor| State { + symbol, + freq: 1, + successor, + }; + let ancestor = decoder + .push_context(Context { + states: vec![state(b'a', Successor::None), state(b'b', Successor::None)].into(), + summ_freq: 3, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + let selected = decoder + .push_context(Context { + states: vec![ + state(b'c', existing_successor), + state(b'd', Successor::None), + ] + .into(), + summ_freq: 3, + suffix: Some(ancestor), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + let expanded = decoder + .push_context(Context { + states: vec![state(b'a', Successor::None), state(b'b', Successor::None)].into(), + summ_freq: 3, + suffix: Some(selected), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.min_context = selected; + decoder.max_context = expanded; + decoder.found_state = StateRef { + context: selected, + index: 0, + }; + decoder.order_fall = 1; + (decoder, expanded) + } + + let (mut decoder, expanded) = chain(Successor::None); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[expanded].states.len(), 3); + assert_eq!( + decoder.contexts[expanded].states[2].successor, + Successor::Raw(1) + ); + assert_eq!(&*decoder.text, b"c"); + + let (mut decoder, expanded) = chain(Successor::Context(2)); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[expanded].states.len(), 3); + assert_eq!( + decoder.contexts[expanded].states[2].successor, + Successor::Context(2) + ); + assert!(decoder.text.is_empty()); +} + +#[test] +fn model_updates_suffix_frequencies_at_the_binary_and_multi_state_caps() { + fn chain(suffix: Vec, symbol: u8) -> (PpmdDecoder, usize) { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 10; + decoder.init_model(4).unwrap(); + let ancestor = decoder + .push_context(Context { + summ_freq: suffix.iter().map(|s| s.freq as u16).sum::() + 1, + states: suffix.into(), + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + let selected = decoder + .push_context(Context { + states: vec![State { + symbol, + freq: 2, + successor: Successor::None, + }] + .into(), + summ_freq: 0, + suffix: Some(ancestor), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.min_context = selected; + decoder.max_context = selected; + decoder.found_state = StateRef { + context: selected, + index: 0, + }; + decoder.order_fall = 1; + (decoder, ancestor) + } + let state = |symbol, freq| State { + symbol, + freq, + successor: Successor::None, + }; + + for (initial, expected) in [(31, 32), (32, 32)] { + let (mut decoder, ancestor) = chain(vec![state(b'a', initial)], b'a'); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[ancestor].states[0].freq, expected); + } + + let (mut decoder, ancestor) = chain(vec![state(b'a', 2), state(b'b', 2)], b'b'); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[ancestor].states[0].symbol, b'b'); + assert_eq!(decoder.contexts[ancestor].states[0].freq, 4); + + let (mut decoder, ancestor) = chain(vec![state(b'a', 1), state(b'b', 115)], b'b'); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[ancestor].states[0].freq, 115); + + let (mut decoder, ancestor) = chain(vec![state(b'a', 2), state(b'b', 2)], b'c'); + decoder.update_model().unwrap(); + assert_eq!(decoder.contexts[ancestor].states[0].freq, 2); + assert_eq!(decoder.contexts[ancestor].states[1].freq, 2); +} + +#[test] +fn successor_creation_materializes_raw_chain_and_handles_limits() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 10; + decoder.init_model(4).unwrap(); + let selected = decoder + .push_context(Context { + states: vec![State { + symbol: b'a', + freq: 2, + successor: Successor::Raw(0), + }] + .into(), + summ_freq: 0, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.contexts[0].states[b'a' as usize].successor = Successor::Raw(0); + decoder.min_context = selected; + decoder.found_state = StateRef { + context: selected, + index: 0, + }; + decoder.order_fall = 1; + decoder.text.push(b'b').unwrap(); + + let mut limited = decoder.clone(); + limited.max_contexts = 2; + assert_eq!(limited.create_successors().unwrap(), None); + let mut conflicting = decoder.clone(); + conflicting.contexts[0].states[b'a' as usize].successor = Successor::None; + assert_eq!(conflicting.create_successors().unwrap(), None); + + let leaf = decoder.create_successors().unwrap().unwrap(); + assert_eq!(decoder.contexts.len(), 4); + assert_eq!(decoder.contexts[leaf].states[0].symbol, b'b'); + assert_eq!(decoder.contexts[leaf].suffix, Some(2)); + assert_eq!( + decoder.contexts[0].states[b'a' as usize].successor, + Successor::Context(2) + ); + assert_eq!( + decoder.contexts[selected].states[0].successor, + Successor::Context(leaf) + ); +} + +#[test] +fn successor_creation_reuses_existing_context_without_materialization() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 4; + decoder.init_model(4).unwrap(); + let selected = decoder + .push_context(Context { + states: vec![State { + symbol: b'a', + freq: 2, + successor: Successor::Raw(0), + }] + .into(), + summ_freq: 0, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.min_context = selected; + decoder.found_state = StateRef { + context: selected, + index: 0, + }; + decoder.order_fall = 0; + decoder.contexts[0].states[b'a' as usize].successor = Successor::Context(0); + assert_eq!(decoder.create_successors().unwrap(), Some(0)); + assert_eq!(decoder.contexts.len(), 2); + + decoder.contexts[selected].states[0].successor = Successor::Context(0); + assert_eq!(decoder.create_successors().unwrap(), Some(0)); + decoder.order_fall = 1; + assert_eq!(decoder.create_successors().unwrap(), None); + decoder.order_fall = 0; + decoder.contexts[selected].states[0].successor = Successor::None; + assert_eq!(decoder.create_successors().unwrap(), None); + + decoder.min_context = 0; + decoder.found_state = StateRef { + context: 0, + index: 0, + }; + decoder.contexts[0].states[0].successor = Successor::Raw(0); + assert_eq!(decoder.create_successors().unwrap(), Some(0)); +} + +#[test] +fn cancelled_model_init_does_not_erase_contexts() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + let token = crate::ReadCancellation::new(); + decoder.set_read_control(crate::read_control::ReadControl::new(Some(&token))); + token.cancel(); + assert_eq!(decoder.init_model(8), Err(Error::Cancelled)); + assert_eq!(decoder.max_order, 4); + assert_eq!(decoder.contexts.len(), 1); +} + +#[test] +fn shrink_array_reuses_a_free_bucket_or_splits_in_place() { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 4; + let state = State { + symbol: 0, + freq: 1, + successor: Successor::None, + }; + let context = Context { + states: vec![state; 12].into(), + summ_freq: 13, + suffix: None, + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }; + let ctx = decoder.push_context(context).unwrap().unwrap(); + let original = decoder.contexts[ctx].array_offset; + decoder.shrink_state_array(ctx, 12, 12).unwrap(); + assert_eq!(decoder.contexts[ctx].array_offset, original); + // Six and five units share a bucket, so this shrink keeps its slot. + decoder.shrink_state_array(ctx, 12, 10).unwrap(); + assert_eq!(decoder.contexts[ctx].array_offset, original); + + // Six to two units has no spare block: keep the prefix and free the + // four-unit residue. A spare two-unit block later selects the swap. + decoder.shrink_state_array(ctx, 12, 4).unwrap(); + assert_eq!(decoder.contexts[ctx].array_offset, original); + assert!(decoder.suballoc.free_lists[3].contains(&(original + 2))); + let spare = decoder.suballoc.alloc(2, AllocSide::Lo, 0).unwrap(); + decoder.suballoc.free(spare, 2); + decoder.shrink_state_array(ctx, 12, 4).unwrap(); + assert_eq!(decoder.contexts[ctx].array_offset, spare); + assert!(decoder.suballoc.free_lists[4].contains(&original)); + + decoder.shrink_state_array(ctx, 4, 1).unwrap(); + assert_eq!(decoder.contexts[ctx].array_offset, NULL_OFFSET); +} + +#[test] +fn rescale_sorts_survivors_and_collapses_or_shrinks_arrays() { + fn model( + freqs: &[(u8, u8)], + sum: u16, + order_fall: usize, + found: usize, + ) -> (PpmdDecoder, usize) { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 4; + decoder.init_model(4).unwrap(); + let ctx = decoder + .push_context(Context { + states: freqs + .iter() + .map(|&(symbol, freq)| State { + symbol, + freq, + successor: Successor::None, + }) + .collect::>() + .into(), + summ_freq: sum, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.min_context = ctx; + decoder.found_state = StateRef { + context: ctx, + index: found, + }; + decoder.order_fall = order_fall; + (decoder, ctx) + } + + let (mut decoder, ctx) = model(&[(b'a', 10), (b'b', 1), (b'c', 80)], 92, 1, 0); + decoder.rescale().unwrap(); + assert_eq!( + decoder.contexts[ctx] + .states + .iter() + .map(|s| (s.symbol, s.freq)) + .collect::>(), + vec![(b'c', 40), (b'a', 7), (b'b', 1)] + ); + assert_eq!(decoder.contexts[ctx].summ_freq, 49); + + let (mut decoder, ctx) = model(&[(b'a', 10), (b'b', 125), (b'c', 2)], 138, 1, 1); + decoder.rescale().unwrap(); + assert_eq!(decoder.found_state.index, 0); + assert_eq!(decoder.contexts[ctx].states[0].symbol, b'b'); + assert_eq!(decoder.contexts[ctx].states[0].freq, 65); + + let (mut decoder, ctx) = model(&[(b'a', 125), (b'b', 1), (b'c', 1)], 130, 0, 0); + decoder.rescale().unwrap(); + assert_eq!(decoder.contexts[ctx].states.len(), 1); + assert_eq!(decoder.contexts[ctx].states[0].freq, 16); + assert_eq!(decoder.contexts[ctx].array_offset, NULL_OFFSET); + + let (mut decoder, ctx) = model(&[(b'a', 125), (b'b', 2), (b'c', 1)], 130, 0, 0); + let old_array = decoder.contexts[ctx].array_offset; + decoder.rescale().unwrap(); + assert_eq!( + decoder.contexts[ctx] + .states + .iter() + .map(|s| (s.symbol, s.freq)) + .collect::>(), + vec![(b'a', 64), (b'b', 1)] + ); + assert_eq!(decoder.contexts[ctx].summ_freq, 67); + assert_eq!(decoder.contexts[ctx].array_offset, old_array); + assert!(decoder.suballoc.free_lists[0].contains(&(old_array + 1))); +} + +#[test] +fn frequency_updates_rescale_at_the_reference_call_sites() { + fn model(freqs: &[(u8, u8)], selected: usize) -> (PpmdDecoder, usize) { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 4; + decoder.init_model(4).unwrap(); + let ctx = decoder + .push_context(Context { + states: freqs + .iter() + .map(|&(symbol, freq)| State { + symbol, + freq, + successor: Successor::None, + }) + .collect::>() + .into(), + summ_freq: freqs.iter().map(|&(_, freq)| freq as u16).sum::() + 1, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + for state in &mut decoder.contexts[ctx].states { + state.successor = Successor::Context(ctx); + } + decoder.min_context = ctx; + decoder.max_context = ctx; + decoder.found_state = StateRef { + context: ctx, + index: selected, + }; + decoder.order_fall = 0; + (decoder, ctx) + } + + let (mut decoder, ctx) = model(&[(b'a', 121), (b'b', 1)], 0); + decoder.update1_0().unwrap(); + assert_eq!(decoder.contexts[ctx].states.len(), 1); + + let (mut decoder, ctx) = model(&[(b'a', 1), (b'b', 121)], 1); + decoder.update1().unwrap(); + assert_eq!(decoder.contexts[ctx].states.len(), 1); + + // Variant H rescales update1 only when the selected state moves + // ahead of its predecessor, even if its frequency now exceeds 124. + let (mut decoder, ctx) = model(&[(b'a', 125), (b'b', 121)], 1); + decoder.update1().unwrap(); + assert_eq!(decoder.contexts[ctx].states[1].freq, 125); + assert_eq!(decoder.contexts[ctx].states.len(), 2); + + let (mut decoder, ctx) = model(&[(b'a', 121), (b'b', 1)], 0); + decoder.update2().unwrap(); + assert_eq!(decoder.contexts[ctx].states.len(), 1); + + let (mut decoder, ctx) = model(&[(b'a', 127)], 0); + decoder.update_bin().unwrap(); + assert_eq!(decoder.contexts[ctx].states[0].freq, 128); + decoder.update_bin().unwrap(); + assert_eq!(decoder.contexts[ctx].states[0].freq, 128); +} + +#[test] +fn make_esc_freq_rejects_invalid_masked_state_count() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + decoder + .contexts + .push(Context { + states: vec![ + State { + symbol: b'a', + freq: 1, + successor: Successor::None, + }, + State { + symbol: b'b', + freq: 1, + successor: Successor::None, + }, + ] + .into(), + summ_freq: 2, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap(); + decoder.min_context = 1; + + assert!(matches!( + decoder.make_esc_freq(2), + Err(Error::InvalidData("RAR PPMd masked-state count is invalid")) + )); +} + +#[test] +fn update_model_rejects_invalid_frequency_arithmetic() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + decoder + .contexts + .push(Context { + states: vec![State { + symbol: b'a', + freq: 10, + successor: Successor::None, + }] + .into(), + summ_freq: 1, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap(); + decoder.min_context = 1; + decoder.max_context = 0; + decoder.found_state = StateRef { + context: 1, + index: 0, + }; + decoder.order_fall = 1; + + assert!(matches!( + decoder.update_model(), + Err(Error::InvalidData("RAR PPMd model frequency is invalid")) + )); +} + +#[test] +fn update_model_rejects_zero_frequency_sum_and_unrepresentable_sum() { + fn chain( + ancestor: Vec, + ancestor_sum: u16, + selected: Vec, + selected_sum: u16, + ) -> PpmdDecoder { + let mut decoder = PpmdDecoder::new(); + decoder.max_contexts = 10; + decoder.init_model(4).unwrap(); + let ancestor = decoder + .push_context(Context { + states: ancestor.into(), + summ_freq: ancestor_sum, + suffix: Some(0), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + let selected = decoder + .push_context(Context { + states: selected.into(), + summ_freq: selected_sum, + suffix: Some(ancestor), + header_offset: NULL_OFFSET, + array_offset: NULL_OFFSET, + }) + .unwrap() + .unwrap(); + decoder.min_context = selected; + decoder.max_context = ancestor; + decoder.found_state = StateRef { + context: selected, + index: 0, + }; + decoder.order_fall = 1; + decoder + } + let state = |symbol, freq| State { + symbol, + freq, + successor: Successor::None, + }; + + let mut decoder = chain( + vec![state(b'a', 0)], + 0, + vec![state(b'c', 31), state(b'd', 1)], + 32, + ); + assert_eq!( + decoder.update_model(), + Err(Error::InvalidData("RAR PPMd model frequency is invalid")) + ); + + let selected = (0..10) + .map(|i| state(b'c' + i, if i == 0 { 31 } else { 1 })) + .collect(); + let mut decoder = chain(vec![state(b'a', 1), state(b'b', 1)], u16::MAX, selected, 40); + assert_eq!( + decoder.update_model(), + Err(Error::InvalidData("RAR PPMd model frequency overflows")) + ); +} + +#[test] +fn update_paths_reject_invalid_state_reference_without_panic() { + let mut decoder = PpmdDecoder::new(); + decoder.init_model(4).unwrap(); + decoder.found_state = StateRef { + context: 99, + index: 0, + }; + + assert_eq!( + decoder.update1_0(), + Err(Error::InvalidData("RAR PPMd state reference is invalid")) + ); + + decoder.found_state = StateRef { + context: 0, + index: 999, + }; + assert_eq!( + decoder.update_bin(), + Err(Error::InvalidData("RAR PPMd state reference is invalid")) + ); +} diff --git a/arca-rar/vendor/rars/src/codec/rar13.rs b/arca-rar/vendor/rars/src/codec/rar13.rs new file mode 100644 index 00000000..14611498 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar13.rs @@ -0,0 +1,923 @@ +#[cfg(feature = "write")] +mod encoder; +#[cfg(feature = "write")] +pub(crate) use encoder::unpack15_encode_with_options_and_progress; +#[cfg(feature = "write")] +pub use encoder::{ + find_long_lz, unpack15_encode, unpack15_encode_with_options, EncodeOptions, LongLz, + Unpack15Encoder, +}; + +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; +use std::io::{Read, Write}; + +const DEC_L1: &[u16] = &[ + 0x8000, 0xa000, 0xc000, 0xd000, 0xe000, 0xea00, 0xee00, 0xf000, 0xf200, 0xf200, 0xffff, +]; +const POS_L1: &[u16] = &[0, 0, 0, 2, 3, 5, 7, 11, 16, 20, 24, 32, 32]; +const DEC_L2: &[u16] = &[ + 0xa000, 0xc000, 0xd000, 0xe000, 0xea00, 0xee00, 0xf000, 0xf200, 0xf240, 0xffff, +]; +const POS_L2: &[u16] = &[0, 0, 0, 0, 5, 7, 9, 13, 18, 22, 26, 34, 36]; +const DEC_HF0: &[u16] = &[ + 0x8000, 0xc000, 0xe000, 0xf200, 0xf200, 0xf200, 0xf200, 0xf200, 0xffff, +]; +const POS_HF0: &[u16] = &[0, 0, 0, 0, 0, 8, 16, 24, 33, 33, 33, 33, 33]; +const DEC_HF1: &[u16] = &[ + 0x2000, 0xc000, 0xe000, 0xf000, 0xf200, 0xf200, 0xf7e0, 0xffff, +]; +const POS_HF1: &[u16] = &[0, 0, 0, 0, 0, 0, 4, 44, 60, 76, 80, 80, 127]; +const DEC_HF2: &[u16] = &[ + 0x1000, 0x2400, 0x8000, 0xc000, 0xfa00, 0xffff, 0xffff, 0xffff, +]; +const POS_HF2: &[u16] = &[0, 0, 0, 0, 0, 0, 2, 7, 53, 117, 233, 0, 0]; +const DEC_HF3: &[u16] = &[0x0800, 0x2400, 0xee00, 0xfe80, 0xffff, 0xffff, 0xffff]; +const POS_HF3: &[u16] = &[0, 0, 0, 0, 0, 0, 0, 2, 16, 218, 251, 0, 0]; +const DEC_HF4: &[u16] = &[0xff00, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff]; +const POS_HF4: &[u16] = &[0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0]; + +const SHORT_LEN1: [u8; 16] = [1, 3, 4, 4, 5, 6, 7, 8, 8, 4, 4, 5, 6, 6, 4, 0]; +const SHORT_XOR1: [u8; 15] = [ + 0x00, 0xa0, 0xd0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe, 0xff, 0xc0, 0x80, 0x90, 0x98, 0x9c, 0xb0, +]; +const SHORT_LEN2: [u8; 16] = [2, 3, 3, 3, 4, 4, 5, 6, 6, 4, 4, 5, 6, 6, 4, 0]; +const SHORT_XOR2: [u8; 15] = [ + 0x00, 0x40, 0x60, 0xa0, 0xd0, 0xe0, 0xf0, 0xf8, 0xfc, 0xc0, 0x80, 0x90, 0x98, 0x9c, 0xb0, +]; + +pub fn unpack15_decode(input: &[u8], output_size: usize) -> Result> { + let mut decoder = Unpack15::new(); + decoder.decode_member(input, output_size, false) +} + +#[derive(Clone)] +pub struct Unpack15 { + pub(crate) read_control: crate::read_control::ReadControl, + state: Reader15State, +} +impl Clone for Reader15State { + fn clone(&self) -> Self { + self.try_clone().expect("unlimited legacy decoder copy") + } +} +impl Unpack15 { + pub fn new() -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + state: Reader15State::with_allowance(&Allowance::default()) + .expect("unlimited legacy decoder"), + } + } + pub fn decode_member(&mut self, input: &[u8], target: usize, solid: bool) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member(input, target, solid) + .map(Buffer::into_vec) + } + pub fn decode_member_to( + &mut self, + input: &[u8], + target: usize, + solid: bool, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state.decode_member_to(input, target, solid, out) + } + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + target: usize, + solid: bool, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_from_reader(input, target, solid, out) + } +} +#[cfg(all(test, feature = "write"))] +impl Reader15State { + fn new() -> Self { + Self::with_allowance(&Allowance::default()).unwrap() + } +} +pub(crate) struct Reader15State { + pub(crate) read_control: crate::read_control::ReadControl, + bits: ReaderBits, + target: usize, + output_written: usize, + window: Buffer, + unp_ptr: usize, + prev_ptr: usize, + first_win_done: bool, + // State names follow RAR13_FORMAT_SPECIFICATION.md §6 so the codec state + // lines up directly with the documented Unpack15 tables and traces. + ch_set: [u16; 256], + ch_set_a: [u16; 256], + ch_set_b: [u16; 256], + ch_set_c: [u16; 256], + n_to_pl: [u8; 256], + n_to_pl_b: [u8; 256], + n_to_pl_c: [u8; 256], + avr_plc: u32, + avr_plc_b: u32, + avr_ln1: u32, + avr_ln2: u32, + avr_ln3: u32, + max_dist3: u32, + nhfb: u32, + nlzb: u32, + num_huf: u32, + buf60: u32, + st_mode: bool, + l_count: u32, + flag_buf: u32, + flags_cnt: i32, + old_dist: [u32; 4], + old_dist_ptr: usize, + last_dist: u32, + last_length: u32, + #[cfg(all(test, feature = "write"))] + token_stats: DecodeTokenStats, + #[cfg(all(test, feature = "write"))] + old_distance_events: Vec, +} + +#[cfg(all(test, feature = "write"))] +#[derive(Clone, Copy, Debug, Default)] +struct DecodeTokenStats { + literals: u64, + st_literals: u64, + st_matches: u64, + st_match_bytes: u64, + short_matches: u64, + short_match_bytes: u64, + repeat_matches: u64, + repeat_match_bytes: u64, + old_distance_matches: u64, + old_distance_match_bytes: u64, + old_distance_codes: [u64; 4], + old_distance_near: u64, + old_distance_far: u64, + long_near_matches: u64, + long_near_match_bytes: u64, + long_far_matches: u64, + long_far_match_bytes: u64, +} + +#[cfg(all(test, feature = "write"))] +#[derive(Clone, Copy, Debug)] +struct OldDistanceEvent { + output_position: usize, + short_code: u32, + distance: u32, + length: u32, + max_dist3: u32, +} + +impl Reader15State { + /// A decoder ready to read either a fresh member or a solid continuation. + /// + /// The starting state comes from `reset_non_solid` rather than being + /// hand-copied, because the two drifted: the tables `init_huff` fills were + /// left zeroed here. Nothing noticed while every archive opened with a + /// non-solid member, since that member resets them before the first symbol + /// is read. A first member carrying the solid flag skips the reset and + /// decoded against zeroes. + pub(crate) fn with_allowance(allowance: &B) -> Result { + let mut decoder = Self { + read_control: crate::read_control::ReadControl::default(), + bits: ReaderBits::with_allowance(&[], allowance)?, + target: 0, + output_written: 0, + window: Buffer::filled(0x10000, 0, allowance)?, + unp_ptr: 0, + prev_ptr: 0, + first_win_done: false, + ch_set: [0; 256], + ch_set_a: [0; 256], + ch_set_b: [0; 256], + ch_set_c: [0; 256], + n_to_pl: [0; 256], + n_to_pl_b: [0; 256], + n_to_pl_c: [0; 256], + avr_plc: 0, + avr_plc_b: 0, + avr_ln1: 0, + avr_ln2: 0, + avr_ln3: 0, + max_dist3: 0, + nhfb: 0, + nlzb: 0, + num_huf: 0, + buf60: 0, + st_mode: false, + l_count: 0, + flag_buf: 0, + flags_cnt: 0, + old_dist: [0; 4], + old_dist_ptr: 0, + last_dist: 0, + last_length: 0, + #[cfg(all(test, feature = "write"))] + token_stats: DecodeTokenStats::default(), + #[cfg(all(test, feature = "write"))] + old_distance_events: Vec::new(), + }; + decoder.reset_non_solid(); + Ok(decoder) + } + + pub(crate) fn try_clone(&self) -> Result { + Ok(Self { + read_control: self.read_control.clone(), + bits: self.bits.try_clone()?, + target: self.target, + output_written: self.output_written, + window: Buffer::copied(&self.window, &self.window.allowance())?, + unp_ptr: self.unp_ptr, + prev_ptr: self.prev_ptr, + first_win_done: self.first_win_done, + ch_set: self.ch_set, + ch_set_a: self.ch_set_a, + ch_set_b: self.ch_set_b, + ch_set_c: self.ch_set_c, + n_to_pl: self.n_to_pl, + n_to_pl_b: self.n_to_pl_b, + n_to_pl_c: self.n_to_pl_c, + avr_plc: self.avr_plc, + avr_plc_b: self.avr_plc_b, + avr_ln1: self.avr_ln1, + avr_ln2: self.avr_ln2, + avr_ln3: self.avr_ln3, + max_dist3: self.max_dist3, + nhfb: self.nhfb, + nlzb: self.nlzb, + num_huf: self.num_huf, + buf60: self.buf60, + st_mode: self.st_mode, + l_count: self.l_count, + flag_buf: self.flag_buf, + flags_cnt: self.flags_cnt, + old_dist: self.old_dist, + old_dist_ptr: self.old_dist_ptr, + last_dist: self.last_dist, + last_length: self.last_length, + #[cfg(all(test, feature = "write"))] + token_stats: self.token_stats, + #[cfg(all(test, feature = "write"))] + old_distance_events: self.old_distance_events.clone(), + }) + } + pub fn decode_member( + &mut self, + input: &[u8], + target: usize, + solid: bool, + ) -> Result> { + self.read_control.check_codec()?; + let mut output = Buffer::with_capacity(target, &self.window.allowance())?; + self.decode_member_to(input, target, solid, &mut output)?; + Ok(output) + } + + pub fn decode_member_to( + &mut self, + input: &[u8], + target: usize, + solid: bool, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + self.init_member(target, solid); + self.bits = ReaderBits::with_allowance(input, &self.window.allowance())?; + self.decode_loop(out) + } + + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + target: usize, + solid: bool, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + const OUTPUT_CHUNK: usize = 64 * 1024; + + self.init_member(target, solid); + self.bits = ReaderBits::with_allowance(&[], &self.window.allowance())?; + self.bits.input.read_to_end(input)?; + + while self.output_written < self.target { + let chunk_target = self + .output_written + .saturating_add(OUTPUT_CHUNK) + .min(self.target); + let mut chunk = Buffer::with_capacity( + chunk_target - self.output_written, + &self.window.allowance(), + )?; + self.decode_loop_until(chunk_target, &mut chunk)?; + out.write_all(&chunk).map_err(Error::from)?; + } + Ok(()) + } + + fn init_member(&mut self, target: usize, solid: bool) { + self.target = target; + self.output_written = 0; + self.flags_cnt = -2; + self.flag_buf = 0; + self.st_mode = false; + self.l_count = 0; + + if !solid { + self.reset_non_solid(); + } + } + + fn decode_loop(&mut self, out: &mut impl Write) -> Result<()> { + if self.target == 0 { + return Ok(()); + } + + self.decode_loop_until(self.target, out) + } + + fn decode_loop_until(&mut self, target: usize, out: &mut impl Write) -> Result<()> { + let mut poller = self.read_control.poller(); + while self.output_written < target { + poller.check_codec(self.output_written)?; + self.decode_step(out)?; + } + + Ok(()) + } + + fn decode_step(&mut self, out: &mut impl Write) -> Result<()> { + if self.flags_cnt == -2 { + self.get_flags_buf()?; + self.flags_cnt = 8; + } + + self.unp_ptr &= 0xffff; + self.first_win_done |= self.prev_ptr > self.unp_ptr; + self.prev_ptr = self.unp_ptr; + + if self.st_mode { + return self.huff_decode(out); + } + + self.flags_cnt -= 1; + if self.flags_cnt < 0 { + self.get_flags_buf()?; + self.flags_cnt = 7; + } + + if self.flag_buf & 0x80 != 0 { + self.flag_buf = (self.flag_buf << 1) & 0xff; + if self.nlzb > self.nhfb { + self.long_lz(out) + } else { + self.huff_decode(out) + } + } else { + self.flag_buf = (self.flag_buf << 1) & 0xff; + self.flags_cnt -= 1; + if self.flags_cnt < 0 { + self.get_flags_buf()?; + self.flags_cnt = 7; + } + if self.flag_buf & 0x80 != 0 { + self.flag_buf = (self.flag_buf << 1) & 0xff; + if self.nlzb > self.nhfb { + self.huff_decode(out) + } else { + self.long_lz(out) + } + } else { + self.flag_buf = (self.flag_buf << 1) & 0xff; + self.short_lz(out) + } + } + } + + fn reset_non_solid(&mut self) { + self.window.fill(0); + self.unp_ptr = 0; + self.prev_ptr = 0; + self.first_win_done = false; + self.avr_plc_b = 0; + self.avr_ln1 = 0; + self.avr_ln2 = 0; + self.avr_ln3 = 0; + self.num_huf = 0; + self.buf60 = 0; + self.avr_plc = 0x3500; + self.max_dist3 = 0x2001; + self.nhfb = 0x80; + self.nlzb = 0x80; + self.old_dist = [u32::MAX; 4]; + self.old_dist_ptr = 0; + self.last_dist = u32::MAX; + self.last_length = 0; + self.init_huff(); + } + + fn short_lz(&mut self, out: &mut impl Write) -> Result<()> { + self.num_huf = 0; + let mut bit_field = self.bits.get_bits(); + if self.l_count == 2 { + self.bits.add_bits(1); + if bit_field >= 0x8000 { + #[cfg(all(test, feature = "write"))] + { + self.token_stats.repeat_matches += 1; + self.token_stats.repeat_match_bytes += u64::from(self.last_length); + } + self.copy_string(self.last_dist, self.last_length, out)?; + return Ok(()); + } + bit_field = (bit_field << 1) & 0xffff; + self.l_count = 0; + } + + let bit_byte = (bit_field >> 8) as u8; + let mut length = 0usize; + // Both prefix tables cover every byte in either Buf60 state. + // The exhaustive table test verifies that a matching entry always exists. + if self.avr_ln1 < 37 { + loop { + let short_len = self.short_len1(length); + let mask = (!(0xffu16 >> short_len)) as u8; + if ((bit_byte ^ SHORT_XOR1[length]) & mask) == 0 { + break; + } + length += 1; + } + self.bits.add_bits(self.short_len1(length) as usize); + } else { + loop { + let short_len = self.short_len2(length); + let mask = (!(0xffu16 >> short_len)) as u8; + if ((bit_byte ^ SHORT_XOR2[length]) & mask) == 0 { + break; + } + length += 1; + } + self.bits.add_bits(self.short_len2(length) as usize); + } + + let mut length = length as u32; + if length >= 9 { + if length == 9 { + self.l_count += 1; + #[cfg(all(test, feature = "write"))] + { + self.token_stats.repeat_matches += 1; + self.token_stats.repeat_match_bytes += u64::from(self.last_length); + } + self.copy_string(self.last_dist, self.last_length, out)?; + return Ok(()); + } + if length == 14 { + self.l_count = 0; + length = self.decode_num(self.bits.get_bits(), 3, DEC_L2, POS_L2) + 5; + let distance = (self.bits.get_bits() >> 1) | 0x8000; + self.bits.add_bits(15); + self.last_length = length; + self.last_dist = distance; + #[cfg(all(test, feature = "write"))] + { + self.token_stats.short_matches += 1; + self.token_stats.short_match_bytes += u64::from(length); + } + self.copy_string(distance, length, out)?; + return Ok(()); + } + + self.l_count = 0; + let save_length = length; + let distance = + self.old_dist[(self.old_dist_ptr.wrapping_sub((length - 9) as usize)) & 3]; + length = self.decode_num(self.bits.get_bits(), 2, DEC_L1, POS_L1) + 2; + if length == 0x101 && save_length == 10 { + self.buf60 ^= 1; + return Ok(()); + } + if distance > 256 { + length += 1; + } + if distance >= self.max_dist3 { + length += 1; + } + + self.remember_match(distance, length); + #[cfg(all(test, feature = "write"))] + { + self.old_distance_events.push(OldDistanceEvent { + output_position: self.output_written, + short_code: save_length, + distance, + length, + max_dist3: self.max_dist3, + }); + self.token_stats.old_distance_matches += 1; + self.token_stats.old_distance_match_bytes += u64::from(length); + self.token_stats.old_distance_codes[(save_length - 10) as usize] += 1; + if distance <= 256 { + self.token_stats.old_distance_near += 1; + } else { + self.token_stats.old_distance_far += 1; + } + } + self.copy_string(distance, length, out)?; + return Ok(()); + } + + self.l_count = 0; + self.avr_ln1 += length; + self.avr_ln1 -= self.avr_ln1 >> 4; + + let distance_place = + (self.decode_num(self.bits.get_bits(), 5, DEC_HF2, POS_HF2) & 0xff) as usize; + let mut distance = self.ch_set_a[distance_place] as u32; + if distance_place > 0 { + let last_distance = self.ch_set_a[distance_place - 1]; + self.ch_set_a[distance_place] = last_distance; + self.ch_set_a[distance_place - 1] = distance as u16; + } + length += 2; + distance += 1; + self.remember_match(distance, length); + #[cfg(all(test, feature = "write"))] + { + self.token_stats.short_matches += 1; + self.token_stats.short_match_bytes += u64::from(length); + } + self.copy_string(distance, length, out) + } + + fn long_lz(&mut self, out: &mut impl Write) -> Result<()> { + self.num_huf = 0; + self.nlzb += 16; + if self.nlzb > 0xff { + self.nlzb = 0x90; + self.nhfb >>= 1; + } + let old_avr2 = self.avr_ln2; + + let bit_field = self.bits.get_bits(); + let mut length = if self.avr_ln2 >= 122 { + self.decode_num(bit_field, 3, DEC_L2, POS_L2) + } else if self.avr_ln2 >= 64 { + self.decode_num(bit_field, 2, DEC_L1, POS_L1) + } else if bit_field < 0x100 { + self.bits.add_bits(16); + bit_field + } else { + let mut length = 0u32; + while ((bit_field << length) & 0x8000) == 0 { + length += 1; + } + self.bits.add_bits((length + 1) as usize); + length + }; + + self.avr_ln2 += length; + self.avr_ln2 -= self.avr_ln2 >> 5; + + let bit_field = self.bits.get_bits(); + let distance_place = if self.avr_plc_b > 0x28ff { + self.decode_num(bit_field, 5, DEC_HF2, POS_HF2) + } else if self.avr_plc_b > 0x06ff { + self.decode_num(bit_field, 5, DEC_HF1, POS_HF1) + } else { + self.decode_num(bit_field, 4, DEC_HF0, POS_HF0) + }; + + self.avr_plc_b += distance_place; + self.avr_plc_b -= self.avr_plc_b >> 8; + + let idx = (distance_place & 0xff) as usize; + let mut distance; + let mut new_distance_place; + loop { + distance = self.ch_set_b[idx] as u32; + new_distance_place = self.n_to_pl_b[(distance & 0xff) as usize] as usize; + self.n_to_pl_b[(distance & 0xff) as usize] = + self.n_to_pl_b[(distance & 0xff) as usize].wrapping_add(1); + distance += 1; + if distance & 0xff == 0 { + corr_huff(&mut self.ch_set_b, &mut self.n_to_pl_b); + } else { + break; + } + } + + self.ch_set_b[idx] = self.ch_set_b[new_distance_place]; + self.ch_set_b[new_distance_place] = distance as u16; + + distance = ((distance & 0xff00) | (self.bits.get_bits() >> 8)) >> 1; + self.bits.add_bits(7); + + let old_avr3 = self.avr_ln3; + if length != 1 && length != 4 { + if length == 0 && distance <= self.max_dist3 { + self.avr_ln3 += 1; + self.avr_ln3 -= self.avr_ln3 >> 8; + } else if self.avr_ln3 > 0 { + self.avr_ln3 -= 1; + } + } + length += 3; + if distance >= self.max_dist3 { + length += 1; + } + if distance <= 256 { + length += 8; + } + if old_avr3 > 0xb0 || (self.avr_plc >= 0x2a00 && old_avr2 < 0x40) { + self.max_dist3 = 0x7f00; + } else { + self.max_dist3 = 0x2001; + } + + self.remember_match(distance, length); + #[cfg(all(test, feature = "write"))] + if distance <= 256 { + self.token_stats.long_near_matches += 1; + self.token_stats.long_near_match_bytes += u64::from(length); + } else { + self.token_stats.long_far_matches += 1; + self.token_stats.long_far_match_bytes += u64::from(length); + } + self.copy_string(distance, length, out) + } + + fn huff_decode(&mut self, out: &mut impl Write) -> Result<()> { + let bit_field = self.bits.get_bits(); + + let mut byte_place = if self.avr_plc > 0x75ff { + self.decode_num(bit_field, 8, DEC_HF4, POS_HF4) + } else if self.avr_plc > 0x5dff { + self.decode_num(bit_field, 6, DEC_HF3, POS_HF3) + } else if self.avr_plc > 0x35ff { + self.decode_num(bit_field, 5, DEC_HF2, POS_HF2) + } else if self.avr_plc > 0x0dff { + self.decode_num(bit_field, 5, DEC_HF1, POS_HF1) + } else { + self.decode_num(bit_field, 4, DEC_HF0, POS_HF0) + } & 0xff; + + if self.st_mode { + if byte_place == 0 && bit_field > 0x0fff { + byte_place = 0x100; + } + if byte_place == 0 { + let bit_field = self.bits.get_bits(); + self.bits.add_bits(1); + if bit_field & 0x8000 != 0 { + self.num_huf = 0; + self.st_mode = false; + return Ok(()); + } + + let length = if bit_field & 0x4000 != 0 { 4 } else { 3 }; + self.bits.add_bits(1); + let mut distance = self.decode_num(self.bits.get_bits(), 5, DEC_HF2, POS_HF2); + distance = (distance << 5) | (self.bits.get_bits() >> 11); + self.bits.add_bits(5); + #[cfg(all(test, feature = "write"))] + { + self.token_stats.st_matches += 1; + self.token_stats.st_match_bytes += u64::from(length); + } + self.copy_string(distance, length, out)?; + return Ok(()); + } + byte_place -= 1; + } else { + if self.num_huf >= 16 && self.flags_cnt == 0 { + self.st_mode = true; + } + self.num_huf += 1; + } + + self.avr_plc += byte_place; + self.avr_plc -= self.avr_plc >> 8; + self.nhfb += 16; + if self.nhfb > 0xff { + self.nhfb = 0x90; + self.nlzb >>= 1; + } + + let byte = (self.ch_set[byte_place as usize] >> 8) as u8; + #[cfg(all(test, feature = "write"))] + if self.st_mode { + self.token_stats.st_literals += 1; + } else { + self.token_stats.literals += 1; + } + self.put_byte(byte, out)?; + + let idx = byte_place as usize; + let mut cur_byte; + let mut new_byte_place; + loop { + cur_byte = self.ch_set[idx] as u32; + new_byte_place = self.n_to_pl[(cur_byte & 0xff) as usize] as usize; + self.n_to_pl[(cur_byte & 0xff) as usize] = + self.n_to_pl[(cur_byte & 0xff) as usize].wrapping_add(1); + cur_byte += 1; + if cur_byte & 0xff > 0xa1 { + corr_huff(&mut self.ch_set, &mut self.n_to_pl); + } else { + break; + } + } + + self.ch_set[idx] = self.ch_set[new_byte_place]; + self.ch_set[new_byte_place] = cur_byte as u16; + Ok(()) + } + + fn get_flags_buf(&mut self) -> Result<()> { + let flags_place = self.decode_num(self.bits.get_bits(), 5, DEC_HF2, POS_HF2) as usize; + if flags_place >= self.ch_set_c.len() { + return Ok(()); + } + + let mut flags; + let mut new_flags_place; + loop { + flags = self.ch_set_c[flags_place] as u32; + new_flags_place = self.n_to_pl_c[(flags & 0xff) as usize] as usize; + self.n_to_pl_c[(flags & 0xff) as usize] = + self.n_to_pl_c[(flags & 0xff) as usize].wrapping_add(1); + self.flag_buf = flags >> 8; + flags += 1; + if flags & 0xff == 0 { + corr_huff(&mut self.ch_set_c, &mut self.n_to_pl_c); + } else { + break; + } + } + + self.ch_set_c[flags_place] = self.ch_set_c[new_flags_place]; + self.ch_set_c[new_flags_place] = flags as u16; + Ok(()) + } + + fn decode_num( + &mut self, + num: u32, + mut start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], + ) -> u32 { + let num = num & 0xfff0; + let mut i = 0usize; + while dec_tab[i] as u32 <= num { + start_pos += 1; + i += 1; + } + self.bits.add_bits(start_pos as usize); + ((num - if i > 0 { dec_tab[i - 1] as u32 } else { 0 }) >> (16 - start_pos)) + + pos_tab[start_pos as usize] as u32 + } + + fn copy_string(&mut self, distance: u32, length: u32, out: &mut impl Write) -> Result<()> { + if self.output_written + length as usize > self.target { + return Err(Error::InvalidData("RAR 1.3 match exceeds output size")); + } + + if (!self.first_win_done && distance as usize > self.unp_ptr) + || distance as usize > 0x10000 + || distance == 0 + { + for _ in 0..length { + self.put_byte(0, out)?; + } + } else { + for _ in 0..length { + let byte = self.window[(self.unp_ptr.wrapping_sub(distance as usize)) & 0xffff]; + self.put_byte(byte, out)?; + } + } + Ok(()) + } + + fn put_byte(&mut self, byte: u8, out: &mut impl Write) -> Result<()> { + if self.output_written >= self.target { + return Err(Error::InvalidData("RAR 1.3 literal exceeds output size")); + } + self.window[self.unp_ptr] = byte; + self.unp_ptr = (self.unp_ptr + 1) & 0xffff; + out.write_all(&[byte]).map_err(Error::from)?; + self.output_written += 1; + Ok(()) + } + + fn remember_match(&mut self, distance: u32, length: u32) { + self.old_dist[self.old_dist_ptr] = distance; + self.old_dist_ptr = (self.old_dist_ptr + 1) & 3; + self.last_length = length; + self.last_dist = distance; + } + + fn short_len1(&self, pos: usize) -> u32 { + if pos == 1 { + self.buf60 + 3 + } else { + SHORT_LEN1[pos] as u32 + } + } + + fn short_len2(&self, pos: usize) -> u32 { + if pos == 3 { + self.buf60 + 3 + } else { + SHORT_LEN2[pos] as u32 + } + } + + fn init_huff(&mut self) { + for i in 0..256 { + self.ch_set[i] = (i as u16) << 8; + self.ch_set_b[i] = (i as u16) << 8; + self.ch_set_a[i] = i as u16; + self.ch_set_c[i] = (0u8.wrapping_sub(i as u8) as u16) << 8; + } + self.n_to_pl = [0; 256]; + self.n_to_pl_b = [0; 256]; + self.n_to_pl_c = [0; 256]; + corr_huff(&mut self.ch_set_b, &mut self.n_to_pl_b); + } +} + +impl Default for Unpack15 { + fn default() -> Self { + Self::new() + } +} + +fn corr_huff(char_set: &mut [u16; 256], num_to_place: &mut [u8; 256]) { + let mut pos = 0usize; + for rank in (0..=7).rev() { + for _ in 0..32 { + char_set[pos] = (char_set[pos] & !0xff) | rank; + pos += 1; + } + } + *num_to_place = [0; 256]; + for rank in (0..=6).rev() { + num_to_place[rank] = ((7 - rank) * 32) as u8; + } +} + +struct ReaderBits { + input: Buffer, + bit_pos: usize, +} + +impl ReaderBits { + fn with_allowance(input: &[u8], allowance: &B) -> Result { + Ok(Self { + input: Buffer::copied(input, allowance)?, + bit_pos: 0, + }) + } + fn try_clone(&self) -> Result { + Ok(Self { + input: Buffer::copied(&self.input, &self.input.allowance())?, + bit_pos: self.bit_pos, + }) + } + + fn get_bits(&self) -> u32 { + let mut value = 0u32; + for i in 0..16 { + value <<= 1; + let bit_index = self.bit_pos + i; + let byte = self.input.get(bit_index / 8).copied().unwrap_or(0); + value |= ((byte >> (7 - (bit_index % 8))) & 1) as u32; + } + value + } + + fn add_bits(&mut self, count: usize) { + self.bit_pos += count; + } +} + +#[cfg(all(test, feature = "write"))] +impl ReaderBits { + fn new(input: &[u8]) -> Self { + Self::with_allowance(input, &Allowance::default()).unwrap() + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar13/encoder.rs b/arca-rar/vendor/rars/src/codec/rar13/encoder.rs new file mode 100644 index 00000000..fb900a6e --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar13/encoder.rs @@ -0,0 +1,1470 @@ +//! RAR13 member encoding and bit emission. +use super::*; + +/// Prebuilt long-LZ candidate index. Unlike the other encoders, rar13 builds +/// the index for the whole member up front and then queries it at arbitrary +/// positions, so candidates are stored per hash as position-sorted arrays: a +/// binary search finds the newest candidate before the query position and +/// iteration proceeds toward older ones with array locality. A linked-chain +/// finder is a poor fit here because its head always points at the newest +/// position in the member, forcing every query to pointer-chase past all +/// not-yet-reached positions. +#[derive(Debug, Clone)] +struct Rar13MatchFinder { + buckets: Vec>, +} + +const LONG_LZ_HASH_BITS: u32 = 16; + +impl Rar13MatchFinder { + fn build(input: &[u8]) -> Self { + let mut buckets = vec![Vec::new(); 1 << LONG_LZ_HASH_BITS]; + for pos in 0..input.len().saturating_sub(2) { + buckets[Self::hash(input, pos)].push(pos); + } + Self { buckets } + } + + fn hash(input: &[u8], pos: usize) -> usize { + let value = u32::from(input[pos]) + | (u32::from(input[pos + 1]) << 8) + | (u32::from(input[pos + 2]) << 16); + (value.wrapping_mul(0x9E37_79B1) >> (32 - LONG_LZ_HASH_BITS)) as usize + } + + /// Candidate positions strictly before `pos` sharing its 3-byte hash, + /// newest first. The caller must ensure 3 bytes are readable at `pos`. + fn candidates_before(&self, input: &[u8], pos: usize) -> impl Iterator + '_ { + let bucket = &self.buckets[Self::hash(input, pos)]; + let end = bucket.partition_point(|&candidate| candidate < pos); + bucket[..end].iter().rev().copied() + } +} + +const MAX_LONG_MATCH_CANDIDATES: usize = 64; + +const MAX_LONG_LZ_DISTANCE: usize = 0x7fff; + +pub fn unpack15_encode(input: &[u8]) -> Result> { + unpack15_encode_with_options(input, EncodeOptions::default()) +} + +pub fn unpack15_encode_with_options(input: &[u8], options: EncodeOptions) -> Result> { + if input.is_empty() { + return Ok(Vec::new()); + } + + let mut encoder = Unpack15Encoder::with_options(options); + encoder.encode_member(input) +} + +pub(crate) fn unpack15_encode_with_options_and_progress( + input: &[u8], + options: EncodeOptions, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + if input.is_empty() { + return Ok(Vec::new()); + } + Unpack15Encoder::with_options(options).encode_member_with_progress(input, progress) +} + +pub struct Unpack15Encoder { + bits: BitWriter, + options: EncodeOptions, + // State names follow RAR13_FORMAT_SPECIFICATION.md §6 so the codec state + // lines up directly with the documented Unpack15 tables and traces. + ch_set: [u16; 256], + ch_set_c: [u16; 256], + ch_set_b: [u16; 256], + n_to_pl: [u8; 256], + n_to_pl_b: [u8; 256], + n_to_pl_c: [u8; 256], + ch_set_a: [u16; 256], + avr_plc: u32, + avr_plc_b: u32, + avr_ln1: u32, + avr_ln2: u32, + avr_ln3: u32, + max_dist3: u32, + nhfb: u32, + nlzb: u32, + num_huf: u32, + old_dist: [u32; 4], + old_dist_ptr: usize, + last_dist: u32, + last_length: u32, + l_count: u32, + #[cfg(test)] + stmode_literal_count: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EncodeOptions { + old_distance_tokens: bool, + lazy_matching: bool, + stmode_literal_runs: bool, + max_long_match_distance: usize, +} + +impl EncodeOptions { + pub const fn new() -> Self { + Self { + old_distance_tokens: true, + lazy_matching: true, + stmode_literal_runs: true, + max_long_match_distance: MAX_LONG_LZ_DISTANCE, + } + } + + pub const fn with_old_distance_tokens(mut self, enabled: bool) -> Self { + self.old_distance_tokens = enabled; + self + } + + pub const fn with_lazy_matching(mut self, enabled: bool) -> Self { + self.lazy_matching = enabled; + self + } + + pub const fn with_stmode_literal_runs(mut self, enabled: bool) -> Self { + self.stmode_literal_runs = enabled; + self + } + + pub const fn with_max_long_match_distance(mut self, distance: usize) -> Self { + self.max_long_match_distance = distance; + self + } + + pub const fn old_distance_tokens_enabled(self) -> bool { + self.old_distance_tokens + } + + #[cfg(test)] + pub(crate) const fn lazy_matching_enabled(self) -> bool { + self.lazy_matching + } +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self::new() + } +} + +impl Unpack15Encoder { + pub fn new() -> Self { + Self::with_options(EncodeOptions::default()) + } + + pub fn with_options(options: EncodeOptions) -> Self { + let mut encoder = Self { + bits: BitWriter::new(), + options, + ch_set: [0; 256], + ch_set_c: [0; 256], + ch_set_b: [0; 256], + n_to_pl: [0; 256], + n_to_pl_b: [0; 256], + n_to_pl_c: [0; 256], + ch_set_a: [0; 256], + avr_plc: 0x3500, + avr_plc_b: 0, + avr_ln1: 0, + avr_ln2: 0, + avr_ln3: 0, + max_dist3: 0x2001, + nhfb: 0x80, + nlzb: 0x80, + num_huf: 0, + old_dist: [u32::MAX; 4], + old_dist_ptr: 0, + last_dist: u32::MAX, + last_length: 0, + l_count: 0, + #[cfg(test)] + stmode_literal_count: 0, + }; + encoder.init_huff(); + encoder + } + + pub fn encode_literals_only(mut self, input: &[u8]) -> Result> { + Ok(self.encode_literals_only_member(input)) + } + + fn encode_literals_only_member(&mut self, input: &[u8]) -> Vec { + if input.is_empty() { + return Vec::new(); + } + self.bits = BitWriter::new(); + let mut pos = 0usize; + while pos < input.len() { + let mut flags = 0u8; + let mut flag_bits = 0usize; + let mut payloads = Vec::new(); + let mut plan_nhfb = self.nhfb; + let mut plan_nlzb = self.nlzb; + let mut plan_num_huf = self.num_huf; + let mut group_enters_stmode = false; + + while flag_bits < 8 && pos < input.len() { + // Literal-only planning can start with two-bit flags after a + // match-heavy solid member. Huffman updates only move that + // choice toward one-bit flags; the switch occurs after an + // even number of bits, so a flag cannot straddle the byte. + let flag = huff_flag_bits(plan_nlzb <= plan_nhfb); + write_planned_flag_bits(&mut flags, flag_bits, flag); + payloads.push(EncodedToken::Literal(input[pos])); + flag_bits += flag.len(); + if flag_bits == 8 && plan_num_huf >= 16 && pos + 1 < input.len() { + group_enters_stmode = true; + } + plan_num_huf += 1; + pos += 1; + plan_huff_effect(&mut plan_nhfb, &mut plan_nlzb); + } + + self.emit_flags_byte(flags); + self.emit_payloads(payloads); + if group_enters_stmode { + if self.options.stmode_literal_runs { + self.emit_stmode_literal_run(input, None, &mut pos); + } + self.emit_stmode_exit(); + } + } + std::mem::take(&mut self.bits).finish() + } + + pub fn encode_member(&mut self, input: &[u8]) -> Result> { + self.encode_member_inner(input, None) + } + + #[cfg(any(test, feature = "write"))] + pub(crate) fn encode_member_with_progress( + &mut self, + input: &[u8], + progress: &mut dyn FnMut(usize) -> bool, + ) -> Result> { + self.encode_member_inner(input, Some(progress)) + } + + fn encode_member_inner( + &mut self, + input: &[u8], + mut progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + if input.is_empty() { + return Ok(Vec::new()); + } + self.bits = BitWriter::new(); + // Everything the decoder drops at a member boundary has to be dropped + // here too. The adaptive tables carry across a solid run, but the + // short-LZ literal run does not: `Unpack15::init_member` clears it for + // every member, solid or not. Leaving it set here made the encoder + // write a break bit at the start of the next member that no decoder + // was going to read, and one stray bit desynchronises the rest of the + // archive. + self.l_count = 0; + let buckets = long_lz_buckets(input); + let mut pos = 0usize; + let mut next_report = 0usize; + let mut straddle: Option = None; + while pos < input.len() || straddle.is_some() { + let mut flags = 0u8; + let mut flag_bits = 0usize; + let mut payloads = Vec::new(); + let mut plan_encoder = self.clone_for_planning(); + let mut group_enters_stmode = false; + + if let Some(carried) = straddle.take() { + write_planned_flag_bits(&mut flags, 0, carried.rest); + flag_bits = carried.rest.len(); + payloads.push(carried.token); + plan_encoder.emit_payloads(vec![carried.token]); + } + + while flag_bits < 8 && pos < input.len() { + let state = plan_encoder.lz_plan_state(); + if let Some(token) = plan_encoder + .choose_lz_token(input, pos, &buckets, state) + .filter(|token| { + !self.options.lazy_matching + || !should_lazy_emit_literal( + input, + pos, + &buckets, + *token, + state.max_dist3, + self.options, + ) + }) + { + let flag = token.flag_bits(state.nlzb, state.nhfb); + let next_pos = pos + token.length() as usize; + if flag_fits(flag_bits, flag) { + write_planned_flag_bits(&mut flags, flag_bits, flag); + flag_bits += flag.len(); + pos = next_pos; + plan_encoder.emit_payloads(vec![token.into()]); + payloads.push(token.into()); + continue; + } + } + + let flag = huff_flag_bits(plan_encoder.nlzb <= plan_encoder.nhfb); + if flag_bits + flag.len() > 8 { + straddle = Some(split_flag( + &mut flags, + flag_bits, + flag, + EncodedToken::Literal(input[pos]), + )); + pos += 1; + break; + } + write_planned_flag_bits(&mut flags, flag_bits, flag); + let literal = input[pos]; + payloads.push(EncodedToken::Literal(input[pos])); + flag_bits += flag.len(); + if flag_bits == 8 && plan_encoder.num_huf >= 16 && pos + 1 < input.len() { + group_enters_stmode = true; + } + pos += 1; + plan_encoder.emit_literal(literal); + } + + self.emit_flags_byte(flags); + self.emit_payloads(payloads); + if group_enters_stmode { + if self.options.stmode_literal_runs { + self.emit_stmode_literal_run(input, Some(&buckets), &mut pos); + } + self.emit_stmode_exit(); + } + if pos >= next_report { + if progress.as_deref_mut().is_some_and(|report| !report(pos)) { + return Err(Error::Cancelled); + } + next_report = pos.saturating_add(1024 * 1024); + } + } + if progress.is_some_and(|report| !report(input.len())) { + return Err(Error::Cancelled); + } + Ok(std::mem::take(&mut self.bits).finish()) + } + + fn clone_for_planning(&self) -> Self { + Self { + bits: BitWriter::new(), + options: self.options, + ch_set: self.ch_set, + ch_set_c: self.ch_set_c, + ch_set_b: self.ch_set_b, + n_to_pl: self.n_to_pl, + n_to_pl_b: self.n_to_pl_b, + n_to_pl_c: self.n_to_pl_c, + ch_set_a: self.ch_set_a, + avr_plc: self.avr_plc, + avr_plc_b: self.avr_plc_b, + avr_ln1: self.avr_ln1, + avr_ln2: self.avr_ln2, + avr_ln3: self.avr_ln3, + max_dist3: self.max_dist3, + nhfb: self.nhfb, + nlzb: self.nlzb, + num_huf: self.num_huf, + old_dist: self.old_dist, + old_dist_ptr: self.old_dist_ptr, + last_dist: self.last_dist, + last_length: self.last_length, + l_count: self.l_count, + #[cfg(test)] + stmode_literal_count: self.stmode_literal_count, + } + } + + fn lz_plan_state(&self) -> LzPlanState { + LzPlanState { + last_dist: self.last_dist, + last_length: self.last_length, + old_dist: self.old_dist, + old_dist_ptr: self.old_dist_ptr, + max_dist3: self.max_dist3, + nlzb: self.nlzb, + nhfb: self.nhfb, + l_count: self.l_count, + } + } + + fn choose_lz_token( + &self, + input: &[u8], + pos: usize, + buckets: &Rar13MatchFinder, + state: LzPlanState, + ) -> Option { + let candidates = find_lz_tokens(input, pos, buckets, state, self.options); + candidates + .into_iter() + .filter_map(|token| self.token_bit_cost(token, state).map(|cost| (token, cost))) + .min_by(|(left, left_cost), (right, right_cost)| { + let left_score = left_cost * 256 / left.length() as usize; + let right_score = right_cost * 256 / right.length() as usize; + left_score + .cmp(&right_score) + .then_with(|| right.length().cmp(&left.length())) + }) + .map(|(token, _)| token) + } + + fn token_bit_cost(&self, token: MatchToken, state: LzPlanState) -> Option { + let flag_cost = token.flag_bits(state.nlzb, state.nhfb).len(); + match token { + MatchToken::RepeatLast(_) => Some(flag_cost + self.repeat_last_bit_cost(state.l_count)), + MatchToken::ShortLz(token) => { + let distance_value = token.distance - 1; + let distance_place = self + .ch_set_a + .iter() + .position(|&value| value as u32 == distance_value) + .expect("short-distance alphabet contains every distance"); + Some( + flag_cost + + l_count_break_bit_cost(state.l_count) + + self.short_lz_prefix_bit_cost(token.length - 2) + + decode_num_bit_cost(distance_place as u32, 5, DEC_HF2, POS_HF2)?, + ) + } + MatchToken::OldDist(token) => { + let length_code = old_dist_lz_length_code( + token.length, + token.distance, + state.max_dist3, + token.short_code, + ) + .expect("old-distance candidate passed both distance thresholds"); + Some( + flag_cost + + l_count_break_bit_cost(state.l_count) + + self.short_lz_prefix_bit_cost(token.short_code) + + decode_num_bit_cost(length_code, 2, DEC_L1, POS_L1)?, + ) + } + MatchToken::LongLz(token) => { + let length_code = long_lz_length_code_for_distance(token, state.max_dist3) + .expect("long-match candidate passed distance threshold validation"); + let distance_place = self.long_lz_distance_place(token.distance); + Some( + flag_cost + + self.long_lz_length_bit_cost(length_code) + + self.long_lz_distance_bit_cost(distance_place) + + 7, + ) + } + } + } + + fn emit_payloads(&mut self, payloads: Vec) { + for payload in payloads { + match payload { + EncodedToken::Literal(byte) => self.emit_literal(byte), + EncodedToken::ShortLz(short_lz) => { + self.emit_short_lz(short_lz); + } + EncodedToken::RepeatLast(repeat) => { + self.emit_repeat_last(repeat); + } + EncodedToken::OldDist(old_lz) => { + self.emit_old_dist_lz(old_lz); + } + EncodedToken::LongLz(long_lz) => { + self.emit_long_lz(long_lz); + } + } + } + } + + fn emit_flags_byte(&mut self, flags: u8) { + let flags_place = self + .ch_set_c + .iter() + .position(|&value| (value >> 8) as u8 == flags) + .expect("flag alphabet contains every byte"); + emit_decode_num(&mut self.bits, flags_place as u32, 5, DEC_HF2, POS_HF2); + + let mut cur_flags; + let mut new_flags_place; + loop { + cur_flags = self.ch_set_c[flags_place] as u32; + new_flags_place = self.n_to_pl_c[(cur_flags & 0xff) as usize] as usize; + self.n_to_pl_c[(cur_flags & 0xff) as usize] = + self.n_to_pl_c[(cur_flags & 0xff) as usize].wrapping_add(1); + cur_flags += 1; + if cur_flags & 0xff == 0 { + corr_huff(&mut self.ch_set_c, &mut self.n_to_pl_c); + } else { + break; + } + } + + self.ch_set_c[flags_place] = self.ch_set_c[new_flags_place]; + self.ch_set_c[new_flags_place] = cur_flags as u16; + } + + fn emit_literal(&mut self, byte: u8) { + let byte_place = self + .ch_set + .iter() + .position(|&value| (value >> 8) as u8 == byte) + .expect("literal alphabet contains every byte"); + self.emit_literal_place(byte_place, byte_place, true) + } + + fn emit_stmode_literal(&mut self, byte: u8) { + let byte_place = self + .ch_set + .iter() + .position(|&value| (value >> 8) as u8 == byte) + .expect("literal alphabet contains every byte"); + #[cfg(test)] + { + self.stmode_literal_count += 1; + } + self.emit_literal_place(byte_place + 1, byte_place, false) + } + + fn emit_stmode_literal_run( + &mut self, + input: &[u8], + buckets: Option<&Rar13MatchFinder>, + pos: &mut usize, + ) { + while *pos + 1 < input.len() { + if buckets + .and_then(|buckets| { + find_lz_token( + input, + *pos, + buckets, + LzPlanState { + last_dist: self.last_dist, + last_length: self.last_length, + old_dist: self.old_dist, + old_dist_ptr: self.old_dist_ptr, + max_dist3: self.max_dist3, + nlzb: self.nlzb, + nhfb: self.nhfb, + l_count: self.l_count, + }, + self.options, + ) + }) + .is_some() + { + break; + } + self.emit_stmode_literal(input[*pos]); + *pos += 1; + } + } + + fn emit_literal_place( + &mut self, + encoded_place: usize, + decoded_place: usize, + update_num_huf: bool, + ) { + debug_assert!(encoded_place <= self.ch_set.len()); + debug_assert!(decoded_place < self.ch_set.len()); + + let (start_pos, dec_tab, pos_tab) = if self.avr_plc > 0x75ff { + (8, DEC_HF4, POS_HF4) + } else if self.avr_plc > 0x5dff { + (6, DEC_HF3, POS_HF3) + } else if self.avr_plc > 0x35ff { + (5, DEC_HF2, POS_HF2) + } else if self.avr_plc > 0x0dff { + (5, DEC_HF1, POS_HF1) + } else { + (4, DEC_HF0, POS_HF0) + }; + emit_decode_num( + &mut self.bits, + encoded_place as u32, + start_pos, + dec_tab, + pos_tab, + ); + + self.avr_plc += decoded_place as u32; + self.avr_plc -= self.avr_plc >> 8; + self.nhfb += 16; + if self.nhfb > 0xff { + self.nhfb = 0x90; + self.nlzb >>= 1; + } + if update_num_huf { + self.num_huf += 1; + } + + let idx = decoded_place; + let mut cur_byte; + let mut new_byte_place; + loop { + cur_byte = self.ch_set[idx] as u32; + new_byte_place = self.n_to_pl[(cur_byte & 0xff) as usize] as usize; + self.n_to_pl[(cur_byte & 0xff) as usize] = + self.n_to_pl[(cur_byte & 0xff) as usize].wrapping_add(1); + cur_byte += 1; + if cur_byte & 0xff > 0xa1 { + corr_huff(&mut self.ch_set, &mut self.n_to_pl); + } else { + break; + } + } + + self.ch_set[idx] = self.ch_set[new_byte_place]; + self.ch_set[new_byte_place] = cur_byte as u16; + } + + fn emit_short_lz(&mut self, short_lz: ShortLz) { + debug_assert!((1..=256).contains(&short_lz.distance)); + debug_assert!((2..=10).contains(&short_lz.length)); + self.num_huf = 0; + if self.l_count == 2 { + self.bits.write_bits(0, 1); + self.l_count = 0; + } + let length_place = short_lz.length - 2; + self.emit_short_lz_code(length_place as usize); + self.l_count = 0; + + self.avr_ln1 += length_place; + self.avr_ln1 -= self.avr_ln1 >> 4; + + let distance_value = short_lz.distance - 1; + let distance_place = self + .ch_set_a + .iter() + .position(|&value| value as u32 == distance_value) + .expect("short-distance alphabet contains every distance"); + emit_decode_num(&mut self.bits, distance_place as u32, 5, DEC_HF2, POS_HF2); + if distance_place > 0 { + let last_distance = self.ch_set_a[distance_place - 1]; + self.ch_set_a[distance_place] = last_distance; + self.ch_set_a[distance_place - 1] = distance_value as u16; + } + self.remember_match(short_lz.distance, short_lz.length); + } + + fn emit_repeat_last(&mut self, repeat: RepeatLastLz) { + debug_assert_eq!(self.last_dist, repeat.distance); + debug_assert_eq!(self.last_length, repeat.length); + self.num_huf = 0; + if self.l_count == 2 { + self.bits.write_bits(1, 1); + } else { + self.emit_short_lz_code(9); + self.l_count += 1; + } + } + + fn emit_old_dist_lz(&mut self, old_lz: OldDistLz) { + self.num_huf = 0; + if self.l_count == 2 { + self.bits.write_bits(0, 1); + self.l_count = 0; + } + self.emit_short_lz_code(old_lz.short_code as usize); + self.l_count = 0; + + let expected_distance = self.old_dist[(self + .old_dist_ptr + .wrapping_sub((old_lz.short_code - 9) as usize)) + & 3]; + debug_assert_eq!(expected_distance, old_lz.distance); + let length_code = old_dist_lz_length_code( + old_lz.length, + old_lz.distance, + self.max_dist3, + old_lz.short_code, + ) + .expect("selected old-distance length passed planner validation"); + emit_decode_num(&mut self.bits, length_code, 2, DEC_L1, POS_L1); + self.remember_match(old_lz.distance, old_lz.length); + } + + fn emit_short_lz_code(&mut self, code: usize) { + let (code_len, code_byte) = if self.avr_ln1 < 37 { + (self.short_len1(code), SHORT_XOR1[code]) + } else { + (self.short_len2(code), SHORT_XOR2[code]) + }; + self.bits + .write_bits((code_byte >> (8 - code_len)) as u32, code_len as usize); + } + + fn short_lz_prefix_bit_cost(&self, code: u32) -> usize { + debug_assert!(code < SHORT_XOR1.len() as u32); + let code = code as usize; + (if self.avr_ln1 < 37 { + self.short_len1(code) + } else { + self.short_len2(code) + }) as usize + } + + fn repeat_last_bit_cost(&self, l_count: u32) -> usize { + if l_count == 2 { + 1 + } else { + self.short_lz_prefix_bit_cost(9) + } + } + + fn emit_long_lz(&mut self, long_lz: LongLz) { + self.num_huf = 0; + self.nlzb += 16; + if self.nlzb > 0xff { + self.nlzb = 0x90; + self.nhfb >>= 1; + } + let old_avr2 = self.avr_ln2; + + let length_code = self + .long_lz_length_code(long_lz) + .expect("selected long-match length passed planner validation"); + emit_long_lz_length(&mut self.bits, self.avr_ln2, length_code); + self.avr_ln2 += length_code; + self.avr_ln2 -= self.avr_ln2 >> 5; + + let distance_place = self.long_lz_distance_place(long_lz.distance); + let (start_pos, dec_tab, pos_tab) = if self.avr_plc_b > 0x28ff { + (5, DEC_HF2, POS_HF2) + } else if self.avr_plc_b > 0x06ff { + (5, DEC_HF1, POS_HF1) + } else { + (4, DEC_HF0, POS_HF0) + }; + emit_decode_num( + &mut self.bits, + distance_place as u32, + start_pos, + dec_tab, + pos_tab, + ); + self.avr_plc_b += distance_place as u32; + self.avr_plc_b -= self.avr_plc_b >> 8; + + let idx = distance_place; + let mut distance; + let mut new_distance_place; + loop { + distance = self.ch_set_b[idx] as u32; + new_distance_place = self.n_to_pl_b[(distance & 0xff) as usize] as usize; + self.n_to_pl_b[(distance & 0xff) as usize] = + self.n_to_pl_b[(distance & 0xff) as usize].wrapping_add(1); + distance += 1; + if distance & 0xff == 0 { + corr_huff(&mut self.ch_set_b, &mut self.n_to_pl_b); + } else { + break; + } + } + + self.ch_set_b[idx] = self.ch_set_b[new_distance_place]; + self.ch_set_b[new_distance_place] = distance as u16; + + let low_byte = ((long_lz.distance << 1) & 0xff) as u8; + self.bits.write_bits((low_byte >> 1) as u32, 7); + + let old_avr3 = self.avr_ln3; + if length_code != 1 && length_code != 4 { + if length_code == 0 && long_lz.distance <= self.max_dist3 { + self.avr_ln3 += 1; + self.avr_ln3 -= self.avr_ln3 >> 8; + } else if self.avr_ln3 > 0 { + self.avr_ln3 -= 1; + } + } + if old_avr3 > 0xb0 || (self.avr_plc >= 0x2a00 && old_avr2 < 0x40) { + self.max_dist3 = 0x7f00; + } else { + self.max_dist3 = 0x2001; + } + + self.remember_match(long_lz.distance, long_lz.length); + } + + fn long_lz_length_code(&self, long_lz: LongLz) -> Option { + long_lz_length_code_for_distance(long_lz, self.max_dist3) + } + + fn long_lz_distance_place(&self, target_distance: u32) -> usize { + let wanted_high = ((target_distance << 1) & 0xff00) as u16; + self.ch_set_b + .iter() + .position(|&value| value & 0xff00 == wanted_high) + .expect("long-distance alphabet contains every high byte") + } + + fn long_lz_length_bit_cost(&self, length_code: u32) -> usize { + debug_assert!(length_code < 0x100); + if self.avr_ln2 >= 122 { + decode_num_bit_cost(length_code, 3, DEC_L2, POS_L2) + .expect("L2 represents every length symbol") + } else if self.avr_ln2 >= 64 { + decode_num_bit_cost(length_code, 2, DEC_L1, POS_L1) + .expect("L1 represents every length symbol") + } else if length_code <= 7 { + length_code as usize + 1 + } else { + 16 + } + } + + fn long_lz_distance_bit_cost(&self, distance_place: usize) -> usize { + debug_assert!(distance_place < 256); + let cost = if self.avr_plc_b > 0x28ff { + decode_num_bit_cost(distance_place as u32, 5, DEC_HF2, POS_HF2) + } else if self.avr_plc_b > 0x06ff { + decode_num_bit_cost(distance_place as u32, 5, DEC_HF1, POS_HF1) + } else { + decode_num_bit_cost(distance_place as u32, 4, DEC_HF0, POS_HF0) + }; + cost.expect("distance codebooks represent every adaptive rank") + } + + fn emit_stmode_exit(&mut self) { + let (start_pos, dec_tab, pos_tab) = if self.avr_plc > 0x75ff { + (8, DEC_HF4, POS_HF4) + } else if self.avr_plc > 0x5dff { + (6, DEC_HF3, POS_HF3) + } else if self.avr_plc > 0x35ff { + (5, DEC_HF2, POS_HF2) + } else if self.avr_plc > 0x0dff { + (5, DEC_HF1, POS_HF1) + } else { + (4, DEC_HF0, POS_HF0) + }; + emit_decode_num(&mut self.bits, 0, start_pos, dec_tab, pos_tab); + self.bits.write_bits(1, 1); + self.num_huf = 0; + } + + fn init_huff(&mut self) { + for i in 0..256 { + self.ch_set[i] = (i as u16) << 8; + self.ch_set_c[i] = (0u8.wrapping_sub(i as u8) as u16) << 8; + self.ch_set_b[i] = (i as u16) << 8; + } + self.n_to_pl = [0; 256]; + self.n_to_pl_b = [0; 256]; + self.n_to_pl_c = [0; 256]; + for i in 0..256 { + self.ch_set_a[i] = i as u16; + } + corr_huff(&mut self.ch_set_b, &mut self.n_to_pl_b); + } + + fn remember_match(&mut self, distance: u32, length: u32) { + self.old_dist[self.old_dist_ptr] = distance; + self.old_dist_ptr = (self.old_dist_ptr + 1) & 3; + self.last_length = length; + self.last_dist = distance; + } + + fn short_len1(&self, pos: usize) -> u8 { + if pos == 1 { + 3 + } else { + SHORT_LEN1[pos] + } + } + + fn short_len2(&self, pos: usize) -> u8 { + if pos == 3 { + 3 + } else { + SHORT_LEN2[pos] + } + } +} + +impl Default for Unpack15Encoder { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, Copy)] +struct LzPlanState { + last_dist: u32, + last_length: u32, + old_dist: [u32; 4], + old_dist_ptr: usize, + max_dist3: u32, + nlzb: u32, + nhfb: u32, + l_count: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EncodedToken { + Literal(u8), + ShortLz(ShortLz), + RepeatLast(RepeatLastLz), + OldDist(OldDistLz), + LongLz(LongLz), +} + +/// Candidates from the match finder; literals take a separate planning path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MatchToken { + ShortLz(ShortLz), + RepeatLast(RepeatLastLz), + OldDist(OldDistLz), + LongLz(LongLz), +} + +impl MatchToken { + fn length(self) -> u32 { + match self { + Self::ShortLz(token) => token.length, + Self::RepeatLast(token) => token.length, + Self::OldDist(token) => token.length, + Self::LongLz(token) => token.length, + } + } + + fn flag_bits(self, nlzb: u32, nhfb: u32) -> &'static [bool] { + match self { + Self::LongLz(_) => long_lz_flag_bits(nlzb > nhfb), + Self::ShortLz(_) | Self::RepeatLast(_) | Self::OldDist(_) => &[false, false], + } + } +} + +impl From for EncodedToken { + fn from(token: MatchToken) -> Self { + match token { + MatchToken::ShortLz(value) => Self::ShortLz(value), + MatchToken::RepeatLast(value) => Self::RepeatLast(value), + MatchToken::OldDist(value) => Self::OldDist(value), + MatchToken::LongLz(value) => Self::LongLz(value), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ShortLz { + pub distance: u32, + pub length: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RepeatLastLz { + pub distance: u32, + pub length: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct OldDistLz { + pub distance: u32, + pub length: u32, + pub short_code: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LongLz { + pub distance: u32, + pub length: u32, +} + +fn huff_flag_bits(prefer_huff_on_one: bool) -> &'static [bool] { + if prefer_huff_on_one { + &[true] + } else { + &[false, true] + } +} + +fn long_lz_flag_bits(prefer_long_lz_on_one: bool) -> &'static [bool] { + if prefer_long_lz_on_one { + &[true] + } else { + &[false, true] + } +} + +/// A token whose flag did not fit in what was left of a flags byte. +/// +/// `Unpack15` fetches the next flags byte the moment it runs out of bits, and +/// it does that in the middle of reading a flag, not between tokens. So a +/// two-bit flag at the last bit of a byte is legal: its first bit closes that +/// byte and its second opens the next one, and only then does the decoder read +/// the token's payload. Padding the byte out instead leaves a bit the decoder +/// still reads as a flag, which desynchronises everything after it. +#[derive(Clone, Copy)] +struct Straddle { + token: EncodedToken, + /// The flag bits that belong at the front of the next flags byte. + rest: &'static [bool], +} + +fn split_flag( + flags: &mut u8, + flag_bits: usize, + flag: &'static [bool], + token: EncodedToken, +) -> Straddle { + let fits = 8 - flag_bits; + write_planned_flag_bits(flags, flag_bits, &flag[..fits]); + Straddle { + token, + rest: &flag[fits..], + } +} + +fn write_planned_flag_bits(flags: &mut u8, start: usize, bits: &[bool]) { + for (offset, &bit) in bits.iter().enumerate() { + if bit { + *flags |= 1 << (7 - start - offset); + } + } +} + +fn flag_fits(used: usize, flag: &[bool]) -> bool { + used + flag.len() <= 8 +} + +fn plan_huff_effect(nhfb: &mut u32, nlzb: &mut u32) { + *nhfb += 16; + if *nhfb > 0xff { + *nhfb = 0x90; + *nlzb >>= 1; + } +} + +fn l_count_break_bit_cost(l_count: u32) -> usize { + usize::from(l_count == 2) +} + +fn find_lz_token( + input: &[u8], + pos: usize, + buckets: &Rar13MatchFinder, + state: LzPlanState, + options: EncodeOptions, +) -> Option { + find_lz_tokens(input, pos, buckets, state, options) + .into_iter() + .next() +} + +fn find_lz_tokens( + input: &[u8], + pos: usize, + buckets: &Rar13MatchFinder, + state: LzPlanState, + options: EncodeOptions, +) -> Vec { + let mut tokens = Vec::with_capacity(4); + if let Some(repeat) = find_repeat_last_lz(input, pos, state.last_dist, state.last_length) { + tokens.push(MatchToken::RepeatLast(repeat)); + } + if options.old_distance_tokens { + if let Some(old_lz) = find_old_dist_lz( + input, + pos, + state.old_dist, + state.old_dist_ptr, + state.max_dist3, + ) { + tokens.push(MatchToken::OldDist(old_lz)); + } + } + if let Some(short_lz) = find_short_lz(input, pos) { + tokens.push(MatchToken::ShortLz(short_lz)); + } + if let Some(long_lz) = find_long_lz_with_buckets( + input, + pos, + options.max_long_match_distance, + buckets, + MAX_LONG_MATCH_CANDIDATES, + ) + .filter(|long_lz| long_lz_length_code_for_distance(*long_lz, state.max_dist3).is_some()) + { + tokens.push(MatchToken::LongLz(long_lz)); + } + tokens +} + +fn should_lazy_emit_literal( + input: &[u8], + pos: usize, + buckets: &Rar13MatchFinder, + current: MatchToken, + max_dist3: u32, + options: EncodeOptions, +) -> bool { + if !matches!(current, MatchToken::ShortLz(_) | MatchToken::LongLz(_)) { + return false; + } + + // Selected matches fit at least two bytes of the remaining input. + debug_assert!(pos + 1 < input.len()); + + let next = find_lz_token( + input, + pos + 1, + buckets, + LzPlanState { + last_dist: u32::MAX, + last_length: 0, + old_dist: [u32::MAX; 4], + old_dist_ptr: 0, + max_dist3, + nlzb: 0, + nhfb: 0, + l_count: 0, + }, + options, + ); + next.is_some_and(|next| { + matches!(next, MatchToken::ShortLz(_) | MatchToken::LongLz(_)) + && next.length() >= current.length() + 2 + }) +} + +fn find_short_lz(input: &[u8], pos: usize) -> Option { + if pos == 0 { + return None; + } + + let max_distance = pos.min(256); + let mut best = ShortLz { + distance: 0, + length: 0, + }; + for distance in 1..=max_distance { + let mut length = 0usize; + while length < 10 + && pos + length < input.len() + && input[pos + length] == input[pos + length - distance] + { + length += 1; + } + // Distances increase, so the first match of a given length is nearest. + if length >= 2 && length > best.length as usize { + best = ShortLz { + distance: distance as u32, + length: length as u32, + }; + } + } + + (best.length >= 2).then_some(best) +} + +fn find_repeat_last_lz( + input: &[u8], + pos: usize, + last_dist: u32, + last_length: u32, +) -> Option { + if last_dist == u32::MAX { + return None; + } + debug_assert!((1..=MAX_LONG_LZ_DISTANCE as u32).contains(&last_dist)); + debug_assert!((2..=258).contains(&last_length)); + debug_assert!(pos <= input.len()); + let distance = last_dist as usize; + let length = last_length as usize; + if distance > pos || pos + length > input.len() { + return None; + } + let matches = (0..length).all(|offset| input[pos + offset] == input[pos + offset - distance]); + matches.then_some(RepeatLastLz { + distance: last_dist, + length: last_length, + }) +} + +fn find_old_dist_lz( + input: &[u8], + pos: usize, + old_dist: [u32; 4], + old_dist_ptr: usize, + _max_dist3: u32, +) -> Option { + let mut best = OldDistLz { + distance: 0, + length: 0, + short_code: 0, + }; + for short_code in 10..=13 { + let distance = old_dist[(old_dist_ptr.wrapping_sub((short_code - 9) as usize)) & 3]; + if distance == u32::MAX { + continue; + } + debug_assert!((1..=MAX_LONG_LZ_DISTANCE as u32).contains(&distance)); + let distance_usize = distance as usize; + if distance_usize > pos { + continue; + } + let mut length = 0usize; + while length < 258 + && pos + length < input.len() + && input[pos + length] == input[pos + length - distance_usize] + { + length += 1; + } + if length >= 3 + && old_dist_lz_is_encodable(length as u32, distance, short_code) + && length > best.length as usize + { + best = OldDistLz { + distance, + length: length as u32, + short_code, + }; + } + } + + (best.length >= 3).then_some(best) +} + +fn old_dist_lz_is_encodable(length: u32, distance: u32, short_code: u32) -> bool { + old_dist_lz_length_code(length, distance, 0x2001, short_code).is_some() + && old_dist_lz_length_code(length, distance, 0x7f00, short_code).is_some() +} + +fn old_dist_lz_length_code( + length: u32, + distance: u32, + max_dist3: u32, + _short_code: u32, +) -> Option { + let decoded_bonus = u32::from(distance > 256) + u32::from(distance >= max_dist3); + let length_code = length.checked_sub(2 + decoded_bonus)?; + // DOS RAR 1.402 does not reliably decode an old-distance match whose + // length symbol reaches the all-ones value. Code 10 also reserves that + // value for the Buf60 toggle, but the compatibility limit applies to all + // four old-distance codes. + if length_code == 0xff { + return None; + } + Some(length_code) +} + +fn long_lz_length_code_for_distance(long_lz: LongLz, max_dist3: u32) -> Option { + let decoded_bonus = + u32::from(long_lz.distance >= max_dist3) + if long_lz.distance <= 256 { 8 } else { 0 }; + long_lz.length.checked_sub(3 + decoded_bonus) +} + +pub fn find_long_lz(input: &[u8], pos: usize, max_match_distance: usize) -> Option { + if pos == 0 { + return None; + } + + let max_distance = pos.min(MAX_LONG_LZ_DISTANCE).min(max_match_distance); + if max_distance == 0 { + return None; + } + let mut best = LongLz { + distance: 0, + length: 0, + }; + for distance in 1..=max_distance { + let length = super::super::fast::match_length(input, pos, distance, 258); + // LongLZ adds eight to the decoded length for distances up to 256, + // so its shortest encodable near-distance match is eleven bytes. + let min_length = if distance <= 256 { 11 } else { 3 }; + if length >= min_length && length > best.length as usize { + best = LongLz { + distance: distance as u32, + length: length as u32, + }; + } + } + + (best.length >= 3).then_some(best) +} + +fn find_long_lz_with_buckets( + input: &[u8], + pos: usize, + max_match_distance: usize, + buckets: &Rar13MatchFinder, + max_candidates: usize, +) -> Option { + if pos == 0 || pos + 2 >= input.len() { + return None; + } + + let max_distance = pos.min(MAX_LONG_LZ_DISTANCE).min(max_match_distance); + if max_distance == 0 { + return None; + } + let max_length = (input.len() - pos).min(258); + let mut best = LongLz { + distance: 0, + length: 0, + }; + let mut checked = 0usize; + for candidate in buckets.candidates_before(input, pos) { + let distance = pos - candidate; + if distance > max_distance { + break; + } + // Near-distance candidates are a separate, bounded set and must not + // consume the established search budget for older matches. + if distance > 256 { + if checked >= max_candidates { + break; + } + checked += 1; + } + // A candidate can only improve on the current best when it matches at + // least one byte past the best length, so probe that byte first. The + // loop breaks once best reaches `max_length`, so the probe index stays + // in bounds. + if best.length == 0 + || input[candidate + best.length as usize] == input[pos + best.length as usize] + { + let length = super::super::fast::match_length(input, pos, distance, max_length); + let min_length = if distance <= 256 { 11 } else { 3 }; + // Candidates arrive nearest first, so equal lengths cannot win. + if length >= min_length && length > best.length as usize { + best = LongLz { + distance: distance as u32, + length: length as u32, + }; + if length == max_length { + break; + } + } + } + } + + (best.length >= 3).then_some(best) +} + +fn long_lz_buckets(input: &[u8]) -> Rar13MatchFinder { + Rar13MatchFinder::build(input) +} + +fn emit_long_lz_length(bits: &mut BitWriter, avr_ln2: u32, length_code: u32) { + debug_assert!(length_code < 0x100); + if avr_ln2 >= 122 { + emit_decode_num(bits, length_code, 3, DEC_L2, POS_L2); + } else if avr_ln2 >= 64 { + emit_decode_num(bits, length_code, 2, DEC_L1, POS_L1); + } else if length_code <= 7 { + bits.write_bits(1, length_code as usize + 1); + } else { + bits.write_bits(length_code, 16); + } +} + +fn emit_decode_num( + bits: &mut BitWriter, + target: u32, + start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], +) { + let (code, len) = encode_decode_num_prefix(target, start_pos, dec_tab, pos_tab) + .expect("emitted symbol passed fixed codebook validation"); + bits.write_bits(code, len); +} + +fn decode_num_bit_cost( + target: u32, + start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], +) -> Option { + encode_decode_num_prefix(target, start_pos, dec_tab, pos_tab).map(|(_, len)| len) +} + +fn encode_decode_num_prefix( + target: u32, + start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], +) -> Option<(u32, usize)> { + // Fixed table/start pairs cover every visited index and have positive thresholds. + let end = 16.min(pos_tab.len().saturating_sub(1)); + for (len, &base) in pos_tab + .iter() + .enumerate() + .take(end + 1) + .skip(start_pos as usize) + { + let dec_index = len - start_pos as usize; + let upper = u32::from(dec_tab[dec_index]); + let previous = if dec_index == 0 { + 0 + } else { + u32::from(dec_tab[dec_index - 1]) + }; + let max_num = (upper - 1) & !0xf; + if max_num < previous { + continue; + } + let base = u32::from(base); + let max_target = ((max_num - previous) >> (16 - len)) + base; + if target <= max_target { + // Nonempty fixed-table intervals are contiguous, so earlier ones + // already handle every target below this base. + debug_assert!(target >= base); + let num = previous + ((target - base) << (16 - len)); + return Some((num >> (16 - len), len)); + } + } + None +} + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; + +#[derive(Default)] +struct BitWriter { + output: Vec, + bit_pos: usize, +} + +impl BitWriter { + fn new() -> Self { + Self { + output: Vec::new(), + bit_pos: 0, + } + } + + fn write_bits(&mut self, value: u32, count: usize) { + super::super::fast::write_msb_bits( + &mut self.output, + &mut self.bit_pos, + u64::from(value), + count, + ); + } + + fn finish(self) -> Vec { + self.output + } +} + +#[cfg(test)] +#[path = "solid_regressions.rs"] +mod solid_regressions; diff --git a/arca-rar/vendor/rars/src/codec/rar13/solid_regressions.rs b/arca-rar/vendor/rars/src/codec/rar13/solid_regressions.rs new file mode 100644 index 00000000..2eabdad0 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar13/solid_regressions.rs @@ -0,0 +1,307 @@ +use super::*; +type Unpack15 = Reader15State; +type BitReader = ReaderBits; + +/// The options the RAR 1.3 writer uses at its default level. +fn rar13_options() -> EncodeOptions { + EncodeOptions::new() + .with_old_distance_tokens(false) + .with_lazy_matching(false) +} + +fn encode_solid_run(members: &[&[u8]], options: EncodeOptions) -> Vec> { + let mut encoder = Unpack15Encoder::with_options(options); + members + .iter() + .map(|member| encoder.encode_member(member).unwrap()) + .collect() +} + +fn decode_solid_run(packed: &[Vec], members: &[&[u8]]) -> Vec> { + let mut decoder = Unpack15::new(); + packed + .iter() + .zip(members) + .map(|(packed, member)| { + decoder + .decode_member(packed, member.len(), true) + .unwrap() + .into_vec() + }) + .collect() +} + +/// The short-LZ literal run does not survive a member boundary: the decoder +/// clears it for every member, solid or not. The encoder kept it, so where +/// a member happened to end mid-run the next one opened with a break bit +/// nothing would read, and every symbol after it was a bit out of step. +/// +/// This pair is the smallest one that ends a member with the run at two. +#[test] +fn a_member_boundary_clears_the_short_lz_literal_run() { + let first: Vec = b"abcdefgh".iter().cycle().take(128).copied().collect(); + let second = vec![0u8; 16]; + let members: Vec<&[u8]> = vec![&first, &second]; + + let packed = encode_solid_run(&members, rar13_options()); + let decoded = decode_solid_run(&packed, &members); + + assert_eq!(decoded[0], first); + assert_eq!( + decoded[1], second, + "the second member decoded a bit out of step" + ); +} + +/// A decoder handed a solid member first skips the non-solid reset, so +/// whatever it was constructed with is what decodes the member. It used to +/// be constructed with the Huffman tables left at zero. +#[test] +fn a_first_member_marked_solid_decodes_against_real_tables() { + let member = b"the quick brown fox jumps over the lazy dog\n".repeat(40); + let members: Vec<&[u8]> = vec![&member]; + let packed = encode_solid_run(&members, rar13_options()); + + let mut fresh = Unpack15::new(); + let solid = fresh.decode_member(&packed[0], member.len(), true).unwrap(); + assert_eq!(solid, member); + + // And it agrees with the same member read as a fresh one. + let mut other = Unpack15::new(); + let plain = other + .decode_member(&packed[0], member.len(), false) + .unwrap(); + assert_eq!(plain, member); +} + +/// Several members in a row, with the shapes that move the adaptive state +/// around: a long run, incompressible bytes, a short member and an empty +/// one. +#[test] +fn a_long_solid_run_round_trips() { + let repetitive = b"solid chain payload ".repeat(500); + let counted: Vec = (0..30_000u32) + .map(|index| (index * 7 % 251) as u8) + .collect(); + let short = b"tail".to_vec(); + let empty = Vec::new(); + let members: Vec<&[u8]> = vec![&repetitive, &counted, &short, &empty, &repetitive]; + + for options in [ + rar13_options(), + EncodeOptions::new().with_lazy_matching(false), + ] { + let packed = encode_solid_run(&members, options); + let decoded = decode_solid_run(&packed, &members); + for (index, (got, want)) in decoded.iter().zip(&members).enumerate() { + assert_eq!(got, want, "member {index} did not survive the solid run"); + } + } +} + +#[test] +fn final_encoder_progress_report_can_cancel() { + let mut encoder = Unpack15Encoder::new(); + let mut reports = Vec::new(); + let error = encoder + .encode_member_with_progress(b"nonempty", &mut |position| { + reports.push(position); + reports.len() == 1 + }) + .unwrap_err(); + assert_eq!(error, Error::Cancelled); + assert_eq!(reports, [8, 8]); +} + +#[test] +fn literal_only_encoder_exits_stmode_without_literal_runs() { + let input: Vec<_> = (0..128).map(|index| (index * 73 + 19) as u8).collect(); + let mut encoder = + Unpack15Encoder::with_options(EncodeOptions::new().with_stmode_literal_runs(false)); + let packed = encoder.encode_literals_only_member(&input); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn default_encoder_round_trips_low_rank_literal_history() { + let input = vec![b'x'; 4096]; + let mut encoder = Unpack15Encoder::default(); + let packed = encoder.encode_literals_only_member(&input); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); + assert!(encoder.avr_plc <= 0x0dff); +} + +#[test] +fn long_match_search_rejects_zero_history_or_distance_budget() { + let input = b"repeated repeated"; + assert_eq!(find_long_lz(input, 0, 16), None); + assert_eq!(find_long_lz(input, 9, 0), None); + let buckets = long_lz_buckets(input); + assert_eq!(find_long_lz_with_buckets(input, 9, 0, &buckets, 8), None); +} + +#[test] +fn equal_length_match_candidates_keep_nearest_distance() { + assert_eq!( + super::find_long_lz(&[0; 24], 11, 11), + Some(super::LongLz { + distance: 1, + length: 13 + }) + ); + let short = b"abcXabcYabcZ"; + assert_eq!( + find_short_lz(short, 8), + Some(ShortLz { + distance: 4, + length: 3, + }) + ); + + let prefix = b"abcdefghijkl"; + let mut long = Vec::new(); + for suffix in *b"XYZ" { + long.extend_from_slice(prefix); + long.push(suffix); + } + let buckets = long_lz_buckets(&long); + assert_eq!( + find_long_lz_with_buckets(&long, 26, 0x7fff, &buckets, 64), + Some(LongLz { + distance: 13, + length: 12, + }) + ); +} + +#[test] +fn decoder_refuses_literal_past_declared_output_size() { + let mut decoder = Unpack15::new(); + decoder.target = 0; + assert_eq!( + decoder.put_byte(b'x', &mut Vec::new()), + Err(Error::InvalidData("RAR 1.3 literal exceeds output size")) + ); +} + +#[test] +fn number_prefix_search_rejects_unrepresentable_candidates() { + assert_eq!( + super::encode_decode_num_prefix(u32::MAX, 4, super::DEC_HF0, super::POS_HF0), + None + ); + assert_eq!( + super::decode_num_bit_cost(256, 2, super::DEC_L1, super::POS_L1), + None + ); + assert_eq!( + super::decode_num_bit_cost(256, 3, super::DEC_L2, super::POS_L2), + None + ); +} + +#[test] +fn decoder_reads_far_short_match_token() { + let mut encoder = Unpack15Encoder::new(); + encoder.emit_short_lz_code(10); + emit_decode_num(&mut encoder.bits, 0xff, 2, DEC_L1, POS_L1); + encoder.emit_short_lz_code(14); + emit_decode_num(&mut encoder.bits, 0, 3, DEC_L2, POS_L2); + encoder.bits.write_bits(0, 15); + + let mut decoder = Unpack15::new(); + decoder.bits = BitReader::new(&encoder.bits.finish()); + decoder.target = 0x8000 + 5; + decoder.output_written = 0x8000; + decoder.unp_ptr = 0x8000; + decoder.window[..5].copy_from_slice(b"abcde"); + let mut output = Vec::new(); + decoder.short_lz(&mut output).unwrap(); + assert_eq!(decoder.buf60, 1); + assert!(output.is_empty(), "Buf60 toggle emits no match"); + decoder.short_lz(&mut output).unwrap(); + assert_eq!(output, b"abcde"); + assert_eq!(decoder.token_stats.short_matches, 1); +} + +#[test] +fn short_lz_prefix_tables_cover_every_byte_with_each_buf60_state() { + // RAR13_FORMAT_SPECIFICATION.md §6.13: Buf60 changes one prefix in + // each table. Completeness must hold for both wire states. + for (lengths, prefixes, adjusted) in [ + (&super::SHORT_LEN1, &super::SHORT_XOR1, 1), + (&super::SHORT_LEN2, &super::SHORT_XOR2, 3), + ] { + for buf60 in 0..=1 { + for byte in 0..=u8::MAX { + assert!( + prefixes.iter().enumerate().any(|(index, &prefix)| { + let len = if index == adjusted { + buf60 + 3 + } else { + lengths[index] + }; + (byte ^ prefix) & (!(0xffu16 >> len) as u8) == 0 + }), + "byte {byte:#04x}, Buf60={buf60}, adjusted prefix={adjusted}" + ); + } + } + } +} + +#[test] +fn old_distance_all_ones_length_does_not_toggle_buf60_for_other_codes() { + // The historical decoder reserves length 257 only for code 10. + // Current encoders avoid this symbol, but old/crafted streams can + // carry it with the other old-distance codes (§6.13). + for code in 11..=13 { + let mut encoder = Unpack15Encoder::new(); + encoder.emit_short_lz_code(code); + emit_decode_num(&mut encoder.bits, 0xff, 2, DEC_L1, POS_L1); + let mut decoder = Unpack15::new(); + decoder.bits = BitReader::new(&encoder.bits.finish()); + decoder.target = 257; + decoder.unp_ptr = 1; + decoder.window[0] = b'a'; + decoder.old_dist[(0usize.wrapping_sub(code - 9)) & 3] = 1; + let mut output = Vec::new(); + decoder.short_lz(&mut output).unwrap(); + assert_eq!(output, vec![b'a'; 257]); + assert_eq!(decoder.buf60, 0); + } +} + +#[test] +fn zero_distance_after_window_wrap_retains_historical_zero_fill() { + let mut decoder = Unpack15::new(); + decoder.first_win_done = true; + decoder.target = 3; + let mut output = Vec::new(); + decoder.copy_string(0, 3, &mut output).unwrap(); + assert_eq!(output, [0; 3]); +} + +#[test] +fn decoder_reads_stmode_short_match_token() { + for length in [3, 4] { + let mut bits = BitWriter::new(); + emit_decode_num(&mut bits, 0, 5, DEC_HF1, POS_HF1); + bits.write_bits(0, 1); // ST-mode match rather than exit. + bits.write_bits(u32::from(length == 4), 1); // Three- or four-byte match. + emit_decode_num(&mut bits, 0, 5, DEC_HF2, POS_HF2); + bits.write_bits(1, 5); // Distance one. + + let mut decoder = Unpack15::new(); + decoder.bits = BitReader::new(&bits.finish()); + decoder.st_mode = true; + decoder.target = 1 + length; + decoder.output_written = 1; + decoder.unp_ptr = 1; + decoder.window[0] = b'A'; + let mut output = Vec::new(); + decoder.huff_decode(&mut output).unwrap(); + assert_eq!(output, vec![b'A'; length]); + assert_eq!(decoder.token_stats.st_matches, 1); + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar13/tests.rs b/arca-rar/vendor/rars/src/codec/rar13/tests.rs new file mode 100644 index 00000000..41f57364 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar13/tests.rs @@ -0,0 +1,1482 @@ +use crate::codec::Error; + +#[test] +fn normal_encoder_round_trips_terminal_literal_flag_groups() { + for length in [23, 24, 25] { + let input: Vec = (0..length).collect(); + let packed = super::unpack15_encode(&input).unwrap(); + assert_eq!(super::unpack15_decode(&packed, input.len()).unwrap(), input); + } +} + +#[test] +fn final_literal_flag_crosses_into_a_terminal_flags_group() { + let first = match_heavy_payload(176, 8000); + let mut encoder = + super::Unpack15Encoder::with_options(super::EncodeOptions::new().with_lazy_matching(false)); + let first_packed = encoder.encode_member(&first).unwrap(); + let mut decoder = super::Unpack15::new(); + assert_eq!( + decoder + .decode_member(&first_packed, first.len(), false) + .unwrap(), + first + ); + let input: Vec = [b'A'; 24].into_iter().chain(*b"XYZ").collect(); + let mut planned = encoder.clone_for_planning(); + planned.emit_literal(b'A'); + let buckets = long_lz_buckets(&input); + assert_eq!( + planned.choose_lz_token(&input, 1, &buckets, planned.lz_plan_state()), + Some(MatchToken::LongLz(LongLz { + distance: 1, + length: 23 + })) + ); + let packed = encoder.encode_member(&input).unwrap(); + let mut partial = decoder.clone(); + assert_eq!( + decoder.decode_member(&packed, input.len(), true).unwrap(), + input + ); + assert_eq!( + partial + .decode_member(&packed, input.len() - 1, true) + .unwrap(), + input[..input.len() - 1] + ); + assert_eq!( + partial.state.flags_cnt, 1, + "one flag bit remains before the terminal literal" + ); + partial.state.target = input.len(); + let mut tail = Vec::new(); + partial.state.decode_step(&mut tail).unwrap(); + assert_eq!(tail, b"Z"); + assert_eq!( + partial.state.flags_cnt, 7, + "the second flag bit comes from the terminal flags group" + ); +} + +#[test] +fn empty_encoder_fast_paths_skip_progress_and_nonempty_work_can_cancel() { + assert!( + super::unpack15_encode_with_options(&[], super::EncodeOptions::default()) + .unwrap() + .is_empty() + ); + assert!(super::unpack15_encode_with_options_and_progress( + &[], + super::EncodeOptions::default(), + &mut |_| panic!("empty input has no codec work") + ) + .unwrap() + .is_empty()); + let mut checkpoints = Vec::new(); + assert_eq!( + super::unpack15_encode_with_options_and_progress( + b"abc", + super::EncodeOptions::default(), + &mut |position| { + checkpoints.push(position); + false + } + ), + Err(Error::Cancelled) + ); + assert_eq!(checkpoints, [3]); +} + +#[test] +fn public_decoder_renormalizes_repeated_flags_without_losing_the_alphabet() { + let mut encoder = super::Unpack15Encoder::new(); + let first = encoder.encode_member(&[0]).unwrap(); + let mut decoder = super::Unpack15::new(); + assert_eq!(decoder.decode_member(&first, 1, false).unwrap(), [0]); + for _ in 0..300 { + encoder.emit_flags_byte(0); + for _ in 0..4 { + encoder.emit_short_lz(super::ShortLz { + distance: 1, + length: 2, + }); + } + } + let packed = std::mem::take(&mut encoder.bits).finish(); + assert_eq!( + decoder.decode_member(&packed, 2400, true).unwrap(), + vec![0; 2400] + ); + assert_eq!(decoder.state.ch_set_c, encoder.ch_set_c); + let frequency = decoder + .state + .ch_set_c + .iter() + .find(|&&entry| entry >> 8 == 0) + .unwrap() + & 0xff; + assert_eq!( + frequency, 52, + "256th update renormalizes the hot entry to 7, then increments it" + ); +} + +#[test] +fn invalid_flag_rank_preserves_the_table_and_match_overrun_writes_nothing() { + let mut bits = super::BitWriter::new(); + super::emit_decode_num(&mut bits, 256, 5, super::DEC_HF2, super::POS_HF2); + let packed = bits.finish(); + let mut decoder = super::Unpack15::new(); + let alphabet = decoder.state.ch_set_c; + assert_eq!(decoder.decode_member(&packed, 2, false).unwrap(), [0; 2]); + assert_eq!(decoder.state.ch_set_c, alphabet); + let mut streaming = super::Unpack15::new(); + let mut output = Vec::new(); + streaming + .decode_member_from_reader(&mut packed.as_slice(), 2, false, &mut output) + .unwrap(); + assert_eq!(output, [0; 2]); + assert_eq!(streaming.state.ch_set_c, alphabet); + assert_eq!( + super::Unpack15::new().decode_member(&[], 1, false), + Err(Error::InvalidData("RAR 1.3 match exceeds output size")) + ); + output.clear(); + assert_eq!( + super::Unpack15::new().decode_member_from_reader(&mut &[][..], 1, false, &mut output), + Err(Error::InvalidData("RAR 1.3 match exceeds output size")) + ); + assert!(output.is_empty()); +} + +#[test] +fn consecutive_minimum_long_matches_cross_the_distance_threshold() { + let mut encoder = super::Unpack15Encoder::new(); + let first = encoder.encode_literals_only_member(&[0; 256]); + let mut decoder = super::Unpack15::new(); + assert_eq!(decoder.decode_member(&first, 256, false).unwrap(), [0; 256]); + assert!(decoder.state.avr_plc < 0x2a00); + for _ in 0..178 { + encoder.emit_long_lz(super::LongLz { + distance: 1, + length: 11, + }); + } + let payload = std::mem::take(&mut encoder.bits).finish(); + decoder.state.init_member(178 * 11, true); + decoder.state.bits = + super::ReaderBits::with_allowance(&payload, &decoder.state.window.allowance()).unwrap(); + let mut output = Vec::new(); + for index in 0..178 { + decoder.state.long_lz(&mut output).unwrap(); + if index < 177 { + assert_eq!(decoder.state.max_dist3, 0x2001); + } + } + assert_eq!(output, vec![0; 178 * 11]); + assert_eq!(decoder.state.avr_ln3, 178); + assert_eq!(decoder.state.max_dist3, 0x7f00); + assert_eq!(decoder.state.ch_set_b, encoder.ch_set_b); +} + +#[test] +fn maximum_far_distance_zero_fills_then_reads_wrapped_solid_history() { + fn far_member(encoder: &mut super::Unpack15Encoder) -> Vec { + encoder.emit_flags_byte(0); + // Code 14 is distinct from code 1 only after the Buf60 toggle. + encoder.emit_short_lz_code(10); + super::emit_decode_num(&mut encoder.bits, 255, 2, super::DEC_L1, super::POS_L1); + encoder.emit_short_lz_code(14); + super::emit_decode_num(&mut encoder.bits, 0, 3, super::DEC_L2, super::POS_L2); + encoder.bits.write_bits(0x7fff, 15); + std::mem::take(&mut encoder.bits).finish() + } + let mut encoder = super::Unpack15Encoder::new(); + let packed = far_member(&mut encoder); + let mut decoder = super::Unpack15::new(); + assert_eq!(decoder.decode_member(&packed, 5, false).unwrap(), [0; 5]); + assert_eq!(decoder.state.last_dist, 0xffff); + + let mut first = vec![0; 0x10000]; + first[1..6].copy_from_slice(b"abcde"); + let mut encoder = super::Unpack15Encoder::new(); + let packed_first = encoder.encode_literals_only_member(&first); + let mut decoder = super::Unpack15::new(); + assert_eq!( + decoder + .decode_member(&packed_first, first.len(), false) + .unwrap(), + first + ); + assert_eq!(decoder.state.unp_ptr, 0); + assert_eq!(decoder.state.old_dist, [u32::MAX; 4]); + // The next decoding step observes the wrap before reading a match. + let mut unused_history = decoder.clone(); + let mut unused_streaming = decoder.clone(); + let mut old_encoder = encoder.clone_for_planning(); + old_encoder.emit_flags_byte(0); + old_encoder.emit_short_lz_code(10); + super::emit_decode_num(&mut old_encoder.bits, 0, 2, super::DEC_L1, super::POS_L1); + let old_packed = std::mem::take(&mut old_encoder.bits).finish(); + assert_eq!( + unused_history.decode_member(&old_packed, 4, true).unwrap(), + [0; 4] + ); + assert!(unused_history.state.first_win_done); + assert_eq!(unused_history.state.last_dist, u32::MAX); + let mut zeros = Vec::new(); + unused_streaming + .decode_member_from_reader(&mut old_packed.as_slice(), 4, true, &mut zeros) + .unwrap(); + assert_eq!(zeros, [0; 4]); + let packed = far_member(&mut encoder); + let mut streaming = decoder.clone(); + assert_eq!(decoder.decode_member(&packed, 5, true).unwrap(), b"abcde"); + assert_eq!(decoder.state.last_dist, 0xffff); + assert!(decoder.state.first_win_done); + let mut output = Vec::new(); + streaming + .decode_member_from_reader(&mut packed.as_slice(), 5, true, &mut output) + .unwrap(); + assert_eq!(output, b"abcde"); +} + +#[test] +fn old_distance_finder_keeps_recent_entry_when_match_lengths_tie() { + let mut encoder = super::Unpack15Encoder::new(); + for distance in 1..=4 { + encoder.remember_match(distance, 3); + } + let token = super::find_old_dist_lz( + &[0; 16], + 8, + encoder.old_dist, + encoder.old_dist_ptr, + encoder.max_dist3, + ) + .unwrap(); + assert_eq!( + token, + super::OldDistLz { + distance: 4, + length: 8, + short_code: 10 + } + ); + for _ in 0..4 { + encoder.remember_match(4, 3); + } + assert_eq!( + super::find_old_dist_lz( + &[0; 16], + 8, + encoder.old_dist, + encoder.old_dist_ptr, + encoder.max_dist3 + ), + Some(token) + ); +} + +#[test] +fn repeat_and_old_finders_refuse_history_before_the_member_prefix() { + assert_eq!(super::find_repeat_last_lz(&[0; 8], 4, u32::MAX, 0), None); + assert_eq!(super::find_repeat_last_lz(&[0; 8], 4, 5, 3), None); + assert_eq!(super::find_repeat_last_lz(&[0; 8], 4, 4, 5), None); + assert_eq!(super::find_repeat_last_lz(b"abcdabce", 4, 4, 4), None); + assert_eq!( + super::find_repeat_last_lz(b"abcdabcd", 4, 4, 4), + Some(super::RepeatLastLz { + distance: 4, + length: 4 + }) + ); + assert_eq!( + super::find_old_dist_lz(&[0; 8], 4, [u32::MAX; 4], 0, 0x2001), + None + ); + assert_eq!(super::find_old_dist_lz(&[0; 8], 4, [5; 4], 0, 0x2001), None); +} + +#[test] +fn long_distance_updates_preserve_every_high_byte_through_counter_wraps() { + let mut encoder = super::Unpack15Encoder::new(); + let mut wraps = 0; + for distance in std::iter::repeat_n(128, 768).chain((0..256).map(|high| (high * 128).max(1))) { + let before = encoder.ch_set_b.iter().find(|&&v| v >> 8 == 1).unwrap() & 0xff; + let token = super::LongLz { + distance, + length: 11, + }; + assert!(encoder + .token_bit_cost(super::MatchToken::LongLz(token), encoder.lz_plan_state()) + .is_some()); + encoder.emit_long_lz(token); + let after = encoder.ch_set_b.iter().find(|&&v| v >> 8 == 1).unwrap() & 0xff; + wraps += usize::from(distance == 128 && after < before); + let mut counts = [0u16; 256]; + for &entry in &encoder.ch_set_b { + counts[(entry >> 8) as usize] += 1; + } + assert_eq!(counts, [1; 256]); + } + assert!(wraps >= 2); +} + +#[test] +fn long_length_cost_matches_emitted_bits_and_decoder_at_every_mode_boundary() { + for average in [0, 63, 64, 121, 122, 1000] { + let mut encoder = super::Unpack15Encoder::new(); + encoder.avr_ln2 = average; + for code in 0..=255 { + let length = encoder.long_lz_length_bit_cost(code); + let mut bits = super::BitWriter::new(); + super::emit_long_lz_length(&mut bits, average, code); + assert_eq!(bits.bit_pos, length); + let field = (u32::from(bits.output[0]) << 8) + | u32::from(bits.output.get(1).copied().unwrap_or(0)); + if average >= 64 { + let (start, thresholds, ranks) = if average >= 122 { + (3, DEC_L2, POS_L2) + } else { + (2, DEC_L1, POS_L1) + }; + for suffix in 0..(1u32 << (16 - length)) { + assert_eq!( + simulate_decode_num(field | suffix, start, thresholds, ranks), + (code, length) + ); + } + } else if code <= 7 { + assert_eq!(field, 1 << (15 - code)); + } else { + assert_eq!(field, code); + } + } + } +} + +#[test] +fn planned_old_distance_and_repeat_tokens_replay_after_flag_updates() { + fn state(encoder: &super::Unpack15Encoder) -> (u32, u32, [u32; 4], usize, u32, u32, u32, u32) { + let s = encoder.lz_plan_state(); + ( + s.last_dist, + s.last_length, + s.old_dist, + s.old_dist_ptr, + s.max_dist3, + s.nlzb, + s.nhfb, + s.l_count, + ) + } + for maximum in [0x2001, 0x7f00] { + let mut encoder = super::Unpack15Encoder::new(); + encoder.max_dist3 = maximum; + for distance in 1..=4 { + encoder.emit_short_lz(super::ShortLz { + distance, + length: 3, + }); + } + for code in 10..=13 { + for length in [3, 4, 256] { + let distance = + encoder.old_dist[(encoder.old_dist_ptr.wrapping_sub((code - 9) as usize)) & 3]; + let token = super::OldDistLz { + distance, + length, + short_code: code, + }; + assert!(encoder + .token_bit_cost( + super::MatchToken::OldDist(super::OldDistLz { + length: 258, + ..token + }), + encoder.lz_plan_state(), + ) + .is_none()); + assert!(encoder + .token_bit_cost(super::MatchToken::OldDist(token), encoder.lz_plan_state()) + .is_some()); + let mut planned = encoder.clone_for_planning(); + planned.emit_old_dist_lz(token); + // Real emission writes the flags table before replaying payloads. + // Wrap its adaptive counter to check that this remains independent. + for _ in 0..512 { + encoder.emit_flags_byte(255); + } + encoder.emit_old_dist_lz(token); + assert_eq!(state(&encoder), state(&planned)); + for _ in 0..3 { + let repeat = super::find_repeat_last_lz( + &[0; 1024], + 512, + encoder.last_dist, + encoder.last_length, + ) + .unwrap(); + planned.emit_repeat_last(repeat); + encoder.emit_flags_byte(0); + encoder.emit_repeat_last(repeat); + assert_eq!(state(&encoder), state(&planned)); + } + } + } + } +} + +#[test] +fn flags_and_short_distance_updates_preserve_complete_alphabets() { + let mut encoder = super::Unpack15Encoder::new(); + let mut renormalizations = 0; + for flags in std::iter::repeat_n(255, 768).chain(0..=255) { + let before = encoder.ch_set_c.iter().find(|&&v| v >> 8 == 255).unwrap() & 0xff; + encoder.emit_flags_byte(flags); + let after = encoder.ch_set_c.iter().find(|&&v| v >> 8 == 255).unwrap() & 0xff; + renormalizations += usize::from(flags == 255 && after < before); + let mut counts = [0u16; 256]; + for &entry in &encoder.ch_set_c { + counts[(entry >> 8) as usize] += 1; + } + assert_eq!(counts, [1; 256]); + } + assert!(renormalizations >= 2); + for length in 2..=10 { + for distance in (1..=256).rev() { + encoder.emit_short_lz(super::ShortLz { distance, length }); + let mut counts = [0u16; 256]; + for &entry in &encoder.ch_set_a { + counts[entry as usize] += 1; + } + assert_eq!(counts, [1; 256]); + } + } + let mut input: Vec = (0..=255).collect(); + input.extend_from_slice(&[0, 1]); + assert_eq!( + super::find_short_lz(&input, 256), + Some(super::ShortLz { + distance: 256, + length: 2 + }) + ); +} + +#[test] +fn adaptive_literal_updates_preserve_every_byte_through_renormalization() { + fn assert_alphabet(encoder: &super::Unpack15Encoder) { + let mut counts = [0u16; 256]; + for &entry in &encoder.ch_set { + counts[(entry >> 8) as usize] += 1; + assert!(entry & 0xff <= 0xa1); + } + assert_eq!(counts, [1; 256]); + } + for stmode in [false, true] { + for average in [0, 0x0e00, 0x3600, 0x5e00, 0x7600] { + let mut encoder = super::Unpack15Encoder::new(); + encoder.avr_plc = average; + let mut renormalizations = 0; + for byte in std::iter::repeat_n(0, 512).chain(0..=255) { + let before = encoder + .ch_set + .iter() + .find(|&&entry| entry >> 8 == 0) + .unwrap() + & 0xff; + if stmode { + encoder.emit_stmode_literal(byte); + } else { + encoder.emit_literal(byte); + } + let after = encoder + .ch_set + .iter() + .find(|&&entry| entry >> 8 == 0) + .unwrap() + & 0xff; + renormalizations += usize::from(byte == 0 && after < before); + assert_alphabet(&encoder); + } + assert!(renormalizations >= 2); + } + } +} + +#[test] +fn public_decoder_clone_keeps_independent_solid_history_and_tables() { + let first = b"legacy adaptive solid history ".repeat(32); + let second = b"legacy adaptive solid history ".repeat(8); + let mut encoder = super::Unpack15Encoder::new(); + let packed_first = encoder.encode_member(&first).unwrap(); + let packed_second = encoder.encode_member(&second).unwrap(); + let mut original = super::Unpack15::new(); + assert_eq!( + original + .decode_member(&packed_first, first.len(), false) + .unwrap(), + first + ); + let mut copied = original.clone(); + let other = b"a different non-solid member"; + assert_eq!( + original + .decode_member(&super::unpack15_encode(other).unwrap(), other.len(), false) + .unwrap(), + other + ); + drop(original); + assert_eq!( + copied + .decode_member(&packed_second, second.len(), true) + .unwrap(), + second + ); +} + +#[test] +fn reader15_workspace_refusals_release_window_input_output_and_checkpoint() { + use crate::codec::workspace::{Allowance, RefusingBudget}; + let data = b"legacy reader owned capacity\n".repeat(32); + let packed = unpack15_encode(&data).unwrap(); + for streaming in [false, true] { + let run = |budget: &RefusingBudget| -> super::Result<()> { + let mut decoder = super::Reader15State::with_allowance(budget)?; + if streaming { + let mut output = super::Buffer::new(budget); + decoder.decode_member_from_reader( + &mut packed.as_slice(), + data.len(), + false, + &mut output, + )?; + assert_eq!(&*output, &data); + } else { + let output = decoder.decode_member(&packed, data.len(), false)?; + assert_eq!(&*output, &data); + } + let checkpoint = decoder.try_clone()?; + assert_eq!(checkpoint.window, decoder.window); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + let attempts = baseline.attempts(); + assert!(attempts >= 5); + assert_eq!(baseline.used(), 0); + for index in 0..attempts { + let budget = RefusingBudget::new(index); + assert!( + matches!(run(&budget), Err(Error::Cancelled)), + "allocation {index}, streaming={streaming}" + ); + assert_eq!(budget.used(), 0); + } + } + let limit = Allowance::limited(0xffff); + assert!(matches!( + super::Reader15State::with_allowance(&limit), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(limit.used(), 0); +} + +#[test] +fn reader15_workspace_releases_input_and_chunk_on_sink_failure() { + use crate::codec::workspace::Allowance; + struct FailingSink; + impl std::io::Write for FailingSink { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "sink denied", + )) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let data = b"legacy output failure".repeat(10); + let packed = unpack15_encode(&data).unwrap(); + let ledger = Allowance::limited(128 * 1024); + let mut decoder = super::Reader15State::with_allowance(&ledger).unwrap(); + assert!(matches!( + decoder.decode_member_from_reader( + &mut packed.as_slice(), + data.len(), + false, + &mut FailingSink + ), + Err(Error::Io(_)) + )); + assert_eq!( + ledger.used(), + 0x10000 + decoder.bits.input.capacity() as u64 + ); + drop(decoder); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn cancellation_interrupts_buffered_symbol_work() { + let data = b"cancellable legacy symbols ".repeat(16384); + let packed = unpack15_encode(&data).unwrap(); + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack15::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + decoder.read_control.cancel_after_checks(3); + assert_eq!( + decoder + .decode_member(&packed, data.len(), false) + .unwrap_err(), + Error::Cancelled + ); + assert!(decoder.state.output_written > 0 && decoder.state.output_written < data.len()); +} +use super::{ + decode_num_bit_cost, find_long_lz, find_long_lz_with_buckets, find_lz_token, find_old_dist_lz, + find_short_lz, flag_fits, long_lz_buckets, should_lazy_emit_literal, unpack15_decode, + unpack15_encode, unpack15_encode_with_options, EncodeOptions, LongLz, LzPlanState, MatchToken, + OldDistLz, Rar13MatchFinder, ShortLz, Unpack15, Unpack15Encoder, DEC_HF0, DEC_HF1, DEC_HF2, + DEC_HF3, DEC_HF4, DEC_L1, DEC_L2, POS_HF0, POS_HF1, POS_HF2, POS_HF3, POS_HF4, POS_L1, POS_L2, +}; + +fn decode_num_prefix_is_stable( + code: u32, + len: usize, + target: u32, + start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], +) -> bool { + let relevant_tail_bits = 16usize.saturating_sub(len + 4); + for tail in 0..(1u32 << relevant_tail_bits) { + let bit_field = (code << (16 - len)) | (tail << 4); + let (decoded, consumed) = simulate_decode_num(bit_field, start_pos, dec_tab, pos_tab); + if decoded != target || consumed != len { + return false; + } + } + true +} + +fn simulate_decode_num( + bit_field: u32, + mut start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], +) -> (u32, usize) { + let num = bit_field & 0xfff0; + let mut i = 0usize; + while dec_tab[i] as u32 <= num { + start_pos += 1; + i += 1; + } + ( + ((num - if i > 0 { dec_tab[i - 1] as u32 } else { 0 }) >> (16 - start_pos)) + + pos_tab[start_pos as usize] as u32, + start_pos as usize, + ) +} + +#[test] +fn probe_rar15_solid() { + let Ok(dir) = std::env::var("RARS_PROBE_DIR") else { + return; + }; + let options = EncodeOptions::new().with_lazy_matching(false); + let mut names: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + names.sort(); + let mut enc = Unpack15Encoder::with_options(options); + let mut dec = Unpack15::new(); + for path in &names { + let data = std::fs::read(path).unwrap(); + let packed = enc.encode_member(&data).unwrap(); + match dec.decode_member(&packed, data.len(), true) { + Ok(out) if out == data => println!("{:?} solid OK", path.file_name().unwrap()), + Ok(out) => println!( + "{:?} solid WRONG at {:?}", + path.file_name().unwrap(), + out.iter().zip(data.iter()).position(|(a, b)| a != b) + ), + Err(e) => println!("{:?} solid ERROR {e:?}", path.file_name().unwrap()), + } + } +} + +fn brute_decode_num_bit_cost( + target: u32, + start_pos: u32, + dec_tab: &[u16], + pos_tab: &[u16], +) -> Option { + for len in start_pos as usize..=16 { + for code in 0..(1u32 << len) { + if decode_num_prefix_is_stable(code, len, target, start_pos, dec_tab, pos_tab) { + return Some(len); + } + } + } + None +} + +#[test] +fn decode_member_from_reader_accepts_incremental_input() { + struct TinyReader<'a> { + input: &'a [u8], + } + + impl std::io::Read for TinyReader<'_> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.input.is_empty() { + return Ok(0); + } + let len = self.input.len().min(out.len()).min(2); + out[..len].copy_from_slice(&self.input[..len]); + self.input = &self.input[len..]; + Ok(len) + } + } + + let expected = b"RAR 1.4 incremental input fixture\n".repeat(32); + let packed = unpack15_encode(&expected).unwrap(); + assert_eq!(unpack15_decode(&packed, expected.len()).unwrap(), expected); + + let mut reader = TinyReader { input: &packed }; + let mut decoder = Unpack15::new(); + let mut output = Vec::new(); + decoder + .decode_member_from_reader(&mut reader, expected.len(), false, &mut output) + .unwrap(); + + assert_eq!(output, expected); +} + +#[test] +fn literal_only_encoder_round_trips_flag_and_mode_boundaries() { + for length in 0..=96 { + let input: Vec = (0..length) + .map(|index| (index * 73 + length * 17) as u8) + .collect(); + let packed = Unpack15Encoder::new().encode_literals_only(&input).unwrap(); + let decoded = unpack15_decode(&packed, input.len()).unwrap(); + assert_eq!(decoded, input, "literal-only length {length}"); + } +} + +#[test] +fn literal_only_encoder_follows_match_heavy_solid_history() { + let second: Vec = (0..96).map(|index| (index * 73 + 17) as u8).collect(); + let first = match_heavy_payload(176, 8000); + let mut encoder = Unpack15Encoder::with_options(EncodeOptions::new().with_lazy_matching(false)); + let first_packed = encoder.encode_member(&first).unwrap(); + assert!( + encoder.nlzb > encoder.nhfb, + "nlzb={} nhfb={}", + encoder.nlzb, + encoder.nhfb + ); + let second_packed = encoder.encode_literals_only(&second).unwrap(); + + let mut decoder = Unpack15::new(); + assert_eq!( + decoder + .decode_member(&first_packed, first.len(), false) + .unwrap(), + first + ); + assert_eq!( + decoder + .decode_member(&second_packed, second.len(), true) + .unwrap(), + second + ); +} + +#[test] +fn final_input_zero_pads_missing_bits_in_both_decode_paths() { + let mut decoder = Unpack15::new(); + let direct = decoder.decode_member(&[], 8, false).unwrap(); + assert_eq!(direct.len(), 8); + + let mut decoder = Unpack15::new(); + let mut from_reader = Vec::new(); + decoder + .decode_member_from_reader(&mut &[][..], 8, false, &mut from_reader) + .unwrap(); + assert_eq!(from_reader, direct); +} + +#[test] +fn fixed_number_tables_reencode_every_decoder_prefix() { + for (start, thresholds, ranks, maximum) in [ + (4, DEC_HF0, POS_HF0, 256), + (5, DEC_HF1, POS_HF1, 256), + (5, DEC_HF2, POS_HF2, 256), + (6, DEC_HF3, POS_HF3, 256), + (8, DEC_HF4, POS_HF4, 256), + (2, DEC_L1, POS_L1, 255), + (3, DEC_L2, POS_L2, 255), + ] { + assert!(ranks.len() <= 17); + assert!(ranks.len() > start as usize); + assert!(thresholds.len() >= ranks.len() - start as usize); + assert!(thresholds.iter().all(|&threshold| threshold != 0)); + assert!(thresholds.windows(2).all(|pair| pair[0] <= pair[1])); + assert_eq!(thresholds.last(), Some(&u16::MAX)); + let mut intervals = [None::<(u32, u32)>; 17]; + for field in 0..=u16::MAX { + let (target, consumed) = + simulate_decode_num(u32::from(field), start, thresholds, ranks); + assert!(target <= maximum); + let span = intervals[consumed].get_or_insert((target, target)); + span.0 = span.0.min(target); + span.1 = span.1.max(target); + let (prefix, length) = + super::encode_decode_num_prefix(target, start, thresholds, ranks).unwrap(); + assert_eq!(length, consumed); + assert_eq!(prefix, u32::from(field) >> (16 - length)); + } + let mut next = 0; + for (minimum, maximum) in intervals.into_iter().flatten() { + assert_eq!( + minimum, next, + "nonempty intervals are contiguous in bit-length order" + ); + next = maximum + 1; + } + assert_eq!(next, maximum + 1); + } +} + +#[test] +fn literal_codebooks_decode_every_rank_independently_of_following_bits() { + for (start, thresholds, ranks) in [ + (4, DEC_HF0, POS_HF0), + (5, DEC_HF1, POS_HF1), + (5, DEC_HF2, POS_HF2), + (6, DEC_HF3, POS_HF3), + (8, DEC_HF4, POS_HF4), + ] { + for rank in 0..=256 { + let (prefix, length) = + super::encode_decode_num_prefix(rank, start, thresholds, ranks).unwrap(); + for suffix in 0..(1u32 << (16 - length)) { + let field = (prefix << (16 - length)) | suffix; + assert_eq!( + simulate_decode_num(field, start, thresholds, ranks), + (rank, length), + "start={start}, rank={rank}, suffix={suffix}" + ); + } + } + } +} + +#[test] +fn decode_num_bit_cost_matches_prefix_search_at_table_boundaries() { + let tables = [ + (4, DEC_HF0, POS_HF0), + (5, DEC_HF1, POS_HF1), + (5, DEC_HF2, POS_HF2), + (6, DEC_HF3, POS_HF3), + (8, DEC_HF4, POS_HF4), + (2, DEC_L1, POS_L1), + (3, DEC_L2, POS_L2), + ]; + let targets = [0, 1, 2, 3, 7, 8, 16, 24, 32, 33, 53, 117, 233, 255, 256]; + + for (start_pos, dec_tab, pos_tab) in tables { + for target in targets { + assert_eq!( + decode_num_bit_cost(target, start_pos, dec_tab, pos_tab), + brute_decode_num_bit_cost(target, start_pos, dec_tab, pos_tab), + "target {target}, start_pos {start_pos}" + ); + } + } +} + +#[test] +fn encoder_emits_rar15_very_long_lz_matches() { + let mut input: Vec<_> = (0u8..=255).cycle().take(300).collect(); + input.extend_from_within(..258); + + assert_eq!( + find_long_lz(&input, 300, 0x8000), + Some(LongLz { + distance: 300, + length: 258 + }) + ); + let packed = unpack15_encode(&input).unwrap(); + + assert!( + packed.len() < 330, + "very-long LongLZ should encode a 258-byte repeat compactly, got {} bytes", + packed.len() + ); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn short_lz_accepts_the_two_byte_wire_minimum() { + let input = b"abXabY"; + + assert_eq!( + find_short_lz(input, 3), + Some(ShortLz { + distance: 3, + length: 2, + }) + ); + let packed = unpack15_encode(input).unwrap(); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn one_bit_match_flags_fit_at_every_open_position() { + for used in 0..8 { + assert!(flag_fits(used, &[true]), "flag bit {used}"); + } + assert!(!flag_fits(8, &[true])); +} + +#[test] +fn long_lz_search_accepts_near_distance_boundaries() { + let distance_one = vec![b'A'; 64]; + assert_eq!( + find_long_lz(&distance_one, 1, 0x8000), + Some(LongLz { + distance: 1, + length: 63, + }) + ); + + let distance_sixteen = b"abcdefghijklmnop".repeat(4); + assert_eq!( + find_long_lz(&distance_sixteen, 16, 0x8000), + Some(LongLz { + distance: 16, + length: 48, + }) + ); + + let distance_256: Vec<_> = (0u8..=255).cycle().take(512).collect(); + assert_eq!( + find_long_lz(&distance_256, 256, 0x8000), + Some(LongLz { + distance: 256, + length: 256, + }) + ); +} + +#[test] +fn near_long_lz_requires_the_eleven_byte_wire_minimum() { + let input = b"abcdefghijXabcdefghijY"; + + assert_eq!(find_long_lz(input, 11, 0x8000), None); +} + +#[test] +fn distance_257_remains_a_far_long_lz_match() { + let mut state = 0x1234_5678u32; + let mut input: Vec<_> = (0..257) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let repeated = input[..32].to_vec(); + input.extend_from_slice(&repeated); + + assert_eq!( + find_long_lz(&input, 257, 0x8000), + Some(LongLz { + distance: 257, + length: 32, + }) + ); +} + +#[test] +fn near_candidates_do_not_spend_the_far_match_budget() { + let pos = 2048; + let mut state = 0x9e37_79b9u32; + let mut input: Vec<_> = (0..pos + 32) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let pattern = b"ABCDEFGHIJKLMNOPQRST"; + input[128..128 + pattern.len()].copy_from_slice(pattern); + input[pos..pos + pattern.len()].copy_from_slice(pattern); + + let far_distractors: Vec<_> = (0..63).map(|index| 512 + index * 4).collect(); + let near_distractors: Vec<_> = (0..64).map(|index| pos - 256 + index * 4).collect(); + for &candidate in far_distractors.iter().chain(&near_distractors) { + input[candidate..candidate + 4].copy_from_slice(b"ABC!"); + } + + let mut positions = vec![128]; + positions.extend(far_distractors); + positions.extend(near_distractors); + let mut buckets = vec![Vec::new(); 1 << super::LONG_LZ_HASH_BITS]; + buckets[Rar13MatchFinder::hash(&input, pos)] = positions; + let finder = Rar13MatchFinder { buckets }; + + assert_eq!( + find_long_lz_with_buckets(&input, pos, 0x8000, &finder, 64), + Some(LongLz { + distance: (pos - 128) as u32, + length: pattern.len() as u32, + }) + ); +} + +#[test] +fn encoder_selects_and_round_trips_a_near_long_lz_match() { + let input = b"abcdefghijklmnop".repeat(64); + let buckets = long_lz_buckets(&input); + let encoder = Unpack15Encoder::new(); + + assert!(matches!( + encoder.choose_lz_token(&input, 16, &buckets, encoder.lz_plan_state()), + Some(MatchToken::LongLz(LongLz { + distance: 16, + length: 258, + })) + )); + + let packed = unpack15_encode(&input).unwrap(); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +/// Ad-hoc token census for comparing equivalent RAR 1.4 archives. +/// +/// Run with a platform-path-separated list of archive paths in +/// `RARS_RAR14_TOKEN_ARCHIVES` and `--ignored --nocapture`. +#[test] +#[ignore = "requires RARS_RAR14_TOKEN_ARCHIVES"] +fn report_rar14_token_census() { + let paths = + std::env::var_os("RARS_RAR14_TOKEN_ARCHIVES").expect("set RARS_RAR14_TOKEN_ARCHIVES"); + for path in std::env::split_paths(&paths) { + let bytes = std::fs::read(&path).unwrap(); + let archive = crate::rar13::Archive::parse(&bytes).unwrap(); + let mut decoder = Unpack15::new(); + println!("{}", path.display()); + for entry in &archive.entries { + if entry.is_stored() || entry.is_directory() { + continue; + } + decoder.state.token_stats = super::DecodeTokenStats::default(); + decoder.state.old_distance_events.clear(); + decoder + .decode_member( + entry.packed_data(&archive).unwrap(), + entry.header.unp_size as usize, + entry.header.flags & 0x10 != 0, + ) + .unwrap(); + if decoder.state.token_stats.old_distance_matches != 0 { + println!( + " {}: {:?}", + String::from_utf8_lossy(&entry.name), + decoder.state.token_stats + ); + if let Ok(from) = std::env::var("RARS_RAR14_EVENT_FROM") { + let from: usize = from.parse().unwrap(); + for event in decoder.state.old_distance_events.iter().filter(|event| { + event.output_position >= from + && event.output_position < from.saturating_add(500) + }) { + println!( + " pos={} code={} distance={} length={} max_dist3={}", + event.output_position, + event.short_code, + event.distance, + event.length, + event.max_dist3 + ); + } + } + } + } + } +} + +#[test] +fn encoder_adjusts_rar15_long_lz_length_for_far_distance_bonus() { + let mut input: Vec<_> = (0..9000).map(|index| (index * 73 + 19) as u8).collect(); + input.extend_from_within(..10); + + let packed = unpack15_encode(&input).unwrap(); + + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_reuses_rar15_repeat_last_token() { + let input = b"abcdefghijklmnop".repeat(64); + let packed = unpack15_encode(&input).unwrap(); + + assert!( + packed.len() < 100, + "repeat-last tokens should keep a simple repeated pattern compact, got {} bytes", + packed.len() + ); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn old_distance_finder_maps_ring_entries_to_short_lz_codes() { + let mut input: Vec<_> = (0..80).map(|index| (index * 37 + 11) as u8).collect(); + let pos = input.len(); + input.extend_from_within(pos - 33..pos - 13); + + assert_eq!( + find_old_dist_lz(&input, pos, [11, 22, 33, 44], 0, 0x2001), + Some(OldDistLz { + distance: 33, + length: 20, + short_code: 11, + }) + ); +} + +#[test] +fn old_distance_finder_rejects_dos_incompatible_maximum_length() { + let mut input = b"abcd".repeat(128); + let pos = input.len(); + input.extend((0..257).map(|index| b"abcd"[index % 4])); + + assert_eq!( + find_old_dist_lz(&input, pos, [u32::MAX, 4, u32::MAX, u32::MAX], 2, 0x2001), + None, + "the unsafe old-distance candidate should fall back to another token kind" + ); +} + +#[test] +fn old_distance_length_rejects_all_ones_symbol_for_every_short_code() { + for short_code in 10..=13 { + assert_eq!( + super::old_dist_lz_length_code(257, 4, 0x2001, short_code), + None, + "near old-distance code {short_code}" + ); + assert_eq!( + super::old_dist_lz_length_code(258, 330, 0x2001, short_code), + None, + "far old-distance code {short_code}" + ); + assert_eq!( + super::old_dist_lz_length_code(256, 4, 0x2001, short_code), + Some(254) + ); + assert_eq!( + super::old_dist_lz_length_code(257, 330, 0x2001, short_code), + Some(254) + ); + } +} + +#[test] +fn planner_emits_safe_old_distance_token() { + let mut input: Vec<_> = (0..80).map(|index| (index * 37 + 11) as u8).collect(); + let pos = input.len(); + input.extend_from_within(pos - 33..pos - 13); + + let encoder = Unpack15Encoder::new(); + let buckets = long_lz_buckets(&input); + let token = encoder + .choose_lz_token( + &input, + pos, + &buckets, + LzPlanState { + last_dist: u32::MAX, + last_length: 0, + old_dist: [11, 22, 33, 44], + old_dist_ptr: 0, + max_dist3: 0x2001, + nlzb: encoder.nlzb, + nhfb: encoder.nhfb, + l_count: encoder.l_count, + }, + ) + .expect("old-distance candidate should be selected"); + + assert_eq!( + token, + MatchToken::OldDist(OldDistLz { + distance: 33, + length: 20, + short_code: 11, + }) + ); +} + +#[test] +fn encoder_exits_stmode_when_literal_runs_trigger_decoder_mode() { + let input: Vec<_> = (0..96).map(|index| (index * 73 + 19) as u8).collect(); + let packed = unpack15_encode(&input).unwrap(); + + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_stmode_literals_for_long_literal_runs() { + let input: Vec<_> = (0..128).map(|index| (index * 73 + 19) as u8).collect(); + let mut encoder = Unpack15Encoder::new(); + let packed = encoder.encode_member(&input).unwrap(); + + assert!( + encoder.stmode_literal_count > 0, + "long literal runs should use stmode literals before exiting stmode" + ); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_options_can_disable_stmode_literal_runs() { + let input: Vec<_> = (0..128).map(|index| (index * 73 + 19) as u8).collect(); + let mut encoder = + Unpack15Encoder::with_options(EncodeOptions::new().with_stmode_literal_runs(false)); + let packed = encoder.encode_member(&input).unwrap(); + + assert_eq!(encoder.stmode_literal_count, 0); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_options_bound_long_lz_search_distance() { + let mut input: Vec<_> = (0u8..=255).cycle().take(300).collect(); + input.extend_from_within(..64); + + assert_eq!( + find_long_lz(&input, 300, 256), + Some(LongLz { + distance: 44, + length: 44, + }) + ); + assert_eq!( + find_long_lz(&input, 300, 0x8000), + Some(LongLz { + distance: 300, + length: 64 + }) + ); +} + +#[test] +fn long_lz_search_rejects_unencodable_32k_distance() { + let mut input = Vec::with_capacity(0x8000 + 64); + let mut state = 0x1234_5678u32; + for _ in 0..0x8000 + 64 { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + input.push((state >> 24) as u8); + } + let repeated = input[..64].to_vec(); + input[0x8000..0x8000 + 64].copy_from_slice(&repeated); + + let token = find_long_lz(&input, 0x8000, 0x8000); + + assert_ne!(token.map(|token| token.distance), Some(0x8000)); +} + +#[test] +fn lazy_match_prefers_longer_next_position_match() { + let input = b"abcXbcQRSTabcQRSTUV"; + let buckets = long_lz_buckets(input); + let token = find_lz_token( + input, + 10, + &buckets, + LzPlanState { + last_dist: u32::MAX, + last_length: 0, + old_dist: [u32::MAX; 4], + old_dist_ptr: 0, + max_dist3: 0x2001, + nlzb: 0, + nhfb: 0, + l_count: 0, + }, + EncodeOptions::default(), + ) + .unwrap(); + + assert!(matches!( + token, + MatchToken::ShortLz(super::ShortLz { length: 3, .. }) + )); + assert!(should_lazy_emit_literal( + input, + 10, + &buckets, + token, + 0x2001, + EncodeOptions::default() + )); + + let packed = unpack15_encode(input).unwrap(); + assert_eq!(unpack15_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn cost_aware_selection_prefers_better_bits_per_byte_token() { + let mut input = vec![b'Z'; 40]; + input[7] = b'A'; + input[8] = b'A'; + input[9] = b'A'; + input[10] = b'B'; + input[39] = b'A'; + let pos = input.len(); + input.extend_from_slice(b"AAAAAAAAAA"); + + let mut encoder = Unpack15Encoder::new(); + encoder.old_dist = [u32::MAX, u32::MAX, u32::MAX, 33]; + let buckets = long_lz_buckets(&input); + let token = encoder + .choose_lz_token( + &input, + pos, + &buckets, + LzPlanState { + last_dist: u32::MAX, + last_length: 0, + old_dist: encoder.old_dist, + old_dist_ptr: encoder.old_dist_ptr, + max_dist3: encoder.max_dist3, + nlzb: encoder.nlzb, + nhfb: encoder.nhfb, + l_count: encoder.l_count, + }, + ) + .unwrap(); + + assert_eq!( + token, + MatchToken::ShortLz(ShortLz { + distance: 1, + length: 10, + }) + ); +} + +#[test] +fn planner_uses_simulated_max_dist3_for_old_distance_candidates() { + let mut state = 0x1234_5678u32; + let mut input = Vec::with_capacity(9004); + for _ in 0..9004 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + input.push(state as u8); + } + let pos = 9000; + let prefix = [input[0], input[1], input[2]]; + input[pos..pos + 3].copy_from_slice(&prefix); + input[pos + 3] = input[3].wrapping_add(1); + + let mut encoder = Unpack15Encoder::new(); + encoder.max_dist3 = 0x7f00; + let buckets = long_lz_buckets(&input); + let token = encoder.choose_lz_token( + &input, + pos, + &buckets, + LzPlanState { + last_dist: u32::MAX, + last_length: 0, + old_dist: [u32::MAX, u32::MAX, u32::MAX, 9000], + old_dist_ptr: 0, + max_dist3: 0x2001, + nlzb: encoder.nlzb, + nhfb: encoder.nhfb, + l_count: encoder.l_count, + }, + ); + + assert_eq!(token, None); +} + +#[test] +fn encoder_round_trips_source_shaped_payload() { + let source = concat!(include_str!("../rar13.rs"), include_str!("encoder.rs")).as_bytes(); + let input = &source[..source.len().min(50_902)]; + + let packed = unpack15_encode(input).unwrap(); + let decoded = unpack15_decode(&packed, input.len()).unwrap(); + + let first_diff = decoded + .iter() + .zip(input) + .position(|(actual, expected)| actual != expected); + assert_eq!(first_diff, None, "first differing byte in decoded payload"); + assert_eq!(decoded, input); +} + +/// Match-heavy input drives `nlzb` above `nhfb`, which widens the literal +/// flag to two bits, which is what lets a flag reach the last bit of a +/// flags byte with a bit still to place. The encoder used to pad the byte +/// and start the next group cleanly; the decoder read that padding as a +/// flag and everything after it decoded to the wrong bytes. +#[test] +fn a_flag_straddling_two_flags_bytes_round_trips() { + let input = match_heavy_payload(176, 8000); + + let packed = + unpack15_encode_with_options(&input, EncodeOptions::new().with_lazy_matching(false)) + .unwrap(); + let decoded = unpack15_decode(&packed, input.len()).unwrap(); + + let first_diff = decoded + .iter() + .zip(&input) + .position(|(actual, expected)| actual != expected); + assert_eq!(first_diff, None, "first differing byte in decoded payload"); +} + +fn match_heavy_payload(seed: u64, len: usize) -> Vec { + let mut state = seed | 1; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let mut out = Vec::with_capacity(len); + while out.len() < len { + let value = next(); + if out.len() > 4096 && value % 3 != 0 { + let distance = 64 + (value >> 8) as usize % (out.len() - 64); + let length = 3 + (value >> 40) as usize % 24; + let start = out.len() - distance; + for index in 0..length { + let byte = out[start + index % distance]; + out.push(byte); + } + } else { + out.push((value >> 24) as u8); + } + } + out.truncate(len); + out +} diff --git a/arca-rar/vendor/rars/src/codec/rar20.rs b/arca-rar/vendor/rars/src/codec/rar20.rs new file mode 100644 index 00000000..b6cc0597 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar20.rs @@ -0,0 +1,760 @@ +#[cfg(feature = "write")] +mod encoder; +#[cfg(feature = "write")] +pub(crate) use encoder::unpack20_encode_auto_with_options_and_progress; +#[cfg(feature = "write")] +pub use encoder::{ + unpack20_encode_auto, unpack20_encode_auto_with_options, unpack20_encode_literals, + unpack20_encode_literals_with_options, EncodeOptions, Unpack20Encoder, +}; + +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; +use std::io::{Read, Write}; + +const MAIN_COUNT: usize = 298; +const OFFSET_COUNT: usize = 48; +const LENGTH_COUNT: usize = 28; +const LEVEL_COUNT: usize = 19; +const TABLE_COUNT: usize = MAIN_COUNT + OFFSET_COUNT + LENGTH_COUNT; +const AUDIO_COUNT: usize = 257; +const MAX_CHANNELS: usize = 4; +const OLD_LEVEL_COUNT: usize = AUDIO_COUNT * MAX_CHANNELS; +const MAX_HISTORY: usize = 1024 * 1024; + +const LENGTH_BASES: [usize; LENGTH_COUNT] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, + 160, 192, 224, +]; +const LENGTH_BITS: [u8; LENGTH_COUNT] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, +]; +const OFFSET_BASES: [usize; OFFSET_COUNT] = [ + 0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, + 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, + 262144, 327680, 393216, 458752, 524288, 589824, 655360, 720896, 786432, 851968, 917504, 983040, +]; +const OFFSET_BITS: [u8; OFFSET_COUNT] = [ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, + 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, +]; +const SHORT_BASES: [usize; 8] = [0, 4, 8, 16, 32, 64, 128, 192]; +const SHORT_BITS: [u8; 8] = [2, 2, 3, 4, 5, 6, 6, 6]; + +pub fn unpack20_decode(input: &[u8], output_size: usize) -> Result> { + let mut decoder = Unpack20::new(); + decoder.decode_member(input, output_size) +} + +#[derive(Debug, Clone)] +pub struct Unpack20 { + pub(crate) read_control: crate::read_control::ReadControl, + state: Reader20State, +} +impl Clone for Reader20State { + fn clone(&self) -> Self { + self.try_clone().expect("unlimited legacy decoder copy") + } +} +impl Unpack20 { + pub fn new() -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + state: Reader20State::with_allowance(&Allowance::default()), + } + } + pub fn decode_member(&mut self, input: &[u8], output_size: usize) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_owned(input, output_size) + .map(Buffer::into_vec) + } + pub fn decode_member_to( + &mut self, + input: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state.decode_member_to(input, output_size, out) + } + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_from_reader(input, output_size, out) + } +} +#[cfg(all(test, feature = "write"))] +impl Reader20State { + fn new() -> Self { + Self::with_allowance(&Allowance::default()) + } + fn decode_member(&mut self, input: &[u8], output_size: usize) -> Result> { + self.decode_member_owned(input, output_size) + .map(Buffer::into_vec) + } +} +#[derive(Debug)] +pub(crate) struct Reader20State { + pub(crate) read_control: crate::read_control::ReadControl, + bits: BitReader, + levels: [u8; OLD_LEVEL_COUNT], + main: Huffman, + offsets: Huffman, + lengths: Huffman, + audio_tables: [Huffman; MAX_CHANNELS], + audio_block: bool, + channels: usize, + cur_channel: usize, + audio: [AudioState; MAX_CHANNELS], + channel_delta: i32, + old_offsets: [usize; 4], + last_offset: usize, + last_length: usize, + pending_match: Option<(usize, usize)>, + in_block: bool, + output: Buffer, + base_offset: usize, +} + +impl Reader20State { + pub(crate) fn with_allowance(allowance: &B) -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + bits: BitReader::with_allowance(allowance), + levels: [0; OLD_LEVEL_COUNT], + main: Huffman::with_allowance(allowance), + offsets: Huffman::with_allowance(allowance), + lengths: Huffman::with_allowance(allowance), + audio_tables: std::array::from_fn(|_| Huffman::with_allowance(allowance)), + audio_block: false, + channels: 1, + cur_channel: 0, + audio: [AudioState::default(); MAX_CHANNELS], + channel_delta: 0, + old_offsets: [0; 4], + last_offset: 0, + last_length: 0, + pending_match: None, + in_block: false, + output: Buffer::new(allowance), + base_offset: 0, + } + } + + pub(crate) fn try_clone(&self) -> Result { + Ok(Self { + read_control: self.read_control.clone(), + bits: self.bits.try_clone()?, + levels: self.levels, + main: self.main.try_clone()?, + offsets: self.offsets.try_clone()?, + lengths: self.lengths.try_clone()?, + audio_tables: [ + self.audio_tables[0].try_clone()?, + self.audio_tables[1].try_clone()?, + self.audio_tables[2].try_clone()?, + self.audio_tables[3].try_clone()?, + ], + audio_block: self.audio_block, + channels: self.channels, + cur_channel: self.cur_channel, + audio: self.audio, + channel_delta: self.channel_delta, + old_offsets: self.old_offsets, + last_offset: self.last_offset, + last_length: self.last_length, + pending_match: self.pending_match, + in_block: self.in_block, + output: Buffer::copied(&self.output, &self.output.allowance())?, + base_offset: self.base_offset, + }) + } + pub fn decode_member_owned( + &mut self, + input: &[u8], + output_size: usize, + ) -> Result> { + self.read_control.check_codec()?; + let start = self.current_pos(); + let target = start + .checked_add(output_size) + .ok_or(Error::InvalidData("RAR 2.0 output size overflows"))?; + if !input.is_empty() { + self.bits = BitReader::with_allowance(&self.output.allowance()); + } + self.bits.append(input)?; + self.decode_until(target).map_err(|error| match error { + Error::NeedMoreInput => Error::InvalidData("RAR 2.0 bitstream is truncated"), + error => error, + })?; + self.read_last_tables()?; + let out = Buffer::copied(self.raw_range(start, target), &self.output.allowance())?; + self.trim_history(target, target); + Ok(out) + } + + pub fn decode_member_to( + &mut self, + input: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + let decoded = self.decode_member_owned(input, output_size)?; + out.write_all(&decoded).map_err(Error::from) + } + + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + let start = self.current_pos(); + let target = start + .checked_add(output_size) + .ok_or(Error::InvalidData("RAR 2.0 output size overflows"))?; + self.bits = BitReader::with_allowance(&self.output.allowance()); + self.bits.input.read_to_end(input)?; + if !self.in_block && self.bits.remaining_bytes_from_current() > 0 { + self.read_tables().map_err(|error| match error { + Error::NeedMoreInput => Error::InvalidData("RAR 2.0 bitstream is truncated"), + error => error, + })?; + self.in_block = true; + } + self.decode_until(target).map_err(|error| match error { + Error::NeedMoreInput => Error::InvalidData("RAR 2.0 bitstream is truncated"), + error => error, + })?; + self.read_last_tables()?; + + let decoded = self.raw_range(start, target); + out.write_all(decoded).map_err(Error::from)?; + self.trim_history(target, target); + Ok(()) + } + + fn decode_until(&mut self, target: usize) -> Result<()> { + let mut poller = self.read_control.poller(); + while self.current_pos() < target { + poller.check_codec(self.current_pos())?; + self.drain_pending_match(target)?; + if self.current_pos() >= target { + break; + } + if !self.in_block { + self.read_tables()?; + self.in_block = true; + } + self.decode_lz(target)?; + } + Ok(()) + } + + fn read_tables(&mut self) -> Result<()> { + let bit_field = self.bits.peek_bits(16)?; + self.audio_block = bit_field & 0x8000 != 0; + let keep_tables = bit_field & 0x4000 != 0; + self.bits.read_bits(2)?; + if !keep_tables { + self.levels = [0; OLD_LEVEL_COUNT]; + } + + let table_size = if self.audio_block { + self.channels = ((bit_field >> 12) as usize & 3) + 1; + if self.cur_channel >= self.channels { + self.cur_channel = 0; + } + self.bits.read_bits(2)?; + AUDIO_COUNT * self.channels + } else { + TABLE_COUNT + }; + + let level_lengths = Self::read_level_lengths(&mut self.bits)?; + let level_decoder = Huffman::with_lengths(&level_lengths, &self.output.allowance())?; + let mut new_levels = [0u8; OLD_LEVEL_COUNT]; + let mut pos = 0usize; + while pos < table_size { + let symbol = level_decoder.decode(&mut self.bits)?; + match symbol { + 0..=15 => { + new_levels[pos] = (self.levels[pos].wrapping_add(symbol as u8)) & 0x0f; + pos += 1; + } + 16 => { + if pos == 0 { + return Err(Error::InvalidData("RAR 2.0 table repeat at start")); + } + let count = 3 + self.bits.read_bits(2)? as usize; + let value = new_levels[pos - 1]; + fill_levels(&mut new_levels, &mut pos, count, value)?; + } + 17 => { + let count = 3 + self.bits.read_bits(3)? as usize; + fill_levels(&mut new_levels, &mut pos, count, 0)?; + } + _ => { + // 18: the pre-table contains exactly 19 symbols. + let count = 11 + self.bits.read_bits(7)? as usize; + fill_levels(&mut new_levels, &mut pos, count, 0)?; + } + } + } + + self.levels = new_levels; + if self.audio_block { + for channel in 0..self.channels { + let start = channel * AUDIO_COUNT; + self.audio_tables[channel] = Huffman::with_lengths( + &self.levels[start..start + AUDIO_COUNT], + &self.output.allowance(), + )?; + } + } else { + self.main = + Huffman::with_lengths(&self.levels[..MAIN_COUNT], &self.output.allowance())?; + self.offsets = Huffman::with_lengths( + &self.levels[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT], + &self.output.allowance(), + )?; + self.lengths = Huffman::with_lengths( + &self.levels[MAIN_COUNT + OFFSET_COUNT..TABLE_COUNT], + &self.output.allowance(), + )?; + } + Ok(()) + } + + fn read_level_lengths(bits: &mut BitReader) -> Result<[u8; LEVEL_COUNT]> { + let mut lengths = [0u8; LEVEL_COUNT]; + for length in &mut lengths { + *length = bits.read_bits(4)? as u8; + } + Ok(lengths) + } + + fn decode_lz(&mut self, output_size: usize) -> Result<()> { + let mut poller = self.read_control.poller(); + while self.current_pos() < output_size { + poller.check_codec(self.current_pos())?; + if self.audio_block { + self.decode_audio_byte()?; + if !self.in_block { + return Ok(()); + } + continue; + } + let symbol = self.main.decode(&mut self.bits)?; + match symbol { + 0..=255 => self.output.try_push(symbol as u8)?, + 256 => { + if self.last_length != 0 { + let length = self.last_length; + let offset = self.last_offset; + self.push_old_offset(offset); + self.copy_match(length, offset, output_size)?; + } + } + 257..=260 => { + let index = symbol - 257; + let offset = self.old_offsets[index]; + let length_slot = self.lengths.decode(&mut self.bits)?; + let mut length = LENGTH_BASES[length_slot] + 2; + if LENGTH_BITS[length_slot] != 0 { + length += self.bits.read_bits(LENGTH_BITS[length_slot])? as usize; + } + if offset >= 0x101 { + length += 1; + } + if offset >= 0x2000 { + length += 1; + } + if offset >= 0x40000 { + length += 1; + } + self.push_old_offset(offset); + self.last_offset = offset; + self.last_length = length; + self.copy_match(length, offset, output_size)?; + } + 261..=268 => { + let index = symbol - 261; + let mut offset = SHORT_BASES[index] + 1; + offset += self.bits.read_bits(SHORT_BITS[index])? as usize; + self.push_old_offset(offset); + self.last_offset = offset; + self.last_length = 2; + self.copy_match(2, offset, output_size)?; + } + 269 => { + self.in_block = false; + return Ok(()); + } + _ => { + // 270..=297: the main table contains exactly 298 symbols. + let length_slot = symbol - 270; + let mut length = LENGTH_BASES[length_slot] + 3; + if LENGTH_BITS[length_slot] != 0 { + length += self.bits.read_bits(LENGTH_BITS[length_slot])? as usize; + } + let offset = self.read_offset()?; + if offset >= 0x2000 { + length += 1; + } + if offset >= 0x40000 { + length += 1; + } + self.push_old_offset(offset); + self.last_offset = offset; + self.last_length = length; + self.copy_match(length, offset, output_size)?; + } + } + } + Ok(()) + } + + fn decode_audio_byte(&mut self) -> Result<()> { + let symbol = self.audio_tables[self.cur_channel].decode(&mut self.bits)?; + if symbol == 256 { + self.in_block = false; + return Ok(()); + } + let byte = self.decode_audio(symbol as u8); + self.output.try_push(byte)?; + self.cur_channel += 1; + if self.cur_channel == self.channels { + self.cur_channel = 0; + } + Ok(()) + } + + fn decode_audio(&mut self, delta: u8) -> u8 { + let state = &mut self.audio[self.cur_channel]; + state.byte_count = state.byte_count.wrapping_add(1); + state.d4 = state.d3; + state.d3 = state.d2; + state.d2 = state.last_delta - state.d1; + state.d1 = state.last_delta; + + let predicted = 8 * state.last_char + + state.k[0] * state.d1 + + state.k[1] * state.d2 + + state.k[2] * state.d3 + + state.k[3] * state.d4 + + state.k[4] * self.channel_delta; + let predicted = (predicted >> 3) & 0xff; + let byte = predicted.wrapping_sub(delta as i32) as u8; + + let d = (delta as i8 as i32) << 3; + state.dif[0] = state.dif[0].wrapping_add(d.unsigned_abs()); + state.dif[1] = state.dif[1].wrapping_add((d - state.d1).unsigned_abs()); + state.dif[2] = state.dif[2].wrapping_add((d + state.d1).unsigned_abs()); + state.dif[3] = state.dif[3].wrapping_add((d - state.d2).unsigned_abs()); + state.dif[4] = state.dif[4].wrapping_add((d + state.d2).unsigned_abs()); + state.dif[5] = state.dif[5].wrapping_add((d - state.d3).unsigned_abs()); + state.dif[6] = state.dif[6].wrapping_add((d + state.d3).unsigned_abs()); + state.dif[7] = state.dif[7].wrapping_add((d - state.d4).unsigned_abs()); + state.dif[8] = state.dif[8].wrapping_add((d + state.d4).unsigned_abs()); + state.dif[9] = state.dif[9].wrapping_add((d - self.channel_delta).unsigned_abs()); + state.dif[10] = state.dif[10].wrapping_add((d + self.channel_delta).unsigned_abs()); + + self.channel_delta = (byte.wrapping_sub(state.last_char as u8)) as i8 as i32; + state.last_delta = self.channel_delta; + state.last_char = byte as i32; + + if state.byte_count & 0x1f == 0 { + let mut min_dif = state.dif[0]; + let mut num_min_dif = 0usize; + state.dif[0] = 0; + for index in 1..state.dif.len() { + if state.dif[index] < min_dif { + min_dif = state.dif[index]; + num_min_dif = index; + } + state.dif[index] = 0; + } + match num_min_dif { + 1 if state.k[0] >= -16 => state.k[0] -= 1, + 2 if state.k[0] < 16 => state.k[0] += 1, + 3 if state.k[1] >= -16 => state.k[1] -= 1, + 4 if state.k[1] < 16 => state.k[1] += 1, + 5 if state.k[2] >= -16 => state.k[2] -= 1, + 6 if state.k[2] < 16 => state.k[2] += 1, + 7 if state.k[3] >= -16 => state.k[3] -= 1, + 8 if state.k[3] < 16 => state.k[3] += 1, + 9 if state.k[4] >= -16 => state.k[4] -= 1, + 10 if state.k[4] < 16 => state.k[4] += 1, + _ => {} + } + } + + byte + } + + fn read_offset(&mut self) -> Result { + let slot = self.offsets.decode(&mut self.bits)?; + let mut offset = OFFSET_BASES[slot] + 1; + if OFFSET_BITS[slot] != 0 { + offset += self.bits.read_bits(OFFSET_BITS[slot])? as usize; + } + Ok(offset) + } + + fn copy_match(&mut self, length: usize, offset: usize, output_size: usize) -> Result<()> { + let offset = if offset == 0 { 1 } else { offset }; + // A match reaching past the start of the stream writes zeroes rather + // than failing. WinRAR never clears its window and guards the copy + // with a first-wrap flag instead, so those bytes read as zero there, + // and an archive that leans on it stays readable here. The decision + // is taken once for the whole match, as it is there: a copy does not + // start on zeroes and cross into real bytes partway. + let before_window = offset > self.current_pos(); + for index in 0..length { + if self.current_pos() >= output_size { + self.pending_match = Some((length - index, offset)); + break; + } + let byte = if before_window { + 0 + } else { + let src = self.current_pos() - offset; + self.raw_byte(src) + }; + self.output.try_push(byte)?; + } + Ok(()) + } + + fn drain_pending_match(&mut self, output_size: usize) -> Result<()> { + let Some((length, offset)) = self.pending_match.take() else { + return Ok(()); + }; + self.copy_match(length, offset, output_size)?; + Ok(()) + } + + fn read_last_tables(&mut self) -> Result<()> { + if self.bits.remaining_bytes_from_current() < 5 { + return Ok(()); + } + if self.audio_block { + if self.audio_tables[self.cur_channel].state.is_empty() { + return Ok(()); + } + if self.audio_tables[self.cur_channel].decode(&mut self.bits)? == 256 { + self.read_tables()?; + self.in_block = true; + } + } else { + if self.main.state.is_empty() { + return Ok(()); + } + if self.main.decode(&mut self.bits)? == 269 { + self.read_tables()?; + self.in_block = true; + } + } + Ok(()) + } + + fn push_old_offset(&mut self, offset: usize) { + self.old_offsets[3] = self.old_offsets[2]; + self.old_offsets[2] = self.old_offsets[1]; + self.old_offsets[1] = self.old_offsets[0]; + self.old_offsets[0] = offset; + } + + fn current_pos(&self) -> usize { + self.base_offset + self.output.len() + } + + fn raw_byte(&self, position: usize) -> u8 { + // Decoder offsets are at most MAX_HISTORY; trimming retains that window. + self.output[position - self.base_offset] + } + + fn raw_range(&self, start: usize, end: usize) -> &[u8] { + // Callers take the range before trimming the completed member. + let rel_start = start - self.base_offset; + let rel_end = end - self.base_offset; + &self.output[rel_start..rel_end] + } + + fn trim_history(&mut self, flushed_pos: usize, current_pos: usize) { + let keep_from = current_pos.saturating_sub(MAX_HISTORY).min(flushed_pos); + if keep_from <= self.base_offset { + return; + } + let drain = keep_from - self.base_offset; + self.output.discard_prefix(drain); + self.base_offset = keep_from; + } +} + +impl Default for Unpack20 { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, Copy, Default)] +struct AudioState { + k: [i32; 5], + d1: i32, + d2: i32, + d3: i32, + d4: i32, + last_delta: i32, + last_char: i32, + byte_count: u32, + dif: [u32; 11], +} + +fn fill_levels(levels: &mut [u8], pos: &mut usize, count: usize, value: u8) -> Result<()> { + let end = pos + .checked_add(count) + .ok_or(Error::InvalidData("RAR 2.0 table run overflows"))?; + let end = end.min(levels.len()); + for item in &mut levels[*pos..end] { + *item = value; + } + *pos = end; + Ok(()) +} + +#[derive(Debug)] +struct Huffman { + state: super::canonical::Huffman, +} + +impl Huffman { + fn with_allowance(allowance: &B) -> Self { + Self { + state: super::canonical::Huffman::with_allowance(allowance), + } + } + fn with_lengths(lengths: &[u8], allowance: &B) -> Result { + let mut count = [0u16; 16]; + for &len in lengths { + if len > 15 { + return Err(Error::InvalidData("RAR 2.0 Huffman length is too large")); + } + if len != 0 { + count[len as usize] += 1; + } + } + if count.iter().all(|&value| value == 0) { + return Ok(Self::with_allowance(allowance)); + } + validate_huffman_counts(&count)?; + + Ok(Self { + state: super::canonical::Huffman::from_counts(lengths, count, allowance)?, + }) + } + fn try_clone(&self) -> Result { + Ok(Self { + state: self.state.try_clone()?, + }) + } + fn decode(&self, bits: &mut BitReader) -> Result { + self.state.decode( + || bits.read_bit().map(|bit| bit as u16), + "RAR 2.0 empty Huffman table", + "RAR 2.0 invalid Huffman code", + ) + } +} + +#[cfg(all(test, feature = "write"))] +impl Huffman { + fn from_lengths(lengths: &[u8]) -> Result { + Self::with_lengths(lengths, &Allowance::default()) + } +} + +fn validate_huffman_counts(count: &[u16; 16]) -> Result<()> { + super::canonical::validate_counts(count, "RAR 2.0 oversubscribed Huffman table") +} + +#[derive(Debug)] +struct BitReader { + input: Buffer, + bit_pos: usize, +} + +impl BitReader { + fn with_allowance(allowance: &B) -> Self { + Self { + input: Buffer::new(allowance), + bit_pos: 0, + } + } + + fn try_clone(&self) -> Result { + Ok(Self { + input: Buffer::copied(&self.input, &self.input.allowance())?, + bit_pos: self.bit_pos, + }) + } + fn append(&mut self, input: &[u8]) -> Result<()> { + self.compact(); + self.input.extend_from_slice(input).map_err(Into::into) + } + + fn compact(&mut self) { + let bytes = self.bit_pos / 8; + if bytes == 0 { + return; + } + self.input.discard_prefix(bytes); + self.bit_pos -= bytes * 8; + } + + fn read_bit(&mut self) -> Result { + self.read_bits(1).map(|value| value as u8) + } + + fn read_bits(&mut self, count: u8) -> Result { + let value = self.peek_bits(count)?; + self.bit_pos += count as usize; + Ok(value) + } + + fn peek_bits(&self, count: u8) -> Result { + if count > 24 { + return Err(Error::InvalidData("RAR 2.0 bit read is too wide")); + } + let mut value = 0u32; + for i in 0..count as usize { + let bit_index = self.bit_pos + i; + let byte = *self.input.get(bit_index / 8).ok_or(Error::NeedMoreInput)?; + let bit = (byte >> (7 - (bit_index % 8))) & 1; + value = (value << 1) | bit as u32; + } + Ok(value) + } + + fn remaining_bytes_from_current(&self) -> usize { + self.input.len().saturating_sub(self.bit_pos / 8) + } +} + +#[cfg(all(test, feature = "write"))] +impl BitReader { + fn new() -> Self { + Self::with_allowance(&Allowance::default()) + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar20/encoder.rs b/arca-rar/vendor/rars/src/codec/rar20/encoder.rs new file mode 100644 index 00000000..89a69531 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar20/encoder.rs @@ -0,0 +1,1533 @@ +//! RAR20 member encoding and bit emission. +use super::super::{huffman, match_finder}; +use super::*; + +const MAX_ENCODER_MATCH_OFFSET: usize = MAX_HISTORY; + +const MAX_ENCODER_MATCH_LENGTH: usize = 258; + +const MAX_MATCH_CANDIDATES: usize = 256; + +type Rar20MatchFinder = match_finder::MatchFinder<3>; + +pub fn unpack20_encode_literals(input: &[u8]) -> Result> { + unpack20_encode_literals_with_options(input, EncodeOptions::default()) +} + +pub fn unpack20_encode_literals_with_options( + input: &[u8], + options: EncodeOptions, +) -> Result> { + encode_member(input, &[], None, options, None) +} + +pub fn unpack20_encode_auto(input: &[u8]) -> Result> { + unpack20_encode_auto_with_options(input, EncodeOptions::default()) +} + +pub fn unpack20_encode_auto_with_options(input: &[u8], options: EncodeOptions) -> Result> { + let lz = unpack20_encode_literals_with_options(input, options)?; + let mut best = lz; + if options.try_audio { + for channels in 1..=MAX_CHANNELS { + if input.len() < channels * 64 { + continue; + } + let audio = encode_audio_member(input, channels)?; + if audio.len() < best.len() { + best = audio; + } + } + } + Ok(best) +} + +pub(crate) fn unpack20_encode_auto_with_options_and_progress( + input: &[u8], + options: EncodeOptions, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + let mut best = encode_member(input, &[], None, options, Some(progress))?; + if options.try_audio { + for channels in 1..=MAX_CHANNELS { + if input.len() < channels * 64 { + continue; + } + let audio = encode_audio_member(input, channels)?; + if audio.len() < best.len() { + best = audio; + } + } + } + Ok(best) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct EncodeOptions { + pub max_match_candidates: usize, + pub max_match_distance: usize, + pub lazy_matching: bool, + pub lazy_lookahead: usize, + /// Parse by shortest path rather than greedily. Costs several times the + /// encode time, so it belongs at the top of the level ladder. + pub optimal_parse: bool, + pub try_audio: bool, +} + +impl EncodeOptions { + pub const fn new(max_match_candidates: usize) -> Self { + Self { + max_match_candidates, + max_match_distance: MAX_ENCODER_MATCH_OFFSET, + lazy_matching: false, + lazy_lookahead: 1, + optimal_parse: false, + try_audio: true, + } + } + + pub const fn with_max_match_distance(mut self, distance: usize) -> Self { + self.max_match_distance = if distance > MAX_ENCODER_MATCH_OFFSET { + MAX_ENCODER_MATCH_OFFSET + } else { + distance + }; + self + } + + pub const fn with_lazy_matching(mut self, enabled: bool) -> Self { + self.lazy_matching = enabled; + self + } + + pub const fn with_optimal_parse(mut self, enabled: bool) -> Self { + self.optimal_parse = enabled; + self + } + + pub const fn with_lazy_lookahead(mut self, bytes: usize) -> Self { + self.lazy_lookahead = bytes; + self + } + + pub const fn with_try_audio(mut self, enabled: bool) -> Self { + self.try_audio = enabled; + self + } + + const fn constrained(mut self) -> Self { + if self.max_match_distance > MAX_ENCODER_MATCH_OFFSET { + self.max_match_distance = MAX_ENCODER_MATCH_OFFSET; + } + self + } +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self::new(MAX_MATCH_CANDIDATES) + } +} + +#[derive(Debug, Clone, Default)] +pub struct Unpack20Encoder { + history: Vec, + table: Option, + options: EncodeOptions, +} + +impl Unpack20Encoder { + pub fn new() -> Self { + Self::default() + } + + pub fn with_options(options: EncodeOptions) -> Self { + Self { + history: Vec::new(), + table: None, + options: options.constrained(), + } + } + + pub fn encode_member(&mut self, input: &[u8]) -> Result> { + self.encode_member_inner(input, None) + } + + pub(crate) fn encode_member_with_progress( + &mut self, + input: &[u8], + progress: &mut dyn FnMut(usize) -> bool, + ) -> Result> { + self.encode_member_inner(input, Some(progress)) + } + + fn encode_member_inner( + &mut self, + input: &[u8], + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + if input.is_empty() { + return Ok(Vec::new()); + } + let table = match self.table { + Some(table) => table, + None => { + let table = FixedEncodeTable::new()?; + self.table = Some(table); + table + } + }; + let packed = encode_member(input, &self.history, Some(table), self.options, progress)?; + self.remember(input); + Ok(packed) + } + + fn remember(&mut self, input: &[u8]) { + self.history.extend_from_slice(input); + let keep_from = self + .history + .len() + .saturating_sub(self.options.max_match_distance); + if keep_from != 0 { + self.history.drain(..keep_from); + } + } +} + +fn encode_member( + input: &[u8], + history: &[u8], + fixed_table: Option, + options: EncodeOptions, + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + if input.is_empty() { + return Ok(Vec::new()); + } + // The public field can be assigned directly, bypassing the builder clamp. + let options = options.constrained(); + + let tokens = match progress.as_mut() { + Some(report) => { + encode_tokens_with_progress(input, history, options, None, Some(&mut **report))? + } + None => encode_tokens_with_progress(input, history, options, None, None)?, + }; + let table_lengths = table_lengths_for_tokens(&tokens, fixed_table)?; + let packed = encode_member_with_tables(&tokens, history, fixed_table, &table_lengths)?; + if fixed_table.is_some() { + return Ok(packed); + } + + // Re-parse against the prices the first pass implies, then again against + // the prices that produced. A greedy parse has converged by the first + // re-parse and a second buys 0.02%, but the shortest-path parse is far more + // sensitive to its prices and a second pass is worth 0.12% to it. + let refinements = if options.optimal_parse { 2 } else { 1 }; + let mut best_tokens = tokens; + let mut best_table = table_lengths; + let mut best_packed = packed; + for _ in 0..refinements { + let cost_model = CostModel::new(&best_table); + let next_tokens = match progress.as_mut() { + Some(report) => encode_tokens_with_progress( + input, + history, + options, + Some(&cost_model), + Some(&mut **report), + )?, + None => encode_tokens_with_progress(input, history, options, Some(&cost_model), None)?, + }; + if next_tokens == best_tokens { + break; + } + let next_table = table_lengths_for_tokens(&next_tokens, fixed_table)?; + let next_packed = + encode_member_with_tables(&next_tokens, history, fixed_table, &next_table)?; + if next_packed.len() >= best_packed.len() { + break; + } + best_tokens = next_tokens; + best_table = next_table; + best_packed = next_packed; + } + Ok(best_packed) +} + +fn table_lengths_for_tokens( + tokens: &[EncodeToken], + fixed_table: Option, +) -> Result<[u8; TABLE_COUNT]> { + let mut main_frequencies = [0usize; MAIN_COUNT]; + let mut offset_frequencies = [0usize; OFFSET_COUNT]; + let mut length_frequencies = [0usize; LENGTH_COUNT]; + for token in tokens { + match *token { + EncodeToken::Literal(byte) => main_frequencies[byte as usize] += 1, + EncodeToken::RepeatLast => main_frequencies[256] += 1, + EncodeToken::OldOffset { + index, + length, + offset, + } => { + main_frequencies[257 + index] += 1; + let (slot, _) = old_length_slot_for_match(length, offset)?; + length_frequencies[slot] += 1; + } + EncodeToken::ShortOffset { offset } => { + let (slot, _) = short_slot_for_match(offset)?; + main_frequencies[261 + slot] += 1; + } + EncodeToken::Match { length, offset } => { + let encoded_length = length - match_length_adjustment(offset); + let (slot, _) = length_slot_for_match(encoded_length)?; + main_frequencies[270 + slot] += 1; + let (offset_slot, _) = offset_slot_for_match(offset)?; + offset_frequencies[offset_slot] += 1; + } + } + } + let mut table_lengths = [0u8; TABLE_COUNT]; + let literal_len = if let Some(table) = fixed_table { + table.length + } else { + let main_symbol_count = main_frequencies + .iter() + .filter(|&&frequency| frequency != 0) + .count() + + offset_frequencies + .iter() + .filter(|&&frequency| frequency != 0) + .count() + + length_frequencies + .iter() + .filter(|&&frequency| frequency != 0) + .count(); + literal_code_len(main_symbol_count) + }; + + if fixed_table.is_some() { + for len in &mut table_lengths[..256] { + *len = literal_len; + } + table_lengths[256] = literal_len; + for len in &mut table_lengths[270..270 + LENGTH_COUNT] { + *len = literal_len; + } + for len in &mut table_lengths[257..269] { + *len = literal_len; + } + for len in &mut table_lengths[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT] { + *len = literal_len; + } + for len in &mut table_lengths[MAIN_COUNT + OFFSET_COUNT..TABLE_COUNT] { + *len = literal_len; + } + } else { + table_lengths[..MAIN_COUNT] + .copy_from_slice(&huffman::lengths_for_frequency_array(&main_frequencies, 15)); + table_lengths[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT].copy_from_slice( + &huffman::lengths_for_frequency_array(&offset_frequencies, 15), + ); + table_lengths[MAIN_COUNT + OFFSET_COUNT..TABLE_COUNT].copy_from_slice( + &huffman::lengths_for_frequency_array(&length_frequencies, 15), + ); + } + Ok(table_lengths) +} + +fn encode_member_with_tables( + tokens: &[EncodeToken], + history: &[u8], + fixed_table: Option, + table_lengths: &[u8; TABLE_COUNT], +) -> Result> { + let level_tokens = encode_table_level_tokens(table_lengths); + let level_lengths = level_code_lengths_for_tokens(&level_tokens); + let level_codes = canonical_codes(&level_lengths)?; + let main_codes = canonical_codes(&table_lengths[..MAIN_COUNT])?; + + let mut bits = BitWriter::default(); + if fixed_table.is_none() || history.is_empty() { + bits.write_bits(0, 2); // LZ block, do not keep previous tables. + for &len in &level_lengths { + bits.write_bits(len as u32, 4); + } + for token in level_tokens { + let code = level_codes[token.symbol].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing level Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + if token.extra_bits != 0 { + bits.write_bits(token.extra_value as u32, token.extra_bits); + } + } + } + let offset_codes = canonical_codes(&table_lengths[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT])?; + let length_codes = canonical_codes(&table_lengths[MAIN_COUNT + OFFSET_COUNT..TABLE_COUNT])?; + for token in tokens { + match *token { + EncodeToken::Literal(byte) => { + let code = main_codes[byte as usize].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing literal Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + } + EncodeToken::RepeatLast => { + let code = main_codes[256].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing repeat-last Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + } + EncodeToken::OldOffset { + index, + length, + offset, + } => { + let code = main_codes[257 + index].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing old-offset Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + let (slot, extra) = old_length_slot_for_match(length, offset)?; + let length_code = length_codes[slot].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing old-offset length Huffman code", + ))?; + bits.write_bits(length_code.code as u32, length_code.len); + if LENGTH_BITS[slot] != 0 { + bits.write_bits(extra as u32, LENGTH_BITS[slot]); + } + } + EncodeToken::ShortOffset { offset } => { + let (slot, extra) = short_slot_for_match(offset)?; + let code = main_codes[261 + slot].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing short-offset Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + bits.write_bits(extra as u32, SHORT_BITS[slot]); + } + EncodeToken::Match { length, offset } => { + let encoded_length = length - match_length_adjustment(offset); + let (slot, extra) = length_slot_for_match(encoded_length)?; + let code = main_codes[270 + slot].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing match Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + if LENGTH_BITS[slot] != 0 { + bits.write_bits(extra as u32, LENGTH_BITS[slot]); + } + let (offset_slot, offset_extra) = offset_slot_for_match(offset)?; + let offset = offset_codes[offset_slot].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing offset Huffman code", + ))?; + bits.write_bits(offset.code as u32, offset.len); + if OFFSET_BITS[offset_slot] != 0 { + bits.write_bits(offset_extra as u32, OFFSET_BITS[offset_slot]); + } + } + } + } + Ok(bits.finish()) +} + +#[derive(Debug, Clone, Copy)] +struct FixedEncodeTable { + length: u8, +} + +impl FixedEncodeTable { + fn new() -> Result { + Ok(Self { + length: literal_code_len(256 + LENGTH_COUNT + OFFSET_COUNT), + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EncodeToken { + Literal(u8), + RepeatLast, + OldOffset { + index: usize, + length: usize, + offset: usize, + }, + ShortOffset { + offset: usize, + }, + Match { + length: usize, + offset: usize, + }, +} + +/// Lengths worth trying for a match at one position. +/// +/// Every length inside a length slot is priced the same: the slot's Huffman +/// code plus its fixed extra bits, with the extra value itself costing nothing. +/// So the parse only needs the longest length in each slot, plus the longest +/// match found. That is at most twenty-nine candidates where trying every +/// length is up to two hundred and fifty-six, for 0.03% of packed size. +fn candidate_lengths(best_length: usize, offset: usize, out: &mut Vec) { + out.clear(); + let adjustment = match_length_adjustment(offset); + for slot in 0..LENGTH_COUNT { + let widest = LENGTH_BASES[slot] + (1usize << LENGTH_BITS[slot]) - 1; + let length = widest + 3 + adjustment; + if (3..=best_length).contains(&length) { + out.push(length); + } + } + // Callers only pass matches of at least three bytes. + out.push(best_length); + out.sort_unstable(); + out.dedup(); +} + +/// Parse the member by shortest path, priced by the previous pass's tables. +/// +/// The greedy parse commits to a match the moment it finds one, with a two +/// position lazy check as its only way out. This asks instead what the cheapest +/// route to the end of the member is, so a match that looks good on its own can +/// lose to one that leaves the positions after it cheaper. Worth about 0.9% on +/// the bench corpus, which is more than everything else tried put together. +/// +/// One approximation. The four recent offsets depend on the route taken, and +/// carrying every reachable rep state would multiply the search out of reach. +/// Each position keeps the rep list of the cheapest route that reached it, +/// which is what LZMA's optimal parser does with the same justification. +fn encode_tokens_optimal( + input: &[u8], + start: usize, + end: usize, + finder: &mut Rar20MatchFinder, + options: EncodeOptions, + cost_model: &CostModel<'_>, +) -> Vec { + const UNREACHED: u64 = u64::MAX / 4; + let span = end - start; + let mut cost = vec![UNREACHED; span + 1]; + let mut from = vec![0usize; span + 1]; + let mut token: Vec> = vec![None; span + 1]; + let mut reps = vec![[0usize; 4]; span + 1]; + let mut lengths = Vec::new(); + cost[0] = 0; + + for index in 0..span { + // Every position is reachable through the one-byte literal edge. + let pos = start + index; + let here = cost[index]; + let node_reps = reps[index]; + + let mut relax = |next: usize, price: u64, what: EncodeToken, next_reps: [usize; 4]| { + // Literal, fresh, old-offset and short-match edges all end within span. + if here + price < cost[next] { + cost[next] = here + price; + from[next] = index; + token[next] = Some(what); + reps[next] = next_reps; + } + }; + + relax( + index + 1, + cost_model.literal_bits(input, pos, 1) as u64, + EncodeToken::Literal(input[pos]), + node_reps, + ); + + let cap = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if let Some((best_length, offset)) = + best_match(input, pos, end, finder, options, Some(cost_model)) + { + let mut pushed = node_reps; + push_old_offset(&mut pushed, offset); + candidate_lengths(best_length.min(cap), offset, &mut lengths); + for &length in &lengths { + let candidate = SelectedMatch::Fresh { length, offset }; + if let Some(price) = cost_model.selected_cost(candidate) { + relax( + index + length, + price as u64, + EncodeToken::Match { length, offset }, + pushed, + ); + } + } + } + + for (rep_index, &offset) in node_reps.iter().enumerate() { + let reach = match_run_length(input, pos, offset, cap); + if reach < 3 { + continue; + } + let mut pushed = node_reps; + push_old_offset(&mut pushed, offset); + candidate_lengths(reach, offset, &mut lengths); + for &length in &lengths { + let candidate = SelectedMatch::OldOffset { + index: rep_index, + length, + offset, + }; + if let Some(price) = cost_model.selected_cost(candidate) { + relax( + index + length, + price as u64, + EncodeToken::OldOffset { + index: rep_index, + length, + offset, + }, + pushed, + ); + } + } + } + + if let Some(SelectedMatch::ShortOffset { offset }) = + best_short_offset_match(input, pos, end) + { + let slot = short_slot_index(offset); + let price = usize::from(cost_model.main[261 + slot]) + usize::from(SHORT_BITS[slot]); + let mut pushed = node_reps; + push_old_offset(&mut pushed, offset); + relax( + index + 2, + price as u64, + EncodeToken::ShortOffset { offset }, + pushed, + ); + } + + finder.insert(input, pos); + } + + let mut out = Vec::new(); + let mut at = span; + while at > 0 { + let what = token[at].expect("every parse position has a literal edge"); + out.push(what); + at = from[at]; + } + out.reverse(); + out +} + +/// How far the bytes at `pos` repeat the bytes `offset` back, up to `cap`. +fn match_run_length(input: &[u8], pos: usize, offset: usize, cap: usize) -> usize { + if offset == 0 { + return 0; + } + let mut length = 0; + while length < cap && input[pos + length] == input[pos - offset + length] { + length += 1; + } + length +} + +fn encode_tokens_with_progress( + input: &[u8], + history: &[u8], + options: EncodeOptions, + cost_model: Option<&CostModel>, + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let mut tokens = Vec::new(); + let history = &history[history.len().saturating_sub(options.max_match_distance)..]; + let mut combined = Vec::with_capacity(history.len() + input.len()); + combined.extend_from_slice(history); + combined.extend_from_slice(input); + let mut finder = Rar20MatchFinder::new(combined.len()); + for history_pos in 0..history.len() { + finder.insert(&combined, history_pos); + } + + if let Some(cost_model) = cost_model.filter(|_| options.optimal_parse) { + let start = history.len(); + let end = combined.len(); + return Ok(encode_tokens_optimal( + &combined, + start, + end, + &mut finder, + options, + cost_model, + )); + } + + let mut pos = history.len(); + let end = combined.len(); + let mut last_match = None; + let mut old_offsets = [0usize; 4]; + let mut next_report = 0usize; + while pos < end { + let selected = select_match( + &combined, + pos, + end, + &finder, + options, + &old_offsets, + cost_model, + ); + if let Some(selected) = selected { + let lazy = LazyMatchContext { + input: &combined, + end, + finder: &finder, + options, + old_offsets: &old_offsets, + cost_model, + }; + if should_lazy_emit_literal(pos, selected, lazy) { + tokens.push(EncodeToken::Literal(combined[pos])); + finder.insert(&combined, pos); + pos += 1; + continue; + } + let (length, offset) = match selected { + SelectedMatch::Fresh { length, offset } => { + if last_match == Some((length, offset)) { + tokens.push(EncodeToken::RepeatLast); + } else { + tokens.push(EncodeToken::Match { length, offset }); + last_match = Some((length, offset)); + } + (length, offset) + } + SelectedMatch::OldOffset { + index, + length, + offset, + } => { + if last_match == Some((length, offset)) { + tokens.push(EncodeToken::RepeatLast); + } else { + tokens.push(EncodeToken::OldOffset { + index, + length, + offset, + }); + last_match = Some((length, offset)); + } + (length, offset) + } + SelectedMatch::ShortOffset { offset } => { + let length = 2; + tokens.push(EncodeToken::ShortOffset { offset }); + last_match = Some((length, offset)); + (length, offset) + } + }; + push_old_offset(&mut old_offsets, offset); + for history_pos in pos..pos + length { + finder.insert(&combined, history_pos); + } + pos += length; + } else { + tokens.push(EncodeToken::Literal(combined[pos])); + finder.insert(&combined, pos); + pos += 1; + } + let consumed = pos.saturating_sub(history.len()); + if consumed >= next_report { + if progress + .as_deref_mut() + .is_some_and(|report| !report(consumed)) + { + return Err(Error::Cancelled); + } + next_report = consumed.saturating_add(1024 * 1024); + } + } + if progress.is_some_and(|report| !report(input.len())) { + return Err(Error::Cancelled); + } + Ok(tokens) +} + +/// Stand-in price for a literal the current table has no code for. +const ABSENT_LITERAL_BITS: usize = 15; + +#[derive(Debug, Clone, Copy)] +struct CostModel<'a> { + main: &'a [u8], + offsets: &'a [u8], + lengths: &'a [u8], +} + +impl<'a> CostModel<'a> { + fn new(table_lengths: &'a [u8; TABLE_COUNT]) -> Self { + Self { + main: &table_lengths[..MAIN_COUNT], + offsets: &table_lengths[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT], + lengths: &table_lengths[MAIN_COUNT + OFFSET_COUNT..TABLE_COUNT], + } + } + + fn selected_cost(self, selected: SelectedMatch) -> Option { + match selected { + SelectedMatch::Fresh { length, offset } => { + let encoded_length = length.checked_sub(match_length_adjustment(offset))?; + let (length_slot, _) = length_slot_for_match(encoded_length).ok()?; + let (offset_slot, _) = offset_slot_for_match(offset).ok()?; + Some( + usize::from(self.main[270 + length_slot]) + + usize::from(LENGTH_BITS[length_slot]) + + usize::from(self.offsets[offset_slot]) + + usize::from(OFFSET_BITS[offset_slot]), + ) + } + SelectedMatch::OldOffset { + index, + length, + offset, + } => { + let (length_slot, _) = old_length_slot_for_match(length, offset).ok()?; + Some( + usize::from(self.main[257 + index]) + + usize::from(self.lengths[length_slot]) + + usize::from(LENGTH_BITS[length_slot]), + ) + } + SelectedMatch::ShortOffset { offset } => { + let (slot, _) = short_slot_for_match(offset).ok()?; + Some(usize::from(self.main[261 + slot]) + usize::from(SHORT_BITS[slot])) + } + } + } + + /// What these bytes cost spelled out one literal at a time, in bits. + /// + /// A symbol the previous pass never emitted as a literal has no code, so it + /// cannot be spelled that way at all. Price it high rather than free. + fn literal_bits(self, input: &[u8], pos: usize, length: usize) -> usize { + input[pos..(pos + length).min(input.len())] + .iter() + .map(|&byte| match self.main[usize::from(byte)] { + 0 => ABSENT_LITERAL_BITS, + bits => usize::from(bits), + }) + .sum() + } + + /// Bits saved by taking this match instead of the literals it covers. + /// + /// The literals are priced from the table rather than assumed to be eight + /// bits each. On text they run nearer four, so a flat eight overstates what + /// every match is worth by around half. + fn selected_score(self, selected: SelectedMatch, input: &[u8], pos: usize) -> Option { + let cost = self.selected_cost(selected)?; + let saved = self.literal_bits(input, pos, selected.length()); + Some(saved as isize - cost as isize) + } +} + +#[derive(Debug, Clone, Copy)] +enum SelectedMatch { + Fresh { + length: usize, + offset: usize, + }, + OldOffset { + index: usize, + length: usize, + offset: usize, + }, + ShortOffset { + offset: usize, + }, +} + +impl SelectedMatch { + fn length(self) -> usize { + match self { + SelectedMatch::Fresh { length, .. } | SelectedMatch::OldOffset { length, .. } => length, + SelectedMatch::ShortOffset { .. } => 2, + } + } + + fn score(self) -> isize { + let length_score = self.length() as isize * 8; + let cost = match self { + SelectedMatch::OldOffset { .. } | SelectedMatch::ShortOffset { .. } => 4, + SelectedMatch::Fresh { offset, .. } => 8 + OFFSET_BITS[offset_slot_index(offset)], + }; + length_score - isize::from(cost) + } +} + +fn select_match( + input: &[u8], + pos: usize, + end: usize, + finder: &Rar20MatchFinder, + options: EncodeOptions, + old_offsets: &[usize; 4], + cost_model: Option<&CostModel<'_>>, +) -> Option { + let fresh = best_match(input, pos, end, finder, options, cost_model); + let old = best_old_offset_match(input, pos, end, old_offsets, cost_model); + if let Some(cost_model) = cost_model { + return [ + fresh.map(|(length, offset)| SelectedMatch::Fresh { length, offset }), + old.map(|(index, length, offset)| SelectedMatch::OldOffset { + index, + length, + offset, + }), + best_short_offset_match(input, pos, end), + ] + .into_iter() + .flatten() + .max_by_key(|&selected| { + ( + cost_model + .selected_score(selected, input, pos) + .unwrap_or(isize::MIN), + selected.length(), + ) + }); + } + + match (fresh, old) { + (Some((fresh_length, _)), Some((index, old_length, old_offset))) + if old_length + 1 >= fresh_length => + { + Some(SelectedMatch::OldOffset { + index, + length: old_length, + offset: old_offset, + }) + } + (Some((length, offset)), _) => Some(SelectedMatch::Fresh { length, offset }), + (None, Some((index, length, offset))) => Some(SelectedMatch::OldOffset { + index, + length, + offset, + }), + (None, None) => best_short_offset_match(input, pos, end), + } +} + +struct LazyMatchContext<'a> { + input: &'a [u8], + end: usize, + finder: &'a Rar20MatchFinder, + options: EncodeOptions, + old_offsets: &'a [usize; 4], + cost_model: Option<&'a CostModel<'a>>, +} + +fn should_lazy_emit_literal( + pos: usize, + current: SelectedMatch, + context: LazyMatchContext<'_>, +) -> bool { + if !context.options.lazy_matching { + return false; + } + let lookahead = context.options.lazy_lookahead.max(1); + (1..=lookahead) + .take_while(|offset| pos + offset < context.end) + .any(|offset| { + select_match( + context.input, + pos + offset, + context.end, + context.finder, + context.options, + context.old_offsets, + context.cost_model, + ) + .is_some_and(|next| { + let current_score = context + .cost_model + .and_then(|cost_model| cost_model.selected_score(current, context.input, pos)) + .unwrap_or_else(|| current.score()); + let next_score = context + .cost_model + .and_then(|cost_model| { + cost_model.selected_score(next, context.input, pos + offset) + }) + .unwrap_or_else(|| next.score()); + let skipped_literal_score = context + .cost_model + .map_or(offset as isize * 8, |cost_model| { + cost_model.literal_bits(context.input, pos, offset) as isize + }); + next_score > current_score + skipped_literal_score + }) + }) +} + +fn best_match( + input: &[u8], + pos: usize, + end: usize, + finder: &Rar20MatchFinder, + options: EncodeOptions, + cost_model: Option<&CostModel<'_>>, +) -> Option<(usize, usize)> { + let max_offset = pos.min(options.max_match_distance); + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if options.max_match_candidates == 0 || max_offset == 0 || max_length < 3 { + return None; + } + let mut best = None; + let mut checked = 0usize; + let mut candidate = finder.first(input, pos); + while candidate != match_finder::NO_POSITION { + // Callers search before inserting this position, so every candidate is older. + let offset = pos - candidate; + if offset > max_offset { + break; + } + checked += 1; + // Without a cost model, a candidate can only improve on the current + // best when it matches at least one byte past the best length, so + // probe that byte first. The cost-model pass may prefer shorter but + // cheaper matches, so it must evaluate every candidate. Probing is + // safe because a best match reaching `max_length` breaks the loop. + let best_length = best.map_or(0, |(length, _)| length); + let probe_ok = cost_model.is_some() + || best_length == 0 + || input[candidate + best_length] == input[pos + best_length]; + if probe_ok { + let length = super::super::fast::match_length(input, pos, offset, max_length); + let encodable = length >= 3 + match_length_adjustment(offset); + if encodable && is_better_fresh_match(cost_model, input, pos, length, offset, best) { + best = Some((length, offset)); + if length == max_length { + break; + } + } + } + if checked >= options.max_match_candidates { + break; + } + candidate = finder.previous(candidate); + } + best +} + +fn offset_slot_index(offset: usize) -> usize { + offset_slot_for_match(offset) + .map(|(slot, _)| slot) + .unwrap_or(OFFSET_BITS.len() - 1) +} + +fn is_better_fresh_match( + cost_model: Option<&CostModel<'_>>, + input: &[u8], + pos: usize, + length: usize, + offset: usize, + best: Option<(usize, usize)>, +) -> bool { + let Some((best_length, best_offset)) = best else { + return true; + }; + if let Some(cost_model) = cost_model { + let candidate = SelectedMatch::Fresh { length, offset }; + let best = SelectedMatch::Fresh { + length: best_length, + offset: best_offset, + }; + let candidate_score = cost_model + .selected_score(candidate, input, pos) + .unwrap_or(isize::MIN); + let best_score = cost_model + .selected_score(best, input, pos) + .unwrap_or(isize::MIN); + return candidate_score > best_score + || (candidate_score == best_score && length > best_length); + } + // The hash chain visits nearest matches first, so an equal-length later + // candidate cannot have a smaller offset. + length > best_length +} + +fn best_old_offset_match( + input: &[u8], + pos: usize, + end: usize, + old_offsets: &[usize; 4], + cost_model: Option<&CostModel<'_>>, +) -> Option<(usize, usize, usize)> { + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + let mut best = None; + for (index, &offset) in old_offsets.iter().enumerate() { + if offset == 0 { + continue; + } + let length = match_length_at_offset(input, pos, max_length, offset); + if old_length_slot_for_match(length, offset).is_ok() + && is_better_old_offset_match(cost_model, input, pos, index, length, offset, best) + { + best = Some((index, length, offset)); + } + } + best +} + +fn is_better_old_offset_match( + cost_model: Option<&CostModel<'_>>, + input: &[u8], + pos: usize, + index: usize, + length: usize, + offset: usize, + best: Option<(usize, usize, usize)>, +) -> bool { + let Some((best_index, best_length, best_offset)) = best else { + return true; + }; + if let Some(cost_model) = cost_model { + let candidate = SelectedMatch::OldOffset { + index, + length, + offset, + }; + let best = SelectedMatch::OldOffset { + index: best_index, + length: best_length, + offset: best_offset, + }; + let candidate_score = cost_model + .selected_score(candidate, input, pos) + .unwrap_or(isize::MIN); + let best_score = cost_model + .selected_score(best, input, pos) + .unwrap_or(isize::MIN); + return candidate_score > best_score + || (candidate_score == best_score + && (length > best_length || (length == best_length && offset < best_offset))); + } + length > best_length || (length == best_length && offset < best_offset) +} + +fn best_short_offset_match(input: &[u8], pos: usize, end: usize) -> Option { + if end - pos < 2 { + return None; + } + let max_offset = pos.min(256); + (1..=max_offset) + .find(|&offset| { + input[pos] == input[pos - offset] && input[pos + 1] == input[pos + 1 - offset] + }) + .map(|offset| SelectedMatch::ShortOffset { offset }) +} + +fn match_length_at_offset(input: &[u8], pos: usize, max_length: usize, offset: usize) -> usize { + super::super::fast::match_length(input, pos, offset, max_length) +} + +fn match_length_adjustment(offset: usize) -> usize { + usize::from(offset >= 0x2000) + usize::from(offset >= 0x40000) +} + +fn old_length_adjustment(offset: usize) -> usize { + usize::from(offset >= 0x101) + usize::from(offset >= 0x2000) + usize::from(offset >= 0x40000) +} + +fn push_old_offset(old_offsets: &mut [usize; 4], offset: usize) { + old_offsets[3] = old_offsets[2]; + old_offsets[2] = old_offsets[1]; + old_offsets[1] = old_offsets[0]; + old_offsets[0] = offset; +} + +fn length_slot_for_match(length: usize) -> Result<(usize, usize)> { + if length < 3 { + return Err(Error::InvalidData("RAR 2.0 match length is too short")); + } + if length > MAX_ENCODER_MATCH_LENGTH { + return Err(Error::InvalidData("RAR 2.0 match length is too long")); + } + let adjusted = length - 3; + let slot = LENGTH_BASES.partition_point(|&base| base <= adjusted) - 1; + Ok((slot, adjusted - LENGTH_BASES[slot])) +} + +fn old_length_slot_for_match(length: usize, offset: usize) -> Result<(usize, usize)> { + let encoded = length + .checked_sub(old_length_adjustment(offset)) + .ok_or(Error::InvalidData( + "RAR 2.0 adjusted old-offset length underflows", + ))?; + if encoded < 2 { + return Err(Error::InvalidData( + "RAR 2.0 old-offset match length is too short", + )); + } + let adjusted = encoded - 2; + if adjusted > 255 { + return Err(Error::InvalidData( + "RAR 2.0 old-offset match length is too long", + )); + } + let slot = LENGTH_BASES.partition_point(|&base| base <= adjusted) - 1; + Ok((slot, adjusted - LENGTH_BASES[slot])) +} + +fn offset_slot_for_match(offset: usize) -> Result<(usize, usize)> { + if offset == 0 { + return Err(Error::InvalidData("RAR 2.0 match offset is zero")); + } + if offset > MAX_ENCODER_MATCH_OFFSET { + return Err(Error::InvalidData("RAR 2.0 match offset is too large")); + } + let adjusted = offset - 1; + let slot = OFFSET_BASES.partition_point(|&base| base <= adjusted) - 1; + Ok((slot, adjusted - OFFSET_BASES[slot])) +} + +fn short_slot_for_match(offset: usize) -> Result<(usize, usize)> { + if offset == 0 || offset > 256 { + return Err(Error::InvalidData( + "RAR 2.0 short match offset is out of range", + )); + } + let adjusted = offset - 1; + let slot = short_slot_index(offset); + Ok((slot, adjusted - SHORT_BASES[slot])) +} + +fn short_slot_index(offset: usize) -> usize { + let adjusted = offset - 1; + SHORT_BASES.partition_point(|&base| base <= adjusted) - 1 +} + +fn literal_code_len(symbol_count: usize) -> u8 { + // A nonempty member always emits at least one token and there are at most + // TABLE_COUNT distinct symbols, so the result fits in u8. + let len = usize::BITS - (symbol_count - 1).leading_zeros(); + len.max(1) as u8 +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct LevelToken { + symbol: usize, + extra_bits: u8, + extra_value: u8, +} + +impl LevelToken { + const fn plain(symbol: usize) -> Self { + Self { + symbol, + extra_bits: 0, + extra_value: 0, + } + } + + const fn repeat_previous(count: usize) -> Self { + Self { + symbol: 16, + extra_bits: 2, + extra_value: (count - 3) as u8, + } + } + + const fn zero_run_short(count: usize) -> Self { + Self { + symbol: 17, + extra_bits: 3, + extra_value: (count - 3) as u8, + } + } + + const fn zero_run_long(count: usize) -> Self { + Self { + symbol: 18, + extra_bits: 7, + extra_value: (count - 11) as u8, + } + } +} + +fn encode_table_level_tokens(lengths: &[u8; TABLE_COUNT]) -> Vec { + encode_level_tokens(lengths) +} + +fn encode_level_tokens(lengths: &[u8]) -> Vec { + let mut tokens = Vec::new(); + let mut pos = 0usize; + let mut previous = None; + while pos < lengths.len() { + let value = lengths[pos]; + let mut run = 1usize; + while pos + run < lengths.len() && lengths[pos + run] == value { + run += 1; + } + + if value == 0 { + emit_zero_level_run(&mut tokens, run); + previous = Some(0); + pos += run; + continue; + } + + if previous == Some(value) && run >= 3 { + let mut remaining = run; + while remaining != 0 { + let chunk = remaining.min(6); + if chunk >= 3 { + tokens.push(LevelToken::repeat_previous(chunk)); + remaining -= chunk; + } else { + tokens.extend(std::iter::repeat_n( + LevelToken::plain(value as usize), + chunk, + )); + remaining = 0; + } + } + pos += run; + continue; + } + + tokens.push(LevelToken::plain(value as usize)); + previous = Some(value); + pos += 1; + } + tokens +} + +fn emit_zero_level_run(tokens: &mut Vec, mut run: usize) { + while run != 0 { + if run >= 11 { + let mut chunk = run.min(138); + if matches!(run - chunk, 1 | 2) { + chunk -= 3; + } + tokens.push(LevelToken::zero_run_long(chunk)); + run -= chunk; + } else if run >= 3 { + let chunk = run.min(10); + tokens.push(LevelToken::zero_run_short(chunk)); + run -= chunk; + } else { + tokens.extend(std::iter::repeat_n(LevelToken::plain(0), run)); + break; + } + } +} + +fn level_code_lengths_for_tokens(tokens: &[LevelToken]) -> [u8; LEVEL_COUNT] { + let mut used = [false; LEVEL_COUNT]; + for token in tokens { + used[token.symbol] = true; + } + level_code_lengths_for_used_symbols(used) +} + +fn encode_audio_member(input: &[u8], channels: usize) -> Result> { + if channels == 0 || channels > MAX_CHANNELS { + return Err(Error::InvalidData("RAR 2.0 audio channel count is invalid")); + } + let deltas = audio_encode(input, channels); + let mut levels = vec![0u8; AUDIO_COUNT * channels]; + for channel in 0..channels { + let mut frequencies = [0usize; AUDIO_COUNT]; + for index in (channel..deltas.len()).step_by(channels) { + frequencies[deltas[index] as usize] += 1; + } + let channel_lengths = huffman::lengths_for_frequency_array(&frequencies, 15); + for (symbol, len) in channel_lengths.into_iter().enumerate() { + levels[channel * AUDIO_COUNT + symbol] = len; + } + } + + let level_symbols = encode_audio_table_level_symbols(&levels); + let level_lengths = level_code_lengths_for_symbols(&level_symbols); + let level_codes = canonical_codes(&level_lengths)?; + let mut bits = BitWriter::default(); + bits.write_bits(0b10, 2); // audio block, do not keep previous tables. + bits.write_bits((channels - 1) as u32, 2); + for &len in &level_lengths { + bits.write_bits(len as u32, 4); + } + for symbol in level_symbols { + let code = level_codes[symbol].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing audio-level Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + // Audio levels are emitted directly as 0..=15, without run symbols. + } + + let audio_codes = (0..channels) + .map(|channel| canonical_codes(&levels[channel * AUDIO_COUNT..(channel + 1) * AUDIO_COUNT])) + .collect::>>()?; + for (index, &delta) in deltas.iter().enumerate() { + let channel = index % channels; + let code = audio_codes[channel][delta as usize].ok_or(Error::InvalidData( + "RAR 2.0 encoder missing audio Huffman code", + ))?; + bits.write_bits(code.code as u32, code.len); + } + Ok(bits.finish()) +} + +fn encode_audio_table_level_symbols(levels: &[u8]) -> Vec { + levels.iter().map(|&len| len as usize).collect() +} + +fn level_code_lengths_for_symbols(symbols: &[usize]) -> [u8; LEVEL_COUNT] { + let mut used = [false; LEVEL_COUNT]; + for &symbol in symbols { + used[symbol] = true; + } + level_code_lengths_for_used_symbols(used) +} + +fn level_code_lengths_for_used_symbols(used: [bool; LEVEL_COUNT]) -> [u8; LEVEL_COUNT] { + let mut lengths = [0u8; LEVEL_COUNT]; + for (symbol, is_used) in used.into_iter().enumerate() { + if is_used { + lengths[symbol] = 1; + } + } + huffman::assign_flat_complete_code(&mut lengths); + lengths +} + +fn audio_encode(input: &[u8], channels: usize) -> Vec { + let mut states = [AudioState::default(); MAX_CHANNELS]; + let mut channel_delta = 0i32; + let mut deltas = Vec::with_capacity(input.len()); + for (index, &byte) in input.iter().enumerate() { + let channel = index % channels; + let state = &mut states[channel]; + state.byte_count = state.byte_count.wrapping_add(1); + state.d4 = state.d3; + state.d3 = state.d2; + state.d2 = state.last_delta - state.d1; + state.d1 = state.last_delta; + + let predicted = 8 * state.last_char + + state.k[0] * state.d1 + + state.k[1] * state.d2 + + state.k[2] * state.d3 + + state.k[3] * state.d4 + + state.k[4] * channel_delta; + let predicted = (predicted >> 3) & 0xff; + let delta = (predicted as u8).wrapping_sub(byte); + + let d = (delta as i8 as i32) << 3; + state.dif[0] = state.dif[0].wrapping_add(d.unsigned_abs()); + state.dif[1] = state.dif[1].wrapping_add((d - state.d1).unsigned_abs()); + state.dif[2] = state.dif[2].wrapping_add((d + state.d1).unsigned_abs()); + state.dif[3] = state.dif[3].wrapping_add((d - state.d2).unsigned_abs()); + state.dif[4] = state.dif[4].wrapping_add((d + state.d2).unsigned_abs()); + state.dif[5] = state.dif[5].wrapping_add((d - state.d3).unsigned_abs()); + state.dif[6] = state.dif[6].wrapping_add((d + state.d3).unsigned_abs()); + state.dif[7] = state.dif[7].wrapping_add((d - state.d4).unsigned_abs()); + state.dif[8] = state.dif[8].wrapping_add((d + state.d4).unsigned_abs()); + state.dif[9] = state.dif[9].wrapping_add((d - channel_delta).unsigned_abs()); + state.dif[10] = state.dif[10].wrapping_add((d + channel_delta).unsigned_abs()); + + channel_delta = (byte.wrapping_sub(state.last_char as u8)) as i8 as i32; + state.last_delta = channel_delta; + state.last_char = byte as i32; + + if state.byte_count & 0x1f == 0 { + let mut min_dif = state.dif[0]; + let mut num_min_dif = 0usize; + state.dif[0] = 0; + for diff_index in 1..state.dif.len() { + if state.dif[diff_index] < min_dif { + min_dif = state.dif[diff_index]; + num_min_dif = diff_index; + } + state.dif[diff_index] = 0; + } + match num_min_dif { + 1 if state.k[0] >= -16 => state.k[0] -= 1, + 2 if state.k[0] < 16 => state.k[0] += 1, + 3 if state.k[1] >= -16 => state.k[1] -= 1, + 4 if state.k[1] < 16 => state.k[1] += 1, + 5 if state.k[2] >= -16 => state.k[2] -= 1, + 6 if state.k[2] < 16 => state.k[2] += 1, + 7 if state.k[3] >= -16 => state.k[3] -= 1, + 8 if state.k[3] < 16 => state.k[3] += 1, + 9 if state.k[4] >= -16 => state.k[4] -= 1, + 10 if state.k[4] < 16 => state.k[4] += 1, + _ => {} + } + } + + deltas.push(delta); + } + deltas +} + +#[derive(Debug, Clone, Copy)] +struct HuffmanCode { + code: u16, + len: u8, +} + +fn canonical_codes(lengths: &[u8]) -> Result>> { + let mut count = [0u16; 16]; + for &len in lengths { + if len > 15 { + return Err(Error::InvalidData("RAR 2.0 Huffman length is too large")); + } + if len != 0 { + count[len as usize] += 1; + } + } + validate_huffman_counts(&count)?; + + let mut next_code = [0u16; 16]; + let mut code = 0u16; + for len in 1..=15 { + code = (code + count[len - 1]) << 1; + next_code[len] = code; + } + + let mut codes = vec![None; lengths.len()]; + for (symbol, &len) in lengths.iter().enumerate() { + if len == 0 { + continue; + } + let code = next_code[len as usize]; + next_code[len as usize] += 1; + codes[symbol] = Some(HuffmanCode { code, len }); + } + Ok(codes) +} + +#[derive(Default)] +struct BitWriter { + bytes: Vec, + bit_pos: usize, +} + +impl BitWriter { + fn write_bits(&mut self, value: u32, count: u8) { + super::super::fast::write_msb_bits( + &mut self.bytes, + &mut self.bit_pos, + u64::from(value), + usize::from(count), + ); + } + + #[cfg(test)] + fn write_bit(&mut self, bit: bool) { + self.write_bits(u32::from(bit), 1); + } + + fn finish(self) -> Vec { + self.bytes + } +} + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/arca-rar/vendor/rars/src/codec/rar20/tests.rs b/arca-rar/vendor/rars/src/codec/rar20/tests.rs new file mode 100644 index 00000000..482faa52 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar20/tests.rs @@ -0,0 +1,1227 @@ +#[test] +fn reader20_workspace_refusals_release_lz_audio_input_and_checkpoints() { + use crate::codec::workspace::RefusingBudget; + let data = b"resource accounting for legacy audio and LZ\n".repeat(24); + let streams = [ + super::unpack20_encode_literals(&data).unwrap(), + super::encode_audio_member(&data, 4).unwrap(), + ]; + for packed in streams { + for streaming in [false, true] { + let run = |budget: &RefusingBudget| -> super::Result<()> { + let mut decoder = super::Reader20State::with_allowance(budget); + if streaming { + let mut output = super::Buffer::new(budget); + decoder.decode_member_from_reader( + &mut packed.as_slice(), + data.len(), + &mut output, + )?; + assert_eq!(&*output, &data); + } else { + let output = decoder.decode_member_owned(&packed, data.len())?; + assert_eq!(&*output, &data); + } + let checkpoint = decoder.try_clone()?; + assert_eq!(checkpoint.output, decoder.output); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + let attempts = baseline.attempts(); + assert!(attempts > 10); + assert_eq!(baseline.used(), 0); + for index in 0..attempts { + let budget = RefusingBudget::new(index); + assert!( + matches!(run(&budget), Err(Error::Cancelled)), + "allocation {index}, streaming={streaming}" + ); + assert_eq!(budget.used(), 0); + } + } + } +} + +#[test] +fn reader20_workspace_keeps_returned_data_charged_after_decoder_drop() { + let data = b"owned legacy result".repeat(32); + let packed = unpack20_encode_literals(&data).unwrap(); + let ledger = super::Allowance::limited(256 * 1024); + let mut decoder = super::Reader20State::with_allowance(&ledger); + let output = decoder.decode_member_owned(&packed, data.len()).unwrap(); + assert!(ledger.used() > output.capacity() as u64); + drop(decoder); + assert_eq!(ledger.used(), output.capacity() as u64); + assert_eq!(&*output, &data); + drop(output); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn cancellation_interrupts_buffered_symbol_work() { + let data = b"cancellable legacy symbols ".repeat(16384); + let packed = unpack20_encode_literals(&data).unwrap(); + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack20::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + decoder.read_control.cancel_after_checks(3); + assert_eq!( + decoder.decode_member(&packed, data.len()).unwrap_err(), + Error::Cancelled + ); + assert!(decoder.current_pos() > 0 && decoder.current_pos() < data.len()); +} +type Unpack20 = super::Reader20State; +use super::{ + encode_tokens_with_progress, level_code_lengths_for_used_symbols, unpack20_decode, + unpack20_encode_literals, BitWriter, CostModel, EncodeOptions, EncodeToken, Error, Huffman, + Unpack20Encoder, LEVEL_COUNT, +}; + +/// 7-Zip builds the RAR pre-table with `k_BuildMode_Full` and refuses a +/// code that leaves part of the code space unassigned, where unrar takes +/// it. Giving every used symbol the same length only fills the space when +/// the count is a power of two, so the flat table we used to emit was +/// under-full for 3, 5, 6, 7, 9 symbols and so on, and 7-Zip rejected the +/// archive before decoding a byte. +#[test] +fn the_pre_table_fills_its_code_space_for_every_symbol_count() { + for count in 1..=LEVEL_COUNT { + let mut used = [false; LEVEL_COUNT]; + for slot in used.iter_mut().take(count) { + *slot = true; + } + let lengths = level_code_lengths_for_used_symbols(used); + + let longest = lengths.iter().copied().max().unwrap(); + let kraft: u32 = lengths + .iter() + .filter(|&&len| len != 0) + .map(|&len| 1u32 << (longest - len)) + .sum(); + assert_eq!( + kraft, + 1 << longest, + "{count} symbols gave the incomplete code {lengths:?}" + ); + } +} + +fn encode_tokens( + input: &[u8], + history: &[u8], + options: EncodeOptions, + cost_model: Option<&CostModel>, +) -> Vec { + encode_tokens_with_progress(input, history, options, cost_model, None) + .expect("encoding without cancellation cannot be cancelled") +} + +const AUTOREJ_PACKED: &[u8] = &[ + 0x09, 0x14, 0x0c, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0xce, 0xf8, 0x1f, 0xc1, 0xe6, 0x05, 0xfc, + 0x39, 0xc3, 0x50, 0x65, 0x08, 0x41, 0x94, 0xc4, 0x1d, 0xf3, 0xcd, 0x0d, 0x8e, 0x20, 0xf5, 0x9d, + 0x8e, 0x76, 0x1d, 0xc5, 0x19, 0xde, 0x16, 0x5b, 0x52, 0xb8, 0x8e, 0x75, 0xcd, 0xaf, 0x1f, 0xfc, + 0x9e, 0xf7, 0x00, 0x01, 0xbe, 0x90, +]; + +#[test] +fn decodes_rar20_lz_member() { + assert_eq!( + unpack20_decode(AUTOREJ_PACKED, expected_text().len()).unwrap(), + expected_text() + ); +} + +#[test] +fn rejects_oversubscribed_rar20_huffman_tables() { + assert!(matches!( + Huffman::from_lengths(&[1, 1, 1]), + Err(Error::InvalidData("RAR 2.0 oversubscribed Huffman table")) + )); +} + +#[test] +fn rejects_an_empty_main_huffman_table() { + let mut bits = BitWriter::default(); + bits.write_bits(0, 2); // LZ block with fresh tables. + for symbol in 0..LEVEL_COUNT { + bits.write_bits(u32::from(symbol == 0 || symbol == 18), 4); + } + for run in [138u32, 138, 98] { + bits.write_bit(true); // Pre-table symbol 18. + bits.write_bits(run - 11, 7); + } + + assert_eq!( + Unpack20::new() + .decode_member(&bits.finish(), 1) + .unwrap_err(), + Error::InvalidData("RAR 2.0 empty Huffman table") + ); +} + +#[test] +fn internal_bit_and_huffman_helpers_reject_out_of_range_requests() { + assert!(matches!( + Huffman::from_lengths(&[16]), + Err(Error::InvalidData("RAR 2.0 Huffman length is too large")) + )); + assert!(matches!( + super::canonical_codes(&[16]), + Err(Error::InvalidData("RAR 2.0 Huffman length is too large")) + )); + assert_eq!( + super::BitReader::new().peek_bits(25).unwrap_err(), + Error::InvalidData("RAR 2.0 bit read is too wide") + ); +} + +#[test] +fn decode_member_from_reader_accepts_incremental_input() { + struct TinyReader<'a> { + input: &'a [u8], + } + + impl std::io::Read for TinyReader<'_> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.input.is_empty() { + return Ok(0); + } + let len = self.input.len().min(out.len()).min(3); + out[..len].copy_from_slice(&self.input[..len]); + self.input = &self.input[len..]; + Ok(len) + } + } + + let mut decoder = Unpack20::new(); + let mut reader = TinyReader { + input: AUTOREJ_PACKED, + }; + let mut output = Vec::new(); + decoder + .decode_member_from_reader(&mut reader, expected_text().len(), &mut output) + .unwrap(); + + assert_eq!(output, expected_text()); +} + +#[test] +fn decode_member_from_reader_rejects_a_truncated_payload() { + for end in [AUTOREJ_PACKED.len() / 2, AUTOREJ_PACKED.len() - 1] { + let mut decoder = Unpack20::new(); + let mut packed = &AUTOREJ_PACKED[..end]; + assert_eq!( + decoder + .decode_member_from_reader(&mut packed, expected_text().len(), &mut Vec::new()) + .unwrap_err(), + Error::InvalidData("RAR 2.0 bitstream is truncated") + ); + } +} + +#[test] +fn decode_member_from_reader_accepts_an_empty_member() { + let mut decoder = Unpack20::new(); + let mut input = &[][..]; + let mut output = Vec::new(); + decoder + .decode_member_from_reader(&mut input, 0, &mut output) + .unwrap(); + assert!(output.is_empty()); +} + +#[test] +fn decoder_reports_truncated_tables_and_member_payloads() { + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&[0], 1).unwrap_err(), + Error::InvalidData("RAR 2.0 bitstream is truncated") + ); + + let input = expected_text(); + for packed in [ + &AUTOREJ_PACKED[..1], + &AUTOREJ_PACKED[..AUTOREJ_PACKED.len() / 2], + ] { + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(packed, input.len()).unwrap_err(), + Error::InvalidData("RAR 2.0 bitstream is truncated") + ); + } +} + +#[test] +fn rejects_a_table_repeat_without_a_previous_level() { + let mut bits = BitWriter::default(); + bits.write_bits(0, 2); // LZ, new tables. + for symbol in 0..LEVEL_COUNT { + bits.write_bits(u32::from(symbol == 0 || symbol == 16), 4); + } + bits.write_bit(true); // Pre-table symbol 16 at position zero. + assert_eq!( + Unpack20::new() + .decode_member(&bits.finish(), 1) + .unwrap_err(), + Error::InvalidData("RAR 2.0 table repeat at start") + ); +} + +#[test] +fn empty_member_does_not_try_to_decode_an_absent_main_table() { + let mut decoder = Unpack20::new(); + assert_eq!(decoder.decode_member(&[0; 5], 0).unwrap(), b""); + + decoder.audio_block = true; + assert_eq!(decoder.decode_member(&[0; 5], 0).unwrap(), b""); +} + +#[test] +fn audio_table_resets_the_channel_when_channel_count_shrinks() { + let mut decoder = Unpack20::new(); + decoder.channels = 4; + decoder.cur_channel = 3; + assert_eq!( + decoder.decode_member(&synthetic_audio_block(1), 1).unwrap(), + b"\0" + ); + assert_eq!(decoder.cur_channel, 0); +} + +#[test] +fn audio_lookahead_keeps_a_block_without_an_end_marker() { + let mut packed = synthetic_audio_block(4); + packed.extend_from_slice(&[0; 5]); + let mut decoder = Unpack20::new(); + assert_eq!(decoder.decode_member(&packed, 4).unwrap(), vec![0; 4]); + assert!(decoder.in_block); +} + +#[test] +fn audio_predictor_coefficients_stop_at_their_format_limits() { + for winning_difference in 1..=10 { + let mut decoder = Unpack20::new(); + let state = &mut decoder.audio[0]; + state.byte_count = 31; + state.dif = [u32::MAX / 4; 11]; + state.dif[winning_difference] = 0; + let coefficient = (winning_difference - 1) / 2; + state.k[coefficient] = if winning_difference % 2 == 1 { -17 } else { 16 }; + let before = state.k; + + decoder.decode_audio(0); + assert_eq!( + decoder.audio[0].k, before, + "difference {winning_difference}" + ); + } +} + +#[test] +fn copy_match_zero_fills_an_offset_that_reaches_past_the_stream() { + let mut decoder = Unpack20::new(); + decoder.output.extend_from_slice(b"AB").unwrap(); + + decoder.copy_match(4, 9, 6).unwrap(); + + assert_eq!(&*decoder.output, b"AB\0\0\0\0"); +} + +#[test] +fn repeat_last_without_a_previous_match_is_a_no_op() { + let mut lengths = [0u8; super::MAIN_COUNT]; + lengths[0] = 1; + lengths[256] = 1; + let mut decoder = Unpack20::new(); + decoder.main = Huffman::from_lengths(&lengths).unwrap(); + decoder.in_block = true; + decoder.bits.append(&[0b1000_0000]).unwrap(); // Repeat last, then literal zero. + + decoder.decode_until(1).unwrap(); + assert_eq!(&*decoder.output, b"\0"); + assert_eq!(decoder.last_length, 0); +} + +#[test] +fn old_offset_match_applies_the_long_distance_length_adjustment() { + let mut main_lengths = [0u8; super::MAIN_COUNT]; + main_lengths[0] = 1; + main_lengths[257] = 1; + let mut length_lengths = [0u8; super::LENGTH_COUNT]; + length_lengths[0] = 1; + let mut decoder = Unpack20::new(); + decoder.main = Huffman::from_lengths(&main_lengths).unwrap(); + decoder.lengths = Huffman::from_lengths(&length_lengths).unwrap(); + decoder.old_offsets[0] = 0x40000; + decoder.in_block = true; + decoder.bits.append(&[0b1000_0000]).unwrap(); // Old offset 0, length slot 0. + + decoder.decode_until(1).unwrap(); + assert_eq!(&*decoder.output, b"\0"); + assert_eq!(decoder.last_length, 5); // 2 + three distance adjustments. + assert_eq!(decoder.pending_match, Some((4, 0x40000))); +} + +#[test] +fn unset_old_offset_match_uses_the_first_dictionary_byte() { + let mut main_lengths = [0u8; super::MAIN_COUNT]; + main_lengths[0] = 1; + main_lengths[257] = 1; + let mut length_lengths = [0u8; super::LENGTH_COUNT]; + length_lengths[0] = 1; + let mut decoder = Unpack20::new(); + decoder.main = Huffman::from_lengths(&main_lengths).unwrap(); + decoder.lengths = Huffman::from_lengths(&length_lengths).unwrap(); + decoder.in_block = true; + decoder.bits.append(&[0b1000_0000]).unwrap(); + + decoder.decode_until(2).unwrap(); + assert_eq!(&*decoder.output, b"\0\0"); + assert_eq!(decoder.last_offset, 0); +} + +#[test] +fn offset_slot_without_extra_bits_decodes_the_first_distance() { + let mut decoder = Unpack20::new(); + decoder.offsets = Huffman::from_lengths(&[1, 1]).unwrap(); + decoder.bits.append(&[0]).unwrap(); + assert_eq!(decoder.read_offset().unwrap(), 1); +} + +#[test] +fn encoder_slot_tables_cover_their_entire_format_ranges() { + for length in 3..=super::MAX_ENCODER_MATCH_LENGTH { + let (slot, extra) = super::length_slot_for_match(length).unwrap(); + assert_eq!(super::LENGTH_BASES[slot] + extra + 3, length); + assert!(extra < 1usize << super::LENGTH_BITS[slot]); + } + for offset in 1..=256 { + let (slot, extra) = super::short_slot_for_match(offset).unwrap(); + assert_eq!(super::SHORT_BASES[slot] + extra + 1, offset); + assert!(extra < 1usize << super::SHORT_BITS[slot]); + } + for slot in 0..super::OFFSET_COUNT { + for extra in [0, (1usize << super::OFFSET_BITS[slot]) - 1] { + let offset = super::OFFSET_BASES[slot] + extra + 1; + let (actual_slot, actual_extra) = super::offset_slot_for_match(offset).unwrap(); + assert_eq!((actual_slot, actual_extra), (slot, extra)); + } + } + assert_eq!( + super::OFFSET_BASES[super::OFFSET_COUNT - 1] + 65536, + super::MAX_HISTORY + ); + + assert!(super::length_slot_for_match(2).is_err()); + assert!(super::length_slot_for_match(259).is_err()); + assert!(super::offset_slot_for_match(0).is_err()); + assert!(super::offset_slot_for_match(super::MAX_HISTORY + 1).is_err()); + assert!(super::short_slot_for_match(0).is_err()); + assert!(super::short_slot_for_match(257).is_err()); + + for offset in [1, 0x40000] { + let adjustment = super::old_length_adjustment(offset); + for adjusted in 0..=255 { + let length = adjusted + 2 + adjustment; + let (slot, extra) = super::old_length_slot_for_match(length, offset).unwrap(); + assert_eq!(super::LENGTH_BASES[slot] + extra, adjusted); + } + assert!(super::old_length_slot_for_match(258 + adjustment, offset).is_err()); + } +} + +#[test] +fn generated_huffman_lengths_are_canonical_for_rar20_table_sizes() { + fn check() { + let mut seed = 0x9e37_79b9u32; + for round in 0..128 { + let mut frequencies = [0usize; N]; + for frequency in &mut frequencies { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + if !seed.is_multiple_of(5) { + *frequency = 1 + (seed as usize % (1 + round * round)); + } + } + if round % 2 == 0 { + frequencies[0] = 1 << 24; + } + let lengths = super::huffman::lengths_for_frequency_array(&frequencies, 15); + assert!( + super::canonical_codes(&lengths).is_ok(), + "size {N}, round {round}" + ); + } + } + check::<{ super::MAIN_COUNT }>(); + check::<{ super::OFFSET_COUNT }>(); + check::<{ super::LENGTH_COUNT }>(); +} + +#[test] +fn decodes_synthetic_audio_block() { + let packed = synthetic_audio_block(8); + let mut decoder = Unpack20::new(); + + assert_eq!(decoder.decode_member(&packed, 8).unwrap(), vec![0; 8]); +} + +#[test] +fn audio_encoder_round_trips_interleaved_pcm_like_payload() { + let input = interleaved_pcm_like_payload(); + let packed = super::encode_audio_member(&input, 4).unwrap(); + let decoded = unpack20_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn auto_encoder_uses_audio_when_it_beats_lz() { + let input = interleaved_pcm_like_payload(); + let lz = unpack20_encode_literals(&input).unwrap(); + let auto = super::unpack20_encode_auto(&input).unwrap(); + let decoded = unpack20_decode(&auto, input.len()).unwrap(); + + assert!(auto.len() < lz.len()); + assert_eq!(decoded, input); +} + +#[test] +fn short_auto_encoded_members_skip_audio_candidates() { + let input = b"short"; + let packed = super::unpack20_encode_auto(input).unwrap(); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); + + let packed = super::unpack20_encode_auto_with_options( + input, + EncodeOptions::default().with_try_audio(false), + ) + .unwrap(); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); + + let mut always_continue = |_| true; + let packed = super::unpack20_encode_auto_with_options_and_progress( + input, + EncodeOptions::default(), + &mut always_continue, + ) + .unwrap(); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_checks_cancellation_after_the_last_symbol() { + let mut calls = 0; + let mut cancel_at_end = |_| { + calls += 1; + calls == 1 + }; + assert_eq!( + super::unpack20_encode_auto_with_options_and_progress( + b"end", + EncodeOptions::new(0).with_try_audio(false), + &mut cancel_at_end, + ) + .unwrap_err(), + Error::Cancelled + ); + assert_eq!(calls, 2); +} + +#[test] +fn encoder_checks_cancellation_during_refinement() { + let mut calls = 0; + let mut cancel_during_refinement = |_| { + calls += 1; + calls <= 2 + }; + assert_eq!( + super::unpack20_encode_auto_with_options_and_progress( + b"ABCDABCDABCD", + EncodeOptions::default().with_try_audio(false), + &mut cancel_during_refinement, + ) + .unwrap_err(), + Error::Cancelled + ); + assert_eq!(calls, 3); +} + +#[test] +fn encoder_rejects_matches_beyond_the_configured_distance() { + let input = b"abcXYabc"; + let mut finder = super::Rar20MatchFinder::new(input.len()); + finder.insert(input, 0); + assert_eq!( + super::best_match( + input, + 5, + input.len(), + &finder, + EncodeOptions::default().with_max_match_distance(2), + None, + ), + None + ); +} + +#[test] +fn missing_literal_code_has_a_nonzero_refinement_cost() { + let lengths = [0u8; super::TABLE_COUNT]; + let prices = CostModel::new(&lengths); + assert_eq!(prices.literal_bits(b"x", 0, 1), super::ABSENT_LITERAL_BITS); +} + +#[test] +fn old_offset_ties_prefer_the_shorter_distance() { + let lengths = [0u8; super::TABLE_COUNT]; + let prices = CostModel::new(&lengths); + assert!(super::is_better_old_offset_match( + Some(&prices), + b"aaa", + 0, + 1, + 3, + 1, + Some((0, 3, 2)), + )); + assert!(!super::is_better_old_offset_match( + Some(&prices), + b"aaa", + 0, + 1, + 3, + 3, + Some((0, 3, 2)), + )); + + // One extra bit for a longer match can exactly cancel the extra + // literal it saves. On that score tie, prefer the longer match. + let mut lengths = [0u8; super::TABLE_COUNT]; + lengths[b'a' as usize] = 1; + lengths[super::MAIN_COUNT + super::OFFSET_COUNT + 2] = 1; + let prices = CostModel::new(&lengths); + let short = super::SelectedMatch::OldOffset { + index: 0, + length: 3, + offset: 1, + }; + let long = super::SelectedMatch::OldOffset { + index: 1, + length: 4, + offset: 1, + }; + assert_eq!( + prices.selected_score(short, b"aaaa", 0), + prices.selected_score(long, b"aaaa", 0) + ); + assert!(super::is_better_old_offset_match( + Some(&prices), + b"aaaa", + 0, + 1, + 4, + 1, + Some((0, 3, 1)), + )); + assert!(!super::is_better_old_offset_match( + Some(&prices), + b"aaaa", + 0, + 0, + 3, + 1, + Some((1, 4, 1)), + )); +} + +#[test] +fn audio_encoder_rejects_channel_counts_outside_the_format() { + for channels in [0, 5] { + assert_eq!( + super::encode_audio_member(b"audio", channels).unwrap_err(), + Error::InvalidData("RAR 2.0 audio channel count is invalid") + ); + } +} + +#[test] +fn default_encode_options_match_legacy_entry_points() { + let input = b"rar20 option plumbing preserves default output ".repeat(128); + assert_eq!( + unpack20_encode_literals(&input).unwrap(), + super::unpack20_encode_literals_with_options(&input, EncodeOptions::default()).unwrap() + ); + assert_eq!( + super::unpack20_encode_auto(&input).unwrap(), + super::unpack20_encode_auto_with_options(&input, EncodeOptions::default()).unwrap() + ); + + let first = b"solid rar20 option seed ".repeat(64); + let second = b"solid rar20 option seed with suffix ".repeat(32); + let mut legacy = Unpack20Encoder::new(); + let mut explicit = Unpack20Encoder::with_options(EncodeOptions::default()); + assert_eq!( + legacy.encode_member(&first).unwrap(), + explicit.encode_member(&first).unwrap() + ); + assert_eq!( + legacy.encode_member(&second).unwrap(), + explicit.encode_member(&second).unwrap() + ); +} + +#[test] +fn direct_option_field_assignment_cannot_exceed_rar20_distance_limit() { + let mut options = EncodeOptions::new(1); + options.max_match_distance = super::MAX_ENCODER_MATCH_OFFSET + 1; + assert_eq!( + options.constrained().max_match_distance, + super::MAX_ENCODER_MATCH_OFFSET + ); + assert_eq!( + Unpack20Encoder::with_options(options) + .options + .max_match_distance, + super::MAX_ENCODER_MATCH_OFFSET + ); +} + +#[test] +fn optimal_parser_skips_an_out_of_format_fresh_match() { + let start = super::MAX_ENCODER_MATCH_OFFSET + 1; + let end = start + 5; + let mut input = vec![b'X'; end]; + input[..5].copy_from_slice(b"abcde"); + input[start..].copy_from_slice(b"abcde"); + let mut finder = super::Rar20MatchFinder::new(input.len()); + finder.insert(&input, 0); + let mut options = EncodeOptions::new(1).with_optimal_parse(true); + options.max_match_distance = start; + let lengths = [0u8; super::TABLE_COUNT]; + let prices = CostModel::new(&lengths); + + let tokens = super::encode_tokens_optimal(&input, start, end, &mut finder, options, &prices); + assert_eq!(tokens.len(), 5); + assert!(tokens + .iter() + .all(|token| matches!(token, EncodeToken::Literal(_)))); +} + +#[test] +fn encode_options_can_disable_fresh_lz_matches() { + let input = b"abcdefabcdefabcdefabcdef"; + let default_tokens = encode_tokens(input, &[], EncodeOptions::default(), None); + let literalish_tokens = encode_tokens(input, &[], EncodeOptions::new(0), None); + + assert!(default_tokens + .iter() + .any(|token| matches!(token, EncodeToken::Match { .. }))); + assert!(!literalish_tokens + .iter() + .any(|token| matches!(token, EncodeToken::Match { .. }))); +} + +#[test] +fn table_level_encoder_uses_rar20_run_symbols() { + let lengths = [0, 0, 0, 0, 5, 5, 5, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]; + let tokens = super::encode_level_tokens(&lengths); + + assert_eq!( + tokens, + vec![ + super::LevelToken::zero_run_short(4), + super::LevelToken::plain(5), + super::LevelToken::repeat_previous(3), + super::LevelToken::plain(7), + super::LevelToken::zero_run_short(10), + super::LevelToken::plain(2), + ] + ); +} + +#[test] +fn decodes_back_to_back_fresh_audio_blocks() { + // No real RAR 2.x encoder we tested emits a mid-stream `audio_block, + // !keep_tables` transition; reference encoders always either keep the + // existing audio tables across boundaries or switch out to LZ. The + // decoder branch that rebuilds `audio_tables` from a freshly-read level + // table inside an audio sequence is therefore only reachable via a + // hand-crafted fixture. + let mut bits = BitWriter::default(); + write_fresh_audio_block(&mut bits, 4, /*emit_end_sentinel=*/ true); + write_fresh_audio_block(&mut bits, 4, /*emit_end_sentinel=*/ false); + let packed = bits.finish(); + + let mut decoder = Unpack20::new(); + + assert_eq!(decoder.decode_member(&packed, 8).unwrap(), vec![0; 8]); +} + +#[test] +fn audio_member_reads_trailing_table_for_next_solid_member() { + let mut bits = BitWriter::default(); + write_fresh_audio_block(&mut bits, 4, /*emit_end_sentinel=*/ true); + write_fresh_audio_block(&mut bits, 4, /*emit_end_sentinel=*/ false); + let packed = bits.finish(); + + let mut decoder = Unpack20::new(); + assert_eq!(decoder.decode_member(&packed, 4).unwrap(), vec![0; 4]); + assert_eq!(decoder.decode_member(&[], 4).unwrap(), vec![0; 4]); +} + +#[test] +fn lz_member_reads_trailing_table_for_next_solid_member() { + let mut bits = BitWriter::default(); + write_fresh_lz_zero_block(&mut bits, 4, true); + write_fresh_lz_zero_block(&mut bits, 4, false); + let packed = bits.finish(); + + let mut decoder = Unpack20::new(); + assert_eq!(decoder.decode_member(&packed, 4).unwrap(), vec![0; 4]); + assert_eq!(decoder.decode_member(&[], 4).unwrap(), vec![0; 4]); +} + +fn write_fresh_lz_zero_block(bits: &mut BitWriter, count: usize, end: bool) { + let mut table = [0u8; super::TABLE_COUNT]; + table[0] = 1; + table[269] = 1; + let level_tokens = super::encode_table_level_tokens(&table); + let level_lengths = super::level_code_lengths_for_tokens(&level_tokens); + let level_codes = super::canonical_codes(&level_lengths).unwrap(); + let main_codes = super::canonical_codes(&table[..super::MAIN_COUNT]).unwrap(); + + bits.write_bits(0, 2); + for &len in &level_lengths { + bits.write_bits(len as u32, 4); + } + for token in level_tokens { + let code = level_codes[token.symbol].unwrap(); + bits.write_bits(code.code as u32, code.len); + bits.write_bits(token.extra_value as u32, token.extra_bits); + } + let zero = main_codes[0].unwrap(); + for _ in 0..count { + bits.write_bits(zero.code as u32, zero.len); + } + if end { + let end_code = main_codes[269].unwrap(); + bits.write_bits(end_code.code as u32, end_code.len); + } +} + +#[test] +fn decoder_and_encoder_retain_only_the_last_window_of_solid_history() { + let first = vec![b'A'; super::MAX_HISTORY / 2 + 64]; + let second = vec![b'B'; super::MAX_HISTORY / 2 + 64]; + let mut encoder = Unpack20Encoder::with_options(EncodeOptions::new(0)); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + assert_eq!(encoder.history.len(), super::MAX_HISTORY); + + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!( + decoder.decode_member(&second_packed, second.len()).unwrap(), + second + ); + assert_eq!(decoder.output.len(), super::MAX_HISTORY); + assert_eq!(decoder.base_offset, 128); +} + +#[test] +fn decode_member_carries_a_match_across_output_boundary() { + let input = b"ABCD".repeat(20); + let packed = unpack20_encode_literals(&input).unwrap(); + let mut decoder = Unpack20::new(); + + let first = decoder.decode_member(&packed, 6).unwrap(); + assert!(decoder.pending_match.is_some()); + let second = decoder + .decode_member(&[], input.len() - first.len()) + .unwrap(); + + assert_eq!([first, second].concat(), input); + assert!(decoder.pending_match.is_none()); +} + +fn expected_text() -> Vec { + b"Hello text not audio.\r\n".repeat(100) +} + +fn interleaved_pcm_like_payload() -> Vec { + let mut input = Vec::new(); + for sample in 0..8192i16 { + let left = sample.wrapping_mul(3).wrapping_add(200); + let right = sample.wrapping_mul(3).wrapping_sub(200); + input.extend_from_slice(&left.to_le_bytes()); + input.extend_from_slice(&right.to_le_bytes()); + } + input +} + +fn synthetic_audio_block(samples: usize) -> Vec { + let mut bits = BitWriter::default(); + + bits.write_bits(0b10, 2); // audio block, do not keep previous tables. + bits.write_bits(0, 2); // one channel. + + for symbol in 0..19 { + let len = if symbol == 1 || symbol == 18 { 1 } else { 0 }; + bits.write_bits(len, 4); + } + + bits.write_bit(false); // level symbol 1: audio delta 0 has code length 1. + bits.write_bit(true); // level symbol 18: 138 zeros. + bits.write_bits(127, 7); + bits.write_bit(true); // level symbol 18: 118 zeros. + bits.write_bits(107, 7); + + for _ in 0..samples { + bits.write_bit(false); // audio delta 0. + } + + bits.finish() +} + +fn write_fresh_audio_block(bits: &mut BitWriter, samples: usize, emit_end_sentinel: bool) { + bits.write_bits(0b10, 2); // audio block, do not keep previous tables. + bits.write_bits(0, 2); // one channel. + + for symbol in 0..19 { + let len = if symbol == 1 || symbol == 18 { 1 } else { 0 }; + bits.write_bits(len, 4); + } + + // Audio table: symbol 0 (delta 0) = "0", symbol 256 (block end) = "1". + bits.write_bit(false); // level symbol 1: audio delta 0 has code length 1. + bits.write_bit(true); // level symbol 18: 138 zeros (audio symbols 1..=138). + bits.write_bits(127, 7); + bits.write_bit(true); // level symbol 18: 117 zeros (audio symbols 139..=255). + bits.write_bits(106, 7); + bits.write_bit(false); // level symbol 1: block-end (256) has code length 1. + + for _ in 0..samples { + bits.write_bit(false); // audio delta 0. + } + if emit_end_sentinel { + bits.write_bit(true); // audio symbol 256: end of audio block. + } +} + +#[test] +fn literal_encoder_round_trips_rar20_lz_blocks() { + let input = b"literal-only RAR 2.0 baseline\nwith repeated text literal-only\n"; + let packed = unpack20_encode_literals(input).unwrap(); + + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar20_offset_one_matches_for_repeated_bytes() { + let input = b"A".repeat(1024); + let packed = unpack20_encode_literals(&input).unwrap(); + + assert!(packed.len() < input.len() / 4); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar20_dictionary_matches_for_repeated_sequences() { + let input = b"abc123xyz-".repeat(128); + let packed = unpack20_encode_literals(&input).unwrap(); + + assert!(packed.len() < input.len() / 2); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar20_repeat_last_matches_for_regular_streams() { + let input = b"\x00\x01\x02\x03".repeat(4096); + let tokens = encode_tokens(&input, &[], EncodeOptions::default(), None); + let packed = unpack20_encode_literals(&input).unwrap(); + + assert!(tokens + .iter() + .any(|token| matches!(token, EncodeToken::RepeatLast))); + assert!(packed.len() < input.len() / 8); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar20_minimum_length_fresh_matches() { + let input = b"abcabc"; + let tokens = encode_tokens(input, &[], EncodeOptions::default(), None); + let packed = unpack20_encode_literals(input).unwrap(); + + assert!(matches!( + tokens.as_slice(), + [ + EncodeToken::Literal(b'a'), + EncodeToken::Literal(b'b'), + EncodeToken::Literal(b'c'), + EncodeToken::Match { + length: 3, + offset: 3 + } + ] + )); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar20_short_offset_matches() { + let input = b"abab"; + let tokens = encode_tokens(input, &[], EncodeOptions::default(), None); + let packed = unpack20_encode_literals(input).unwrap(); + + assert!(matches!( + tokens.as_slice(), + [ + EncodeToken::Literal(b'a'), + EncodeToken::Literal(b'b'), + EncodeToken::ShortOffset { offset: 2 } + ] + )); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar20_old_offset_matches() { + let input = b"abcdabcdXYZXYZwxyzwxyz"; + let tokens = encode_tokens(input, &[], EncodeOptions::default(), None); + let packed = unpack20_encode_literals(input).unwrap(); + + assert!(tokens + .iter() + .any(|token| matches!(token, EncodeToken::OldOffset { .. }))); + assert_eq!(unpack20_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_finds_rar20_matches_beyond_near_offsets() { + let phrase = b"long-distance repeated phrase for rar20 match finder."; + let mut input = Vec::new(); + input.extend_from_slice(phrase); + input.extend(std::iter::repeat_n(0, 300 * 1024)); + input.extend_from_slice(phrase); + input.extend_from_slice(phrase); + let tokens = encode_tokens(&input, &[], EncodeOptions::default(), None); + let packed = unpack20_encode_literals(&input).unwrap(); + + assert!(tokens.iter().any(|token| matches!( + token, + EncodeToken::Match { offset, .. } if *offset > 0x40000 + ))); + assert!(packed.len() < input.len()); + let decoded = unpack20_decode(&packed, input.len()).unwrap(); + assert!( + decoded == input, + "RAR 2.0 long-distance match round-trip failed" + ); +} + +#[test] +fn solid_encoder_emits_rar20_matches_against_previous_member_history() { + let first = b"solid rar20 shared phrase alpha beta gamma ".repeat(4); + let second = b"solid rar20 shared phrase alpha beta gamma ".repeat(2); + let independent = unpack20_encode_literals(&second).unwrap(); + let mut encoder = Unpack20Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + + assert!(second_packed.len() < independent.len()); + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!( + decoder.decode_member(&second_packed, second.len()).unwrap(), + second + ); +} + +#[test] +fn the_optimal_parse_never_loses_to_the_greedy_one() { + // Repeated phrases at varying distances with literal noise between + // them, which is where committing to the first match found costs + // something: the greedy parse takes a short match that a later, longer + // one would have covered for free. + let mut input = Vec::new(); + for round in 0..64u8 { + input.extend_from_slice(b"the quick brown fox jumps over the lazy dog"); + input.extend_from_slice(&[round, round ^ 0x5a, round.wrapping_mul(31)]); + input.extend_from_slice(b"over the lazy dog and the quick brown fox"); + input.push(round ^ 0xa5); + } + + let greedy_options = EncodeOptions::new(256) + .with_lazy_matching(true) + .with_lazy_lookahead(2); + let greedy = super::unpack20_encode_literals_with_options(&input, greedy_options).unwrap(); + let optimal = super::unpack20_encode_literals_with_options( + &input, + greedy_options.with_optimal_parse(true), + ) + .unwrap(); + + assert!( + optimal.len() <= greedy.len(), + "optimal {} greedy {}", + optimal.len(), + greedy.len() + ); + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&optimal, input.len()).unwrap(), + input, + "the optimal parse produced something the decoder disagrees with" + ); +} + +#[test] +fn the_optimal_parse_decodes_whatever_it_is_given() { + // Shapes that stress different corners of the parse: nothing to match, + // one long run, matches that reach back exactly to the rep offsets, and + // bytes with no structure at all. + let mut noise = Vec::new(); + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + for _ in 0..8192 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + noise.push(seed as u8); + } + let cases: Vec> = vec![ + b"a".to_vec(), + b"abc".to_vec(), + vec![0u8; 5000], + (0u8..=255).cycle().take(4096).collect(), + b"xyzzy".repeat(700), + noise, + ]; + + for input in cases { + let options = EncodeOptions::new(256) + .with_lazy_matching(true) + .with_lazy_lookahead(2) + .with_optimal_parse(true); + let packed = super::unpack20_encode_literals_with_options(&input, options).unwrap(); + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&packed, input.len()).unwrap(), + input, + "round trip failed for a {} byte member", + input.len() + ); + } +} + +#[test] +fn solid_encoder_reuses_rar20_tables_at_member_boundary() { + let first: Vec<_> = (0u8..=255).cycle().take(4096).collect(); + let second = b"short literal member after reused rar20 table boundary\n"; + let independent = unpack20_encode_literals(second).unwrap(); + let mut encoder = Unpack20Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(second).unwrap(); + + assert!(second_packed.len() < independent.len()); + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!( + decoder.decode_member(&second_packed, second.len()).unwrap(), + second + ); +} + +#[test] +fn solid_encoder_matches_immediately_after_rar20_table_boundary() { + let phrase = b"rar20 table boundary match phrase with enough bytes "; + let first = phrase.repeat(128); + let second = phrase.repeat(8); + let independent = unpack20_encode_literals(&second).unwrap(); + let mut encoder = Unpack20Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + let tokens = encode_tokens(&second, &first, EncodeOptions::default(), None); + + assert!(matches!(tokens.first(), Some(EncodeToken::Match { .. }))); + assert!(second_packed.len() < independent.len()); + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!( + decoder.decode_member(&second_packed, second.len()).unwrap(), + second + ); +} + +#[test] +fn solid_encoder_carries_rar20_history_across_multiple_members() { + let first = b"rar20 multi member solid seed ".repeat(512); + let second = b"rar20 multi member solid seed with middle tail ".repeat(128); + let third = b"with middle tail ".repeat(64); + let independent = unpack20_encode_literals(&third).unwrap(); + let mut encoder = Unpack20Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + let third_packed = encoder.encode_member(&third).unwrap(); + + assert!(third_packed.len() < independent.len()); + let mut decoder = Unpack20::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!( + decoder.decode_member(&second_packed, second.len()).unwrap(), + second + ); + assert_eq!( + decoder.decode_member(&third_packed, third.len()).unwrap(), + third + ); +} + +#[test] +fn decode_member_to_streams_decoded_payload_through_writer_sink() { + let input = b"abcabcabcabcabcabcabcabcabcabcabcabc"; + let packed = unpack20_encode_literals(input).unwrap(); + + let mut decoder = Unpack20::new(); + let mut sink = Vec::new(); + decoder + .decode_member_to(&packed, input.len(), &mut sink) + .unwrap(); + assert_eq!(sink, input); + + // The error-mapping closure inside decode_member_to fires when the + // sink's write_all returns Err — feed it a writer that always fails. + struct FailingWriter; + impl std::io::Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("disk full")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut decoder = Unpack20::new(); + let err = decoder + .decode_member_to(&packed, input.len(), &mut FailingWriter) + .unwrap_err(); + assert_eq!(err, Error::from(std::io::Error::other("disk full"))); +} diff --git a/arca-rar/vendor/rars/src/codec/rar29.rs b/arca-rar/vendor/rars/src/codec/rar29.rs new file mode 100644 index 00000000..11609bb1 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar29.rs @@ -0,0 +1,1407 @@ +#[cfg(feature = "write")] +mod encoder; +#[cfg(feature = "write")] +pub(crate) use encoder::{ + filtered_members, unpack29_encode_literals_with_options_and_progress, + unpack29_encode_ppmd_with_progress, MAX_VM_DELTA_FILTER_BLOCK_SIZE, +}; +#[cfg(feature = "write")] +pub use encoder::{ + unpack29_encode_literals, unpack29_encode_literals_with_options, unpack29_encode_ppmd, + unpack29_encode_ppmd_literals, unpack29_encode_ppmd_with_filter, ChainEngine, EncodeOptions, + Unpack29Encoder, +}; + +mod standard_filters; +#[cfg(all(test, feature = "write"))] +use standard_filters::{ + apply_standard_filter, apply_standard_filter_with_control, audio_decode_with_control, + itanium_decode, itanium_decode_with_control, rgb_decode_with_control, +}; +use standard_filters::{apply_standard_filter_with_allowance, identify_standard_filter}; + +use super::filters::DeltaErrorMessages; +#[cfg(all(test, feature = "write"))] +use super::filters::MAX_DELTA_CHANNELS; +use super::ppmd::{PpmdByteReader, PpmdState}; +use super::rarvm; +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; +use std::io::{Read, Write}; + +const MAIN_COUNT: usize = 299; +const OFFSET_COUNT: usize = 60; +const LOW_OFFSET_COUNT: usize = 17; +const LENGTH_COUNT: usize = 28; +const LEVEL_COUNT: usize = 20; +const TABLE_COUNT: usize = MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT + LENGTH_COUNT; +const MAX_HISTORY: usize = 4 * 1024 * 1024; +const STREAM_CHUNK: usize = 1024 * 1024; +// RARVM's standard AUDIO filter reserves an eight-bit-ish compatibility +// range wider than WinRAR's usual 1..=4 channel choices. UnRAR accepts up to +// 128; keep this distinct from DELTA's 1024-channel ceiling. +const MAX_AUDIO_CHANNELS: usize = 128; +const MAX_VM_GLOBAL_DATA: usize = 0x2000; +const VM_SYSTEM_GLOBAL_SIZE: usize = 64; +const MAX_VM_USER_GLOBAL_DATA: usize = MAX_VM_GLOBAL_DATA - VM_SYSTEM_GLOBAL_SIZE; +const MAX_VM_CODE_SIZE: usize = 64 * 1024; +const MAX_VM_PROGRAMS: usize = 8192; +const MAX_VM_FILTERS: usize = 8192; + +const LENGTH_BASES: [usize; LENGTH_COUNT] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, + 160, 192, 224, +]; +const LENGTH_BITS: [u8; LENGTH_COUNT] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, +]; +const OFFSET_BASES: [usize; OFFSET_COUNT] = [ + 0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, + 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536, 98304, 131072, 196608, + 262144, 327680, 393216, 458752, 524288, 589824, 655360, 720896, 786432, 851968, 917504, 983040, + 1048576, 1310720, 1572864, 1835008, 2097152, 2359296, 2621440, 2883584, 3145728, 3407872, + 3670016, 3932160, +]; +const OFFSET_BITS: [u8; OFFSET_COUNT] = [ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, + 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, +]; +const SHORT_BASES: [usize; 8] = [0, 4, 8, 16, 32, 64, 128, 192]; +const SHORT_BITS: [u8; 8] = [2, 2, 3, 4, 5, 6, 6, 6]; +const INVALID_MATCH_OFFSET: usize = usize::MAX; + +pub fn unpack29_decode(input: &[u8], output_size: usize) -> Result> { + let mut decoder = Unpack29::new(); + decoder.decode_non_solid_member(input, output_size) +} + +/// The filters the RAR 2.9 family ships as RarVM programs. +/// +/// Narrower than [`crate::FilterKind`], which names every filter any format +/// can apply. The conversion below is where a filter this family cannot encode +/// is turned away. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Rar29Filter { + E8, + E8E9, + Delta { channels: usize }, + Itanium, + Rgb { width: usize, pos_r: usize }, + Audio { channels: usize }, +} + +impl TryFrom for Rar29Filter { + type Error = crate::UnsupportedFilterKind; + + fn try_from(kind: crate::FilterKind) -> std::result::Result { + use crate::FilterKind as Kind; + match kind { + Kind::E8 => Ok(Self::E8), + Kind::E8E9 => Ok(Self::E8E9), + Kind::Delta { channels } => Ok(Self::Delta { channels }), + Kind::Itanium => Ok(Self::Itanium), + Kind::Rgb { width, pos_r } => Ok(Self::Rgb { width, pos_r }), + Kind::Audio { channels } => Ok(Self::Audio { channels }), + // No wildcard arm: an eighth filter has to be decided about here. + kind @ Kind::Arm => Err(crate::UnsupportedFilterKind(kind)), + } + } +} + +fn rar29_delta_messages() -> DeltaErrorMessages { + DeltaErrorMessages { + invalid_channels: "RAR 2.9 DELTA filter channel count is invalid", + zero_channels: "RAR 2.9 DELTA filter has zero channels", + truncated_source: "RAR 2.9 DELTA filter source is truncated", + } +} + +#[derive(Debug, Clone)] +pub struct Unpack29 { + pub(crate) read_control: crate::read_control::ReadControl, + state: Reader29State, +} +impl Clone for Reader29State { + fn clone(&self) -> Self { + self.try_clone().expect("unlimited RAR3 decoder copy") + } +} +impl Default for Unpack29 { + fn default() -> Self { + Self::new() + } +} +impl Unpack29 { + pub fn new() -> Self { + Self { + read_control: Default::default(), + state: Reader29State::with_allowance(&Allowance::default()), + } + } + pub fn reset_non_solid(&mut self) { + self.state.reset_non_solid(); + } + pub fn decode_member(&mut self, input: &[u8], output_size: usize) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_owned(input, output_size) + .map(Buffer::into_vec) + } + pub fn decode_member_to( + &mut self, + input: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state.decode_member_to(input, output_size, out) + } + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_from_reader(input, output_size, out) + } + pub fn decode_non_solid_member(&mut self, input: &[u8], output_size: usize) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_non_solid_member_owned(input, output_size) + .map(Buffer::into_vec) + } + pub fn decode_non_solid_member_to( + &mut self, + input: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_non_solid_member_to(input, output_size, out) + } + pub fn decode_non_solid_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_non_solid_member_from_reader(input, output_size, out) + } +} +#[derive(Debug)] +pub(crate) struct Reader29State { + pub(crate) read_control: crate::read_control::ReadControl, + bits: BitReader, + levels: [u8; TABLE_COUNT], + main: Huffman, + offsets: Huffman, + low_offsets: Huffman, + lengths: Huffman, + old_offsets: [usize; 4], + last_offset: usize, + last_length: usize, + last_low_offset: usize, + low_offset_repeats: usize, + pending_match: Option<(usize, usize)>, + in_lz_block: bool, + block_mode: BlockMode, + ppmd: PpmdState, + ppmd_esc: u8, + filters: Buffer, B>, + programs: Buffer, B>, + last_filter: usize, + base_offset: usize, + output: Buffer, + last_block_end: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BlockMode { + Lz, + Ppmd, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LzBlockEnd { + SameFileNewTable, + NewFileKeepTables, + NewFileNewTables, +} + +fn require_ppmd_symbol(symbol: Option) -> Result { + symbol.ok_or(Error::InvalidData("RAR 2.9 PPMd model is corrupt")) +} + +#[derive(Debug)] +struct VmFilter { + program: usize, + start: usize, + size: usize, + regs: [u32; 7], + global_data: Buffer, +} + +#[derive(Debug)] +struct VmProgram { + kind: VmProgramKind, + block_size: usize, + exec_count: u32, + globals: Buffer, +} + +#[derive(Debug)] +enum VmProgramKind { + Standard(StandardFilter), + Generic(rarvm::OwnedProgram), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StandardFilter { + E8, + E8E9, + Itanium, + Delta, + Rgb, + Audio, +} + +impl Reader29State { + pub(crate) fn with_allowance(allowance: &B) -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + bits: BitReader::with_allowance(allowance), + levels: [0; TABLE_COUNT], + main: Huffman::with_allowance(allowance), + offsets: Huffman::with_allowance(allowance), + low_offsets: Huffman::with_allowance(allowance), + lengths: Huffman::with_allowance(allowance), + // UnRAR uses an invalid all-bits-one distance here. If a malformed + // stream repeats a distance before defining one, CopyString sees + // it as unavailable history and writes deterministic zeroes. + old_offsets: [INVALID_MATCH_OFFSET; 4], + last_offset: INVALID_MATCH_OFFSET, + last_length: 0, + last_low_offset: 0, + low_offset_repeats: 0, + pending_match: None, + in_lz_block: false, + block_mode: BlockMode::Lz, + ppmd: PpmdState::with_allowance(allowance), + ppmd_esc: 2, + filters: Buffer::new(allowance), + programs: Buffer::new(allowance), + last_filter: 0, + base_offset: 0, + output: Buffer::new(allowance), + last_block_end: None, + } + } + + pub(crate) fn try_clone(&self) -> Result { + let allowance = self.output.allowance(); + Ok(Self { + read_control: self.read_control.clone(), + bits: self.bits.try_clone()?, + levels: self.levels, + main: self.main.try_clone()?, + offsets: self.offsets.try_clone()?, + low_offsets: self.low_offsets.try_clone()?, + lengths: self.lengths.try_clone()?, + old_offsets: self.old_offsets, + last_offset: self.last_offset, + last_length: self.last_length, + last_low_offset: self.last_low_offset, + low_offset_repeats: self.low_offset_repeats, + pending_match: self.pending_match, + in_lz_block: self.in_lz_block, + block_mode: self.block_mode, + ppmd: self.ppmd.try_clone()?, + ppmd_esc: self.ppmd_esc, + filters: Buffer::try_collect( + self.filters.iter().map(|filter| { + Ok(VmFilter { + program: filter.program, + start: filter.start, + size: filter.size, + regs: filter.regs, + global_data: Buffer::copied(&filter.global_data, &allowance)?, + }) + }), + &allowance, + )?, + programs: Buffer::try_collect( + self.programs.iter().map(|program| { + Ok(VmProgram { + kind: match &program.kind { + VmProgramKind::Standard(kind) => VmProgramKind::Standard(*kind), + VmProgramKind::Generic(program) => { + VmProgramKind::Generic(program.try_clone()?) + } + }, + block_size: program.block_size, + exec_count: program.exec_count, + globals: Buffer::copied(&program.globals, &allowance)?, + }) + }), + &allowance, + )?, + last_filter: self.last_filter, + base_offset: self.base_offset, + output: Buffer::copied(&self.output, &allowance)?, + last_block_end: self.last_block_end, + }) + } + pub fn reset_non_solid(&mut self) { + let control = self.read_control.clone(); + *self = Self::with_allowance(&self.output.allowance()); + self.read_control = control; + } + + pub fn decode_non_solid_member_owned( + &mut self, + input: &[u8], + output_size: usize, + ) -> Result> { + self.read_control.check_codec()?; + self.reset_non_solid(); + self.decode_member_owned(input, output_size) + } + + pub fn decode_non_solid_member_to( + &mut self, + input: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + self.reset_non_solid(); + self.decode_member_to(input, output_size, out) + } + + pub fn decode_non_solid_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + self.reset_non_solid(); + self.decode_member_from_reader(input, output_size, out) + } + + pub fn decode_member_owned( + &mut self, + input: &[u8], + output_size: usize, + ) -> Result> { + let mut out = Buffer::new(&self.output.allowance()); + self.decode_member_to(input, output_size, &mut out)?; + Ok(out) + } + + pub fn decode_member_to( + &mut self, + input: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.decode_loaded_member_to(input, output_size, out) + } + + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + self.bits = BitReader::with_allowance(&self.output.allowance()); + self.bits.input.read_to_end(input)?; + self.decode_bits_member_to(output_size, out) + } + + /// Decodes one complete packed member while retaining solid dictionary, + /// table, PPMd and VM state for the next member. + fn decode_loaded_member_to( + &mut self, + packed: &[u8], + output_size: usize, + out: &mut impl Write, + ) -> Result<()> { + self.read_control.check_codec()?; + self.bits = BitReader::from_bytes_with_allowance(packed, &self.output.allowance())?; + self.decode_bits_member_to(output_size, out) + } + fn decode_bits_member_to(&mut self, output_size: usize, out: &mut impl Write) -> Result<()> { + // `last_block_end` describes control flow within one member. A solid + // follower legitimately starts after the previous member's new-file + // marker, so do not mistake that marker for an early end in this one. + self.last_block_end = None; + let start = self.current_pos(); + let final_target = start + .checked_add(output_size) + .ok_or(Error::InvalidData("RAR 2.9 output size overflows"))?; + let mut flushed = start; + let mut target = start.saturating_add(STREAM_CHUNK).min(final_target); + // Empty members in solid mode still carry their own block init bytes + // (typically the (esc, 0) end-of-block marker + 4-byte range coder + // flush). When output_size is zero, decode_until skips its loop body + // and never reads tables, so do the init here so finish_member can + // observe the block end. + if final_target == start && !self.in_lz_block && !self.bits.input.is_empty() { + self.read_tables().map_err(|error| match error { + Error::NeedMoreInput => Error::InvalidData("RAR 2.9 bitstream is truncated"), + error => error, + })?; + self.in_lz_block = true; + } + + while flushed < final_target { + self.decode_until(target).map_err(|error| match error { + Error::NeedMoreInput => Error::InvalidData("RAR 2.9 bitstream is truncated"), + error => error, + })?; + + let safe_end = self.safe_flush_end(flushed, target, final_target)?; + if safe_end <= flushed { + target = self + .current_pos() + .saturating_add(STREAM_CHUNK) + .min(final_target); + continue; + } + + let decoded = self.filtered_range_owned(flushed, safe_end, start)?; + out.write_all(&decoded).map_err(Error::from)?; + flushed = safe_end; + self.trim_history(flushed, self.current_pos()); + target = self + .current_pos() + .saturating_add(STREAM_CHUNK) + .min(final_target); + } + if self.pending_match.is_some() { + return Err(Error::InvalidData( + "RAR 2.9 member produces more output than its declared size", + )); + } + self.finish_member().map_err(|error| match error { + Error::NeedMoreInput => Error::InvalidData("RAR 2.9 bitstream is truncated"), + error => error, + })?; + Ok(()) + } + + fn decode_until(&mut self, target: usize) -> Result<()> { + let mut poller = self.read_control.poller(); + while self.current_pos() < target { + poller.check_codec(self.current_pos())?; + self.drain_pending_match(target)?; + if self.current_pos() >= target { + break; + } + if !self.in_lz_block { + if matches!( + self.last_block_end, + Some(LzBlockEnd::NewFileKeepTables | LzBlockEnd::NewFileNewTables) + ) { + return Err(Error::InvalidData( + "RAR 2.9 member ended before its declared size", + )); + } + self.read_tables()?; + self.in_lz_block = true; + } + match self.block_mode { + BlockMode::Lz => self.decode_lz(target)?, + BlockMode::Ppmd => self.decode_ppmd(target)?, + } + } + Ok(()) + } + + fn read_tables(&mut self) -> Result<()> { + self.bits.align_byte(); + if self.bits.peek_bit()? != 0 { + let first_byte = self.bits.read_bits(8)? as u8; + self.ppmd.set_read_control(self.read_control.clone()); + self.ppmd + .decode_init(first_byte, &mut self.bits, &mut self.ppmd_esc)?; + self.block_mode = BlockMode::Ppmd; + return Ok(()); + } + self.bits.read_bit()?; + self.block_mode = BlockMode::Lz; + let keep_tables = self.bits.read_bit()? != 0; + self.last_low_offset = 0; + self.low_offset_repeats = 0; + if !keep_tables { + self.levels = [0; TABLE_COUNT]; + } + + let level_lengths = Self::read_level_lengths(&mut self.bits)?; + let level_decoder = + Huffman::from_lengths_with_allowance(&level_lengths, &self.output.allowance())?; + let mut new_levels = [0u8; TABLE_COUNT]; + let mut pos = 0usize; + while pos < TABLE_COUNT { + let symbol = level_decoder.decode(&mut self.bits)?; + match symbol { + 0..=15 => { + new_levels[pos] = (self.levels[pos].wrapping_add(symbol as u8)) & 0x0f; + pos += 1; + } + 16 => { + if pos == 0 { + return Err(Error::InvalidData("RAR 2.9 table repeat at start")); + } + let count = 3 + self.bits.read_bits(3)? as usize; + let value = new_levels[pos - 1]; + fill_levels(&mut new_levels, &mut pos, count, value)?; + } + 17 => { + if pos == 0 { + return Err(Error::InvalidData("RAR 2.9 long table repeat at start")); + } + let count = 11 + self.bits.read_bits(7)? as usize; + let value = new_levels[pos - 1]; + fill_levels(&mut new_levels, &mut pos, count, value)?; + } + 18 => { + let count = 3 + self.bits.read_bits(3)? as usize; + fill_levels(&mut new_levels, &mut pos, count, 0)?; + } + _ => { + let count = 11 + self.bits.read_bits(7)? as usize; + fill_levels(&mut new_levels, &mut pos, count, 0)?; + } + } + } + + self.levels = new_levels; + self.main = Huffman::from_lengths_with_allowance( + &self.levels[..MAIN_COUNT], + &self.output.allowance(), + )?; + self.offsets = Huffman::from_lengths_with_allowance( + &self.levels[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT], + &self.output.allowance(), + )?; + self.low_offsets = Huffman::from_lengths_with_allowance( + &self.levels[MAIN_COUNT + OFFSET_COUNT..MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT], + &self.output.allowance(), + )?; + self.lengths = Huffman::from_lengths_with_allowance( + &self.levels[MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT..], + &self.output.allowance(), + )?; + Ok(()) + } + + fn read_level_lengths(bits: &mut BitReader) -> Result<[u8; LEVEL_COUNT]> { + let mut lengths = [0u8; LEVEL_COUNT]; + let mut pos = 0usize; + while pos < LEVEL_COUNT { + let value = bits.read_bits(4)? as u8; + if value == 15 { + let zero_count = bits.read_bits(4)? as usize; + if zero_count == 0 { + lengths[pos] = 15; + pos += 1; + } else { + pos = pos.saturating_add(zero_count + 2).min(LEVEL_COUNT); + } + } else { + lengths[pos] = value; + pos += 1; + } + } + Ok(lengths) + } + + fn decode_lz(&mut self, output_size: usize) -> Result<()> { + let mut poller = self.read_control.poller(); + while self.current_pos() < output_size { + poller.check_codec(self.current_pos())?; + let symbol = self.main.decode(&mut self.bits)?; + match symbol { + 0..=255 => self.output.try_push(symbol as u8)?, + 256 => { + self.read_end_of_block()?; + return Ok(()); + } + 257 => { + self.read_vm_code()?; + } + 258 => { + if self.last_length != 0 { + self.copy_match(self.last_length, self.last_offset, output_size)?; + } + } + 259..=262 => { + let index = symbol - 259; + let offset = self.old_offsets[index]; + let length_slot = self.lengths.decode(&mut self.bits)?; + let mut length = LENGTH_BASES[length_slot] + 2; + if LENGTH_BITS[length_slot] != 0 { + length += self.bits.read_bits(LENGTH_BITS[length_slot])? as usize; + } + self.rotate_old_offset(index); + self.last_offset = offset; + self.last_length = length; + self.copy_match(length, offset, output_size)?; + } + 263..=270 => { + let index = symbol - 263; + let mut offset = SHORT_BASES[index] + 1; + offset += self.bits.read_bits(SHORT_BITS[index])? as usize; + self.push_old_offset(offset); + self.last_offset = offset; + self.last_length = 2; + self.copy_match(2, offset, output_size)?; + } + _ => { + let length_slot = symbol - 271; + let mut length = LENGTH_BASES[length_slot] + 3; + if LENGTH_BITS[length_slot] != 0 { + length += self.bits.read_bits(LENGTH_BITS[length_slot])? as usize; + } + let offset = self.read_offset()?; + if offset >= 0x2000 { + length += 1; + } + if offset >= 0x40000 { + length += 1; + } + self.push_old_offset(offset); + self.last_offset = offset; + self.last_length = length; + self.copy_match(length, offset, output_size)?; + } + } + } + Ok(()) + } + + fn decode_ppmd(&mut self, output_size: usize) -> Result<()> { + let mut poller = self.read_control.poller(); + while self.current_pos() < output_size { + poller.check_codec(self.current_pos())?; + let symbol = require_ppmd_symbol(self.ppmd.decode_symbol(&mut self.bits)?)?; + if symbol != self.ppmd_esc { + self.output.try_push(symbol)?; + continue; + } + + let next = require_ppmd_symbol(self.ppmd.decode_symbol(&mut self.bits)?)?; + match next { + 0 => { + self.in_lz_block = false; + return Ok(()); + } + 1 => self.output.try_push(self.ppmd_esc)?, + 2 => { + return Err(Error::InvalidData( + "RAR 2.9 member ended before its declared size", + )); + } + 3 => { + self.read_vm_code_ppmd()?; + } + 4 => { + let mut offset = 0usize; + for _ in 0..3 { + offset = (offset << 8) | self.read_ppmd_required_byte()? as usize; + } + offset += 2; + let length = self.read_ppmd_required_byte()? as usize + 32; + self.copy_match(length, offset, output_size)?; + } + 5 => { + let length = self.read_ppmd_required_byte()? as usize + 4; + self.copy_match(length, 1, output_size)?; + } + 6..=u8::MAX => { + return Err(Error::InvalidData("RAR 2.9 PPMd command is invalid")); + } + } + } + Ok(()) + } + + fn read_ppmd_required_byte(&mut self) -> Result { + require_ppmd_symbol(self.ppmd.decode_symbol(&mut self.bits)?) + } + + fn finish_ppmd_member(&mut self) -> Result { + let symbol = require_ppmd_symbol(self.ppmd.decode_symbol(&mut self.bits)?)?; + if symbol != self.ppmd_esc { + return Err(Error::InvalidData("RAR 2.9 PPMd member has trailing data")); + } + let next = require_ppmd_symbol(self.ppmd.decode_symbol(&mut self.bits)?)?; + match next { + 2 => { + self.in_lz_block = false; + Ok(true) + } + 0 => { + self.in_lz_block = false; + self.read_tables()?; + self.in_lz_block = true; + Ok(false) + } + _ => Err(Error::InvalidData("RAR 2.9 PPMd member has trailing data")), + } + } + + fn finish_member(&mut self) -> Result<()> { + loop { + let finished = match self.block_mode { + BlockMode::Lz => self.finish_lz_member()?, + BlockMode::Ppmd => self.finish_ppmd_member()?, + }; + if finished { + return Ok(()); + } + } + } + + fn finish_lz_member(&mut self) -> Result { + if !self.in_lz_block { + return Ok(true); + } + let symbol = self.main.decode(&mut self.bits)?; + if symbol != 256 { + return Err(Error::InvalidData("RAR 2.9 LZ member has trailing data")); + } + match self.read_end_of_block()? { + LzBlockEnd::SameFileNewTable => { + self.read_tables()?; + self.in_lz_block = true; + Ok(false) + } + LzBlockEnd::NewFileKeepTables | LzBlockEnd::NewFileNewTables => Ok(true), + } + } + + fn read_end_of_block(&mut self) -> Result { + let end = self.read_end_of_block_inner()?; + self.last_block_end = Some(end); + Ok(end) + } + + fn read_end_of_block_inner(&mut self) -> Result { + if self.bits.read_bit()? != 0 { + self.in_lz_block = false; + return Ok(LzBlockEnd::SameFileNewTable); + } + if self.bits.read_bit()? != 0 { + self.in_lz_block = false; + Ok(LzBlockEnd::NewFileNewTables) + } else { + self.in_lz_block = true; + Ok(LzBlockEnd::NewFileKeepTables) + } + } + + fn read_offset(&mut self) -> Result { + let slot = self.offsets.decode(&mut self.bits)?; + let mut offset = OFFSET_BASES[slot] + 1; + let extra_bits = OFFSET_BITS[slot]; + if extra_bits != 0 { + if slot > 9 { + if extra_bits > 4 { + offset += (self.bits.read_bits(extra_bits - 4)? as usize) << 4; + } + if self.low_offset_repeats > 0 { + self.low_offset_repeats -= 1; + offset += self.last_low_offset; + } else { + let low = self.low_offsets.decode(&mut self.bits)?; + if low == 16 { + self.low_offset_repeats = 15; + offset += self.last_low_offset; + } else { + self.last_low_offset = low; + offset += low; + } + } + } else { + offset += self.bits.read_bits(extra_bits)? as usize; + } + } + Ok(offset) + } + + fn read_vm_code(&mut self) -> Result<()> { + let mut poller = self.read_control.poller(); + let first_byte = self.bits.read_bits(8)?; + let mut len = (first_byte & 7) + 1; + if len == 7 { + len = self.bits.read_bits(8)? + 7; + } else if len == 8 { + len = self.bits.read_bits(16)?; + } + let mut data = Buffer::with_capacity(len as usize, &self.output.allowance())?; + for _ in 0..len { + poller.check_codec(data.len())?; + data.push_admitted(self.bits.read_bits(8)? as u8); + } + + self.parse_vm_code_owned(first_byte, data) + } + + fn read_vm_code_ppmd(&mut self) -> Result<()> { + let mut poller = self.read_control.poller(); + let first_byte = u32::from(self.read_ppmd_required_byte()?); + let mut len = (first_byte & 7) + 1; + if len == 7 { + len = u32::from(self.read_ppmd_required_byte()?) + 7; + } else if len == 8 { + len = (u32::from(self.read_ppmd_required_byte()?) << 8) + | u32::from(self.read_ppmd_required_byte()?); + } + let mut data = Buffer::with_capacity(len as usize, &self.output.allowance())?; + for _ in 0..len { + poller.check_codec(data.len())?; + data.push_admitted(self.read_ppmd_required_byte()?); + } + + self.parse_vm_code_owned(first_byte, data) + } + + fn parse_vm_code_owned(&mut self, first_byte: u32, data: Buffer) -> Result<()> { + let mut vm = BitReader { + input: data, + bit_pos: 0, + }; + let program_index = if first_byte & 0x80 != 0 { + let value = vm.read_encoded_u32()?; + if value == 0 { + self.filters.clear(); + self.programs.clear(); + 0 + } else { + usize::try_from(value - 1) + .map_err(|_| Error::InvalidData("RAR 2.9 VM program index overflows"))? + } + } else { + self.last_filter + }; + if program_index > self.programs.len() { + return Err(Error::InvalidData("RAR 2.9 VM program index is invalid")); + } + self.last_filter = program_index; + let new_program = program_index == self.programs.len(); + + let mut block_start = vm.read_encoded_u32()? as usize; + if first_byte & 0x40 != 0 { + block_start += 258; + } + block_start = self + .current_pos() + .checked_add(block_start) + .ok_or(Error::InvalidData("RAR 2.9 VM block start overflows"))?; + + let mut block_size = self + .programs + .get(program_index) + .map(|program| program.block_size) + .unwrap_or(0); + if first_byte & 0x20 != 0 { + block_size = vm.read_encoded_u32()? as usize; + } + + let mut regs = [0u32; 7]; + regs[3] = 0x3c000; + regs[4] = block_size as u32; + if let Some(program) = self.programs.get(program_index) { + regs[5] = program.exec_count; + } + if first_byte & 0x10 != 0 { + let mask = vm.read_bits(7)?; + for (index, reg) in regs.iter_mut().enumerate() { + if mask & (1 << index) != 0 { + *reg = vm.read_encoded_u32()?; + } + } + } + + if new_program { + if self.programs.len() >= MAX_VM_PROGRAMS { + return Err(Error::InvalidData("RAR 2.9 VM program limit exceeded")); + } + let code_size = vm.read_encoded_u32()? as usize; + if code_size == 0 { + return Err(Error::InvalidData("RAR 2.9 VM code is empty")); + } + if code_size >= MAX_VM_CODE_SIZE { + return Err(Error::InvalidData("RAR 2.9 VM code is too large")); + } + let mut code = Buffer::with_capacity(code_size, &self.output.allowance())?; + for _ in 0..code_size { + code.push_admitted(vm.read_bits(8)? as u8); + } + let kind = identify_standard_filter(&code) + .map(VmProgramKind::Standard) + .map_or_else( + || { + rarvm::OwnedProgram::parse(&code, &self.output.allowance()) + .map(VmProgramKind::Generic) + }, + Ok, + )?; + self.programs.try_push(VmProgram { + kind, + block_size, + exec_count: 0, + globals: Buffer::new(&self.output.allowance()), + })?; + } else { + // Equality is the new-program case above, and greater indices were + // rejected before parsing the record. + let program = &mut self.programs[program_index]; + program.exec_count = program.exec_count.wrapping_add(1); + program.block_size = block_size; + } + + let mut global_data = Buffer::new(&self.output.allowance()); + if first_byte & 0x08 != 0 { + let data_size = vm.read_encoded_u32()? as usize; + if data_size > MAX_VM_USER_GLOBAL_DATA { + return Err(Error::InvalidData("RAR 2.9 VM global data is too large")); + } + global_data = + Buffer::with_capacity(VM_SYSTEM_GLOBAL_SIZE + data_size, &self.output.allowance())?; + global_data.resize(VM_SYSTEM_GLOBAL_SIZE, 0)?; + for _ in 0..data_size { + global_data.push_admitted(vm.read_bits(8)? as u8); + } + } + + if self.filters.len() >= MAX_VM_FILTERS { + return Err(Error::InvalidData("RAR 2.9 VM filter limit exceeded")); + } + self.filters.try_push(VmFilter { + program: program_index, + start: block_start, + size: block_size, + regs, + global_data, + })?; + Ok(()) + } + + fn filtered_range_owned( + &mut self, + start: usize, + end: usize, + member_start: usize, + ) -> Result> { + let mut out = Buffer::with_capacity(end - start, &self.output.allowance())?; + let mut pos = start; + let filters = Buffer::collect( + self.filters + .iter() + .enumerate() + .filter_map(|(index, filter)| { + (filter.start >= start && filter.start + filter.size <= end).then_some(index) + }), + &self.output.allowance(), + )?; + let mut applied = Buffer::filled(self.filters.len(), false, &self.output.allowance())?; + let mut index = 0; + while index < filters.len() { + let first = self + .filters + .get(filters[index]) + .ok_or(Error::InvalidData("RAR 2.9 VM filter is missing"))?; + let filter_start = first.start; + let filter_size = first.size; + if filter_start < pos { + return Err(Error::InvalidData("RAR 2.9 VM filters partially overlap")); + } + out.extend_from_slice(self.raw_range(pos, filter_start)?) + .map_err(Into::into)?; + let mut block = Buffer::copied( + self.raw_range(filter_start, filter_start + filter_size)?, + &self.output.allowance(), + )?; + let file_offset = filter_start + .checked_sub(member_start) + .ok_or(Error::InvalidData("RAR 2.9 VM filter starts before file"))? + as u32; + loop { + let (program_index, regs, global_data) = { + let filter = self + .filters + .get(filters[index]) + .ok_or(Error::InvalidData("RAR 2.9 VM filter is missing"))?; + (filter.program, filter.regs, &filter.global_data) + }; + let program = self + .programs + .get_mut(program_index) + .ok_or(Error::InvalidData("RAR 2.9 VM program is missing"))?; + match &program.kind { + VmProgramKind::Standard(standard) => apply_standard_filter_with_allowance( + *standard, + &mut block, + file_offset, + ®s, + &self.read_control, + )?, + VmProgramKind::Generic(generic) => { + let globals = if global_data.is_empty() { + &program.globals[..] + } else { + &global_data[..] + }; + let result = generic.execute_with_control( + rarvm::Invocation { + input: &block, + regs, + global_data: globals, + file_offset: file_offset as u64, + exec_count: program.exec_count, + }, + &self.read_control, + )?; + program.globals = result.globals; + block = result.output; + } + } + applied[filters[index]] = true; + index += 1; + let Some(next) = filters.get(index).and_then(|&next| self.filters.get(next)) else { + break; + }; + if next.start != filter_start || next.size != block.len() { + break; + } + } + out.extend_from_slice(&block).map_err(Into::into)?; + pos = filter_start + filter_size; + } + out.extend_from_slice(self.raw_range(pos, end)?) + .map_err(Into::into)?; + let mut index = 0; + self.filters.retain(|_| { + let keep = !applied[index]; + index += 1; + keep + }); + Ok(out) + } + + fn safe_flush_end(&self, start: usize, end: usize, final_target: usize) -> Result { + let current = self.current_pos(); + let mut safe_end = end; + for filter in &self.filters { + let filter_end = filter + .start + .checked_add(filter.size) + .ok_or(Error::InvalidData("RAR 2.9 VM filter size overflows"))?; + if filter.start >= safe_end || filter_end <= start { + continue; + } + if filter_end > final_target { + return Err(Error::InvalidData( + "RAR 2.9 VM filter extends beyond output", + )); + } + if filter_end > current { + safe_end = safe_end.min(filter.start); + } + } + Ok(safe_end) + } + + fn copy_match(&mut self, length: usize, offset: usize, output_size: usize) -> Result<()> { + // A match reaching past the start of the stream writes zeroes rather + // than failing. Current UnRAR makes the same decision with its + // first-window flag, and libarchive reads from a zero-initialized + // circular dictionary. The decision is taken once for the whole + // match: a copy does not start on zeroes and cross into real bytes + // partway. + let before_window = offset > self.current_pos(); + for index in 0..length { + if self.current_pos() >= output_size { + self.pending_match = Some((length - index, offset)); + break; + } + let byte = if before_window { + 0 + } else { + let src = self.current_pos() - offset; + *self + .raw_byte(src) + .ok_or(Error::InvalidData("RAR 2.9 match distance is out of range"))? + }; + self.output.try_push(byte)?; + } + Ok(()) + } + + fn drain_pending_match(&mut self, output_size: usize) -> Result<()> { + let Some((length, offset)) = self.pending_match.take() else { + return Ok(()); + }; + self.copy_match(length, offset, output_size) + } + + fn push_old_offset(&mut self, offset: usize) { + self.old_offsets[3] = self.old_offsets[2]; + self.old_offsets[2] = self.old_offsets[1]; + self.old_offsets[1] = self.old_offsets[0]; + self.old_offsets[0] = offset; + } + + fn rotate_old_offset(&mut self, index: usize) { + let value = self.old_offsets[index]; + for i in (1..=index).rev() { + self.old_offsets[i] = self.old_offsets[i - 1]; + } + self.old_offsets[0] = value; + } + + fn current_pos(&self) -> usize { + self.base_offset + self.output.len() + } + + fn raw_byte(&self, position: usize) -> Option<&u8> { + self.output.get(position.checked_sub(self.base_offset)?) + } + + fn raw_range(&self, start: usize, end: usize) -> Result<&[u8]> { + if start < self.base_offset || end < start { + return Err(Error::InvalidData( + "RAR 2.9 retained history is unavailable", + )); + } + let rel_start = start - self.base_offset; + let rel_end = end - self.base_offset; + self.output + .get(rel_start..rel_end) + .ok_or(Error::InvalidData( + "RAR 2.9 retained history is unavailable", + )) + } + + fn trim_history(&mut self, flushed_pos: usize, current_pos: usize) { + let keep_from = current_pos.saturating_sub(MAX_HISTORY); + let keep_from = keep_from.min(flushed_pos); + if keep_from <= self.base_offset { + return; + } + let drain = keep_from - self.base_offset; + self.output.discard_prefix(drain); + self.base_offset = keep_from; + self.filters + .retain(|filter| filter.start + filter.size > self.base_offset); + } +} + +#[cfg(all(test, feature = "write"))] +impl Reader29State { + fn filtered_range(&mut self, start: usize, end: usize, member_start: usize) -> Result> { + self.filtered_range_owned(start, end, member_start) + .map(Buffer::into_vec) + } + fn parse_vm_code(&mut self, first_byte: u32, data: Vec) -> Result<()> { + self.parse_vm_code_owned(first_byte, data.into()) + } + + fn new() -> Self { + Self::with_allowance(&Allowance::default()) + } + fn decode_member(&mut self, input: &[u8], output_size: usize) -> Result> { + self.decode_member_owned(input, output_size) + .map(Buffer::into_vec) + } + fn decode_non_solid_member(&mut self, input: &[u8], output_size: usize) -> Result> { + self.decode_non_solid_member_owned(input, output_size) + .map(Buffer::into_vec) + } +} +fn fill_levels(levels: &mut [u8], pos: &mut usize, count: usize, value: u8) -> Result<()> { + let end = pos + .checked_add(count) + .ok_or(Error::InvalidData("RAR 2.9 table run overflows"))?; + let end = end.min(levels.len()); + for item in &mut levels[*pos..end] { + *item = value; + } + *pos = end; + Ok(()) +} + +#[derive(Debug)] +struct Huffman { + state: super::canonical::Huffman, +} + +impl Huffman { + fn with_allowance(allowance: &B) -> Self { + Self { + state: super::canonical::Huffman::with_allowance(allowance), + } + } + fn from_lengths_with_allowance(lengths: &[u8], allowance: &B) -> Result { + let mut count = [0u16; 16]; + for &len in lengths { + if len != 0 { + count[len as usize] += 1; + } + } + if count.iter().all(|&value| value == 0) { + return Ok(Self::with_allowance(allowance)); + } + validate_huffman_counts(&count)?; + + Ok(Self { + state: super::canonical::Huffman::from_counts(lengths, count, allowance)?, + }) + } + fn try_clone(&self) -> Result { + Ok(Self { + state: self.state.try_clone()?, + }) + } + fn decode(&self, bits: &mut BitReader) -> Result { + self.state.decode( + || bits.read_bit().map(|bit| bit as u16), + "RAR 2.9 empty Huffman table", + "RAR 2.9 invalid Huffman code", + ) + } +} + +#[cfg(all(test, feature = "write"))] +impl Huffman { + fn from_lengths(lengths: &[u8]) -> Result { + Self::from_lengths_with_allowance(lengths, &Allowance::default()) + } +} + +fn validate_huffman_counts(count: &[u16; 16]) -> Result<()> { + super::canonical::validate_counts(count, "RAR 2.9 oversubscribed Huffman table") +} + +#[derive(Debug)] +struct BitReader { + input: Buffer, + bit_pos: usize, +} + +impl BitReader { + fn with_allowance(allowance: &B) -> Self { + Self { + input: Buffer::new(allowance), + bit_pos: 0, + } + } + + fn from_bytes_with_allowance(input: &[u8], allowance: &B) -> Result { + Ok(Self { + input: Buffer::copied(input, allowance)?, + bit_pos: 0, + }) + } + fn try_clone(&self) -> Result { + Ok(Self { + input: Buffer::copied(&self.input, &self.input.allowance())?, + bit_pos: self.bit_pos, + }) + } + + #[cfg(all(test, feature = "write"))] + fn append(&mut self, input: &[u8]) { + self.compact(); + self.input + .extend_from_slice(input) + .map_err(Into::into) + .expect("test input allowance"); + } + + #[cfg(all(test, feature = "write"))] + fn compact(&mut self) { + let bytes = self.bit_pos / 8; + if bytes == 0 { + return; + } + self.input.discard_prefix(bytes); + self.bit_pos -= bytes * 8; + } + + fn align_byte(&mut self) { + self.bit_pos = (self.bit_pos + 7) & !7; + } + + fn peek_bit(&self) -> Result { + self.peek_bits(1).map(|value| value as u8) + } + + fn read_bit(&mut self) -> Result { + self.read_bits(1).map(|value| value as u8) + } + + fn read_bits(&mut self, count: u8) -> Result { + let value = self.peek_bits(count)?; + self.bit_pos += count as usize; + Ok(value) + } + + fn peek_bits(&self, count: u8) -> Result { + if count > 24 { + return Err(Error::InvalidData("RAR 2.9 bit read is too wide")); + } + let mut value = 0u32; + for i in 0..count as usize { + let bit_index = self.bit_pos + i; + let byte = *self.input.get(bit_index / 8).ok_or(Error::NeedMoreInput)?; + let bit = (byte >> (7 - (bit_index % 8))) & 1; + value = (value << 1) | bit as u32; + } + Ok(value) + } + + fn read_encoded_u32(&mut self) -> Result { + match self.read_bits(2)? { + 0 => self.read_bits(4), + 1 => { + let high = self.read_bits(8)?; + if high >= 16 { + Ok(high) + } else { + Ok(0xffff_ff00 | (high << 4) | self.read_bits(4)?) + } + } + 2 => self.read_bits(16), + _ => Ok((self.read_bits(16)? << 16) | self.read_bits(16)?), + } + } +} + +impl PpmdByteReader for BitReader { + fn read_ppmd_byte(&mut self) -> Result { + self.read_bits(8).map(|value| value as u8) + } +} + +#[cfg(all(test, feature = "write"))] +impl BitReader { + fn from_bytes(input: &[u8]) -> Self { + Self::from_bytes_with_allowance(input, &Allowance::default()).expect("unlimited RAR3 input") + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar29/encoder.rs b/arca-rar/vendor/rars/src/codec/rar29/encoder.rs new file mode 100644 index 00000000..481abd94 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar29/encoder.rs @@ -0,0 +1,2321 @@ +//! RAR29 LZ and PPMd encoding, filter preparation and table emission. +use super::super::ppmd::{PpmdDecoder, PpmdEncoder}; +use super::super::{ + filters::{self, FilterOp}, + huffman, match_finder, +}; +use super::standard_filters::rgb_predict; +use super::*; + +const MAX_VM_FILTER_BLOCK_SIZE: usize = 128 * 1024; +// The standard AUDIO bytecode uses separate input/output regions inside RARVM +// memory. Keep generated blocks below the overlap boundary accepted by period +// decoders. +pub(crate) const MAX_VM_DELTA_FILTER_BLOCK_SIZE: usize = 120_000; +const MAX_VM_AUDIO_FILTER_BLOCK_SIZE: usize = 120_000; +const MAX_ENCODER_MATCH_OFFSET: usize = 1024 * 1024; +const MAX_ENCODER_MATCH_LENGTH: usize = 258; +const MAX_MATCH_CANDIDATES: usize = 256; +const MAX_PPMD_MATCH_LENGTH: usize = 255; +const MIN_PPMD_MATCH_LENGTH: usize = 32; +const MAX_PPMD_REPEAT_LENGTH: usize = 259; +// The parameters rar 3.00 itself declares at -m5, read out of its streams. +const PPMD_ORDER: usize = 8; +const PPMD_DICTIONARY_MB: u8 = 25; +const PPMD_ESC: u8 = 2; +// Seeds and weights for the escape-token cost model in `encode_ppmd_hybrid`. +// The seeds only steer the first few decisions; measured costs take over as +// the member is coded. All five are tuning knobs, not format constants. +const PPMD_LITERAL_BITS_SEED: f64 = 4.0; +const PPMD_MATCH_BITS_SEED: f64 = 60.0; +const PPMD_REPEAT_BITS_SEED: f64 = 24.0; +const PPMD_LITERAL_EMA_WEIGHT: f64 = 1.0 / 32.0; +const PPMD_TOKEN_EMA_WEIGHT: f64 = 1.0 / 8.0; +// An escape token's copied bytes never reach the model, so the literals right +// after it are predicted from the token's own bytes and pay for the broken +// context. That cost lands on the literals' ledger, not the token's, so the +// token is charged a flat estimate of it here. +const PPMD_CONTEXT_BREAK_BITS: f64 = 16.0; +// After a match is priced out, nearby positions almost always price out the +// same way, so the search sleeps a few bytes rather than re-walking the hash +// chain at every literal. +const PPMD_REJECT_SEARCH_COOLDOWN: usize = 8; + +type Rar29MatchFinder = match_finder::MatchFinder<4>; + +// RAR 3.x standard filters are stored as RARVM bytecode in the compressed +// stream. RAR15_40_FORMAT_SPECIFICATION.md §20 and FILTER_TRANSFORMS.md §9 +// define these blobs by byte length plus CRC32 fingerprint; keep the bytes +// verbatim so writer output and reader recognition use the same wire identity. +const RAR3_E8_FILTER_BYTECODE: &[u8] = &[ + 0x97, 0x1b, 0x01, 0x28, 0x07, 0x06, 0x98, 0x08, 0x00, 0x00, 0x00, 0xd1, 0x3a, 0x10, 0x15, 0x92, + 0xec, 0x50, 0xcb, 0x99, 0x20, 0xb9, 0x25, 0xf0, 0x29, 0x19, 0x15, 0x53, 0x03, 0x12, 0xae, 0x51, + 0x10, 0x35, 0x59, 0x2b, 0x60, 0x04, 0x15, 0x6d, 0x40, 0x66, 0xab, 0x02, 0x34, 0x49, 0x04, 0x36, + 0x02, 0x52, 0x3e, 0x97, 0x00, +]; +const RAR3_E8E9_FILTER_BYTECODE: &[u8] = &[ + 0x84, 0x1b, 0x01, 0x28, 0x11, 0x10, 0x69, 0x80, 0x80, 0x00, 0x00, 0x0d, 0x13, 0xa1, 0x01, 0xc6, + 0x89, 0xd2, 0x80, 0xac, 0x97, 0x62, 0x85, 0x5c, 0xc9, 0x05, 0xc9, 0x2f, 0x81, 0x48, 0xc8, 0xaa, + 0x98, 0x18, 0x95, 0x72, 0x88, 0x81, 0xaa, 0xc9, 0x5b, 0x00, 0x20, 0xab, 0x6a, 0x03, 0x35, 0x58, + 0x11, 0xa2, 0x48, 0x21, 0xb0, 0x12, 0x91, 0xf4, 0xb8, +]; +const RAR3_DELTA_FILTER_BYTECODE: &[u8] = &[ + 0x2f, 0x01, 0x9a, 0x41, 0x80, 0xec, 0x27, 0x48, 0x2f, 0x09, 0x76, 0x6d, 0xd3, 0xea, 0x41, 0x5b, + 0x59, 0x44, 0xe8, 0x17, 0x5c, 0xe1, 0x6c, 0x91, 0x4c, 0x4e, 0x3f, 0x77, 0x00, +]; +const RAR3_ITANIUM_FILTER_BYTECODE: &[u8] = &[ + 0x46, 0x9e, 0x08, 0x08, 0x0c, 0x0c, 0x00, 0x00, 0x0e, 0x0e, 0x08, 0x08, 0x00, 0x00, 0x08, 0x08, + 0x00, 0x00, 0x6c, 0x11, 0x5a, 0x04, 0xac, 0x0c, 0xc4, 0xcc, 0x5c, 0x08, 0x18, 0x46, 0x24, 0x08, + 0xf9, 0xa0, 0x44, 0x25, 0x12, 0x12, 0x45, 0x85, 0x99, 0x0c, 0x14, 0x00, 0x26, 0x25, 0x58, 0x99, + 0x90, 0x03, 0x38, 0x1a, 0x08, 0xdc, 0x02, 0x30, 0x0c, 0x4e, 0xd1, 0x1d, 0x89, 0xa1, 0xe2, 0xd0, + 0x55, 0x11, 0x33, 0x60, 0x8c, 0x5a, 0x23, 0x06, 0xde, 0x06, 0x18, 0x00, 0x7f, 0xff, 0xfc, 0x4d, + 0xcc, 0x19, 0x17, 0xb3, 0x06, 0xc4, 0x44, 0xb2, 0x32, 0x5a, 0x44, 0xc4, 0xa6, 0x01, 0xf4, 0x24, + 0x88, 0x83, 0x38, 0xcc, 0xc4, 0x11, 0x09, 0x87, 0xa6, 0xe0, 0x46, 0x02, 0xb2, 0x24, 0x03, 0xe2, + 0xa0, 0x32, 0x54, 0x83, 0x52, 0xc5, 0xb1, 0x70, +]; +const RAR3_RGB_FILTER_BYTECODE: &[u8] = &[ + 0xc5, 0x01, 0x9a, 0x41, 0x95, 0xc9, 0xa6, 0x4d, 0xba, 0x4b, 0x14, 0x0a, 0xf4, 0x9b, 0x80, 0x4c, + 0x00, 0x15, 0xa6, 0xa8, 0x07, 0x26, 0x2a, 0xc9, 0xc4, 0x8b, 0x86, 0x62, 0x32, 0x0f, 0x86, 0x64, + 0x24, 0x06, 0x66, 0x71, 0x19, 0x98, 0xcc, 0x43, 0x33, 0x31, 0x99, 0x00, 0x66, 0x88, 0x33, 0x30, + 0xcc, 0xd1, 0x0e, 0x98, 0x0b, 0x33, 0x34, 0x40, 0x0c, 0xd1, 0x46, 0x66, 0x19, 0x9a, 0x28, 0xcc, + 0x49, 0x80, 0xb3, 0x33, 0x45, 0x00, 0xcd, 0x18, 0x66, 0x61, 0x99, 0xa3, 0x0c, 0xc8, 0x98, 0x0b, + 0x33, 0x34, 0x60, 0x4c, 0xd1, 0x06, 0x68, 0xa5, 0x20, 0x62, 0x66, 0x88, 0x33, 0x46, 0x28, 0x05, + 0x0f, 0x32, 0x0c, 0x4c, 0xd1, 0x46, 0x68, 0xc5, 0x00, 0x41, 0xe4, 0x8f, 0xc8, 0x85, 0x5e, 0x02, + 0x7c, 0xc9, 0x26, 0x81, 0x83, 0xb0, 0x9d, 0xc2, 0xde, 0x9c, 0x78, 0xac, 0xd6, 0x68, 0xb4, 0x0e, + 0x71, 0xdb, 0xb2, 0x49, 0x38, 0x6e, 0x02, 0x2a, 0x2c, 0x41, 0x2b, 0x10, 0x98, 0x82, 0x49, 0x03, + 0x14, 0xf4, 0xe1, 0x97, 0x00, +]; +const RAR3_AUDIO_FILTER_BYTECODE: &[u8] = &[ + 0x47, 0x01, 0x9a, 0x41, 0x95, 0xe5, 0x72, 0x0d, 0xc2, 0x64, 0x82, 0x74, 0x93, 0x24, 0xb1, 0x40, + 0x06, 0xd8, 0x38, 0x44, 0x00, 0xa8, 0x01, 0x34, 0x11, 0xdc, 0xa1, 0xba, 0x01, 0x99, 0x0c, 0xc4, + 0x03, 0x31, 0x19, 0xa4, 0x06, 0x66, 0x22, 0x60, 0x4d, 0x9a, 0x40, 0x0d, 0x66, 0x8e, 0x60, 0xd0, + 0x30, 0x40, 0x18, 0x26, 0xc1, 0xc8, 0xf6, 0xe6, 0x26, 0x13, 0x78, 0x92, 0x08, 0xe8, 0x50, 0xbc, + 0x5a, 0x07, 0xc6, 0xe9, 0xf5, 0x20, 0xa9, 0xa0, 0xed, 0x37, 0x33, 0x47, 0x39, 0x66, 0x90, 0x70, + 0x19, 0xa3, 0x9b, 0xcf, 0x25, 0x83, 0x80, 0xc1, 0xbd, 0x30, 0x16, 0x6e, 0x23, 0x34, 0x93, 0x81, + 0x16, 0x09, 0xb0, 0x50, 0x18, 0x3b, 0x4d, 0xc8, 0x4c, 0x05, 0x9b, 0x88, 0xc5, 0x28, 0xe0, 0x76, + 0x93, 0x90, 0x98, 0x0b, 0x37, 0x11, 0x8a, 0x59, 0xc4, 0x80, 0x42, 0x48, 0x43, 0xa9, 0x47, 0xee, + 0x43, 0x34, 0x60, 0x47, 0xd4, 0x4a, 0x0d, 0xbb, 0xd3, 0x59, 0xa4, 0x86, 0xee, 0x05, 0x09, 0x40, + 0x26, 0xc9, 0x34, 0x24, 0x76, 0xa0, 0x30, 0x6a, 0x20, 0xea, 0x02, 0x20, 0x04, 0xa0, 0x41, 0x50, + 0x9e, 0x50, 0x3f, 0xe6, 0xe1, 0x28, 0x94, 0x46, 0x01, 0xbd, 0x8b, 0x40, 0xf0, 0x68, 0x11, 0x36, + 0xc9, 0xa1, 0x92, 0x38, 0x11, 0x41, 0x9c, 0xa8, 0x95, 0x10, 0xee, 0x50, 0x66, 0x2b, 0x00, 0x20, + 0x95, 0x11, 0x04, 0x02, 0x62, 0xac, 0x66, 0x8c, 0x6a, 0xca, 0x26, 0x40, 0xb2, 0x67, 0x1b, 0x4b, + 0x26, 0xcc, 0x64, 0x8a, 0x62, 0x71, 0xa2, 0xb8, +]; + +pub fn unpack29_encode_literals(input: &[u8]) -> Result> { + encode_member(input, &[]) +} + +pub fn unpack29_encode_literals_with_options( + input: &[u8], + options: EncodeOptions, +) -> Result> { + encode_member_with_options(input, &[], options) +} + +pub(crate) fn unpack29_encode_literals_with_options_and_progress( + input: &[u8], + options: EncodeOptions, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + encode_member_with_options_and_progress(input, &[], options, &mut [0; TABLE_COUNT], progress) +} + +pub fn unpack29_encode_ppmd_literals(input: &[u8]) -> Result> { + encode_ppmd_member(input, false, &[], 0) +} + +pub(crate) fn unpack29_encode_ppmd_with_progress( + input: &[u8], + lz_escapes: bool, + filter: Option, + max_match_distance: usize, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + if !progress(0) { + return Err(Error::Cancelled); + } + let filtered = if let Some(filter) = filter { + let filters = split_large_filter(input.len(), filter)?; + Some(filtered_members_with_progress( + input, + &filters, + Some(&mut *progress), + )?) + } else { + None + }; + let records = if let Some(filtered) = &filtered { + let refs: Vec<_> = filtered.records.iter().collect(); + encoded_filter_records_at(&refs, 0, usize::MAX, &mut Vec::new())? + } else { + Vec::new() + }; + encode_ppmd_block_with_model( + filtered.as_ref().map_or(input, |filtered| &filtered.data), + lz_escapes, + &records, + max_match_distance, + None, + Some(progress), + ) + .map(|(packed, _)| packed) +} + +/// `max_match_distance` is the dictionary the file header declares. PPMd's +/// escape-4 matches copy out of the same window the LZ decoder uses, so a match +/// that reaches further back than the header promises lands on whatever the +/// decoder still happens to hold, and unrar fails the member on its checksum. +pub fn unpack29_encode_ppmd(input: &[u8], max_match_distance: usize) -> Result> { + encode_ppmd_member(input, true, &[], max_match_distance) +} + +pub fn unpack29_encode_ppmd_with_filter( + input: &[u8], + filter: crate::FilterSpec, + max_match_distance: usize, +) -> Result> { + encode_ppmd_filtered_member(input, filter, true, max_match_distance) +} + +fn encode_ppmd_filtered_member( + input: &[u8], + filter: crate::FilterSpec, + lz_escapes: bool, + max_match_distance: usize, +) -> Result> { + let filters = split_large_filter(input.len(), filter)?; + let filtered = filtered_members(input, &filters)?; + // PPMd codes the member as one unit rather than in LZ blocks, so every + // record is declared at the start and the window does not constrain them. + let refs: Vec<&OwnedVmFilterRecord> = filtered.records.iter().collect(); + let records = encoded_filter_records_at(&refs, 0, usize::MAX, &mut Vec::new())?; + encode_ppmd_member(&filtered.data, lz_escapes, &records, max_match_distance) +} + +pub(crate) fn filtered_members( + input: &[u8], + filters: &[crate::FilterSpec], +) -> Result { + filtered_members_with_progress(input, filters, None) +} + +fn filtered_members_with_progress( + input: &[u8], + filters: &[crate::FilterSpec], + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result { + let mut ordered = filters.iter().collect::>(); + ordered.sort_by_key(|filter| filter.range.as_ref().map_or(0, |range| range.start)); + let mut data = input.to_vec(); + let mut records = Vec::with_capacity(filters.len()); + let mut index = 0; + let mut previous_end = 0; + while index < ordered.len() { + let range = checked_filter_range(input.len(), ordered[index])?; + if range.start < previous_end { + return Err(Error::InvalidData("RAR 2.9 VM filters partially overlap")); + } + + let mut group_end = index + 1; + while group_end < ordered.len() + && ordered[group_end].range.clone().unwrap_or(0..input.len()) == range + { + group_end += 1; + } + let mut group_records = Vec::with_capacity(group_end - index); + // Decoding runs records in wire order. Apply their inverses in reverse + // so each decoder invocation receives the prior one's output. + for filter in ordered[index..group_end].iter().rev() { + // Large ranges have already been split into bounded records. Poll + // between them without advancing encoded-byte progress, so filter + // preprocessing cannot hide cancellation for an entire member. + if progress.as_mut().is_some_and(|report| !report(0)) { + return Err(Error::Cancelled); + } + let filtered = filtered_member(&data, filter)?; + data[range.clone()].copy_from_slice(&filtered.data[range.clone()]); + group_records.push(OwnedVmFilterRecord { + block_start: filtered.block_start, + block_size: filtered.block_size, + init_regs: filtered.init_regs, + code: filtered.code, + global_data: Vec::new(), + }); + } + group_records.reverse(); + records.extend(group_records); + previous_end = range.end; + index = group_end; + } + Ok(FilteredMembers { data, records }) +} + +pub(crate) struct FilteredMembers { + pub(crate) data: Vec, + records: Vec, +} + +fn split_large_filter( + input_len: usize, + filter: crate::FilterSpec, +) -> Result> { + let range = checked_filter_range(input_len, &filter)?; + + // The smallest run of bytes each filter can still transform. A trailing + // chunk shorter than this is left unfiltered rather than handed to a filter + // that cannot process it. + let unit = match rar29_filter(filter.kind)? { + Rar29Filter::Delta { channels } | Rar29Filter::Audio { channels } => channels, + Rar29Filter::Rgb { width, .. } => width.max(3), + Rar29Filter::E8 | Rar29Filter::E8E9 | Rar29Filter::Itanium => 4, + }; + let chunk_size = match rar29_filter(filter.kind)? { + Rar29Filter::Delta { channels } => { + if channels == 0 || channels > MAX_VM_DELTA_FILTER_BLOCK_SIZE { + return Err(Error::InvalidData( + "RAR 2.9 VM filter channel count is invalid", + )); + } + MAX_VM_DELTA_FILTER_BLOCK_SIZE - (MAX_VM_DELTA_FILTER_BLOCK_SIZE % channels) + } + Rar29Filter::Audio { channels } => { + if channels == 0 || channels > MAX_AUDIO_CHANNELS { + return Err(Error::InvalidData( + "RAR 2.9 VM filter channel count is invalid", + )); + } + MAX_VM_AUDIO_FILTER_BLOCK_SIZE - (MAX_VM_AUDIO_FILTER_BLOCK_SIZE % channels) + } + Rar29Filter::Rgb { width, .. } => { + if width == 0 || width > MAX_VM_FILTER_BLOCK_SIZE { + return Err(Error::InvalidData( + "RAR 2.9 RGB filter scanline width is invalid", + )); + } + MAX_VM_FILTER_BLOCK_SIZE - (MAX_VM_FILTER_BLOCK_SIZE % width) + } + Rar29Filter::E8 | Rar29Filter::E8E9 | Rar29Filter::Itanium => MAX_VM_FILTER_BLOCK_SIZE, + }; + if range.len() <= chunk_size { + return Ok(vec![filter]); + } + + let mut filters = Vec::new(); + let mut start = range.start; + while start < range.end { + let end = (start + chunk_size).min(range.end); + // Chunking can leave a remainder the filter has no way to transform. + // Those bytes stay as they are; a filter covering part of a member is + // exactly what a range is for. + if end - start < unit { + break; + } + filters.push(crate::FilterSpec::range(filter.kind, start..end)); + start = end; + } + Ok(filters) +} + +fn checked_filter_range( + input_len: usize, + filter: &crate::FilterSpec, +) -> Result> { + let range = filter.range.clone().unwrap_or(0..input_len); + if range.start >= range.end || range.end > input_len { + return Err(Error::InvalidData("RAR 2.9 VM filter range is invalid")); + } + Ok(range) +} + +struct OwnedVmFilterRecord { + block_start: usize, + block_size: usize, + init_regs: Vec<(usize, u32)>, + code: &'static [u8], + global_data: Vec, +} + +fn encode_ppmd_member( + input: &[u8], + lz_escapes: bool, + initial_filters: &[Vec], + max_match_distance: usize, +) -> Result> { + encode_ppmd_block(input, lz_escapes, initial_filters, max_match_distance) +} + +fn encode_ppmd_block( + input: &[u8], + lz_escapes: bool, + initial_filters: &[Vec], + max_match_distance: usize, +) -> Result> { + encode_ppmd_block_with_model( + input, + lz_escapes, + initial_filters, + max_match_distance, + None, + None, + ) + .map(|(packed, _)| packed) +} + +/// Codes one PPMd block, either starting a model or carrying one on. +/// +/// The header byte's 0x20 bit tells the reader to throw its model away and +/// build a new one from the order and dictionary size that follow. Clearing it +/// leaves the reader's model where it is, which is how a solid chain gets a +/// model that has already read every member before this one. WinRAR does this +/// on 67 of the 68 PPMd blocks in a solid archive of 70 small files, and the +/// difference is most of why that archive is 18% smaller than ours was. +/// +/// The model comes back out so the next block can continue it in turn. +fn encode_ppmd_block_with_model( + input: &[u8], + lz_escapes: bool, + initial_filters: &[Vec], + max_match_distance: usize, + model: Option, + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result<(Vec, PpmdDecoder)> { + if progress.as_mut().is_some_and(|report| !report(0)) { + return Err(Error::Cancelled); + } + let mut out = Vec::new(); + let mut encoder = match model { + Some(model) => { + out.push(0x80 | ((PPMD_ORDER as u8) - 1)); + PpmdEncoder::continuing(model, PPMD_ESC) + } + None => { + out.push(0x80 | 0x20 | ((PPMD_ORDER as u8) - 1)); + out.push(PPMD_DICTIONARY_MB - 1); + PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, usize::from(PPMD_DICTIONARY_MB))? + } + }; + for record in initial_filters { + encoder.encode_vm_filter_record(record)?; + } + if lz_escapes { + let mut report = |position| progress.as_mut().is_none_or(|report| report(position)); + encode_ppmd_hybrid_with_progress( + input, + max_match_distance, + &mut encoder, + |_| (), + Some(&mut report), + )?; + } else { + for (position, &byte) in input.iter().enumerate() { + if position.is_multiple_of(4096) + && progress.as_mut().is_some_and(|report| !report(position)) + { + return Err(Error::Cancelled); + } + encoder.encode_literal(byte)?; + } + } + let (packed, model) = encoder.finish_keeping_model()?; + out.extend_from_slice(&packed); + if progress.as_mut().is_some_and(|report| !report(input.len())) { + return Err(Error::Cancelled); + } + Ok((out, model)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PpmdEncodeToken { + Literal(u8), + RepeatOffsetOne { length: usize }, + Match { offset: usize, length: usize }, +} + +/// The writer rejects these before compressing anything, so reaching this is +/// either a direct `codec` caller or a bug. Either way the codec stays total. +fn rar29_filter(kind: crate::FilterKind) -> Result { + Rar29Filter::try_from(kind) + .map_err(|_| Error::InvalidData("the RAR 2.9 family has no program for this filter")) +} + +struct FilteredMember { + data: Vec, + block_start: usize, + block_size: usize, + init_regs: Vec<(usize, u32)>, + code: &'static [u8], +} + +fn filtered_member(input: &[u8], filter: &crate::FilterSpec) -> Result { + let range = checked_filter_range(input.len(), filter)?; + let mut filtered = input.to_vec(); + let (init_regs, code): (Vec<(usize, u32)>, &'static [u8]) = match rar29_filter(filter.kind)? { + Rar29Filter::E8 => { + filters::e8e9_encode(&mut filtered[range.clone()], range.start as u32, false); + (Vec::new(), RAR3_E8_FILTER_BYTECODE) + } + Rar29Filter::E8E9 => { + filters::e8e9_encode(&mut filtered[range.clone()], range.start as u32, true); + (Vec::new(), RAR3_E8E9_FILTER_BYTECODE) + } + Rar29Filter::Delta { channels } => { + filters::encode_in_place( + FilterOp::Delta { channels }, + &mut filtered[range.clone()], + 0, + rar29_delta_messages(), + )?; + (vec![(0, channels as u32)], RAR3_DELTA_FILTER_BYTECODE) + } + Rar29Filter::Itanium => { + itanium_encode(&mut filtered[range.clone()], range.start as u32); + (Vec::new(), RAR3_ITANIUM_FILTER_BYTECODE) + } + Rar29Filter::Rgb { width, pos_r } => { + filtered[range.clone()].copy_from_slice(&rgb_encode( + &input[range.clone()], + width, + pos_r, + )?); + let init_regs = if pos_r == 0 { + vec![(0, width as u32 + 3)] + } else { + vec![(0, width as u32 + 3), (1, pos_r as u32)] + }; + (init_regs, RAR3_RGB_FILTER_BYTECODE) + } + Rar29Filter::Audio { channels } => { + filtered[range.clone()] + .copy_from_slice(&audio_encode(&input[range.clone()], channels)?); + (vec![(0, channels as u32)], RAR3_AUDIO_FILTER_BYTECODE) + } + }; + Ok(FilteredMember { + data: filtered, + block_start: range.start, + block_size: range.end - range.start, + init_regs, + code, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct EncodeOptions { + pub max_match_candidates: usize, + pub lazy_matching: bool, + pub lazy_lookahead: usize, + pub max_match_distance: usize, + pub block_size: Option, +} + +impl EncodeOptions { + pub const fn new(max_match_candidates: usize) -> Self { + Self { + max_match_candidates, + lazy_matching: false, + lazy_lookahead: 1, + max_match_distance: MAX_ENCODER_MATCH_OFFSET, + block_size: None, + } + } + + pub const fn with_lazy_matching(mut self, enabled: bool) -> Self { + self.lazy_matching = enabled; + self + } + + pub const fn with_lazy_lookahead(mut self, bytes: usize) -> Self { + self.lazy_lookahead = bytes; + self + } + + pub const fn with_max_match_distance(mut self, distance: usize) -> Self { + self.max_match_distance = if distance > MAX_HISTORY { + MAX_HISTORY + } else { + distance + }; + self + } + + pub const fn with_block_size(mut self, bytes: usize) -> Self { + self.block_size = Some(bytes); + self + } + + const fn constrained(mut self) -> Self { + if self.max_match_distance > MAX_HISTORY { + self.max_match_distance = MAX_HISTORY; + } + self + } +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self::new(MAX_MATCH_CANDIDATES) + } +} + +/// Which engine a member of a solid chain is coded with. +/// +/// A chain used to be LZ and nothing else. Both engines keep state that carries +/// from member to member, so this is a per-member choice inside one chain +/// rather than a property of the archive. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChainEngine { + Lz, + Ppmd, + /// Code it both ways and keep whichever is smaller. Worth asking for when + /// the content might go either way, and not otherwise: it doubles the work + /// and copying the model to try costs more the longer the chain has run. + Smaller, +} + +/// One way of coding a member into the chain, and the state it would leave. +/// +/// The state is carried rather than committed because a candidate that loses +/// must not move the chain: the reader rebuilds its code-length table from the +/// bytes it actually reads, so committing a table the winner never wrote leaves +/// every later member coded against a table no decoder holds. +struct LzCandidate { + packed: Vec, + /// The bytes the LZ layer coded, when a filter rewrote them. `None` when + /// the candidate took no filter and coded the input as it came. + coded: Option>, + levels: [u8; TABLE_COUNT], +} + +#[derive(Debug, Clone)] +pub struct Unpack29Encoder { + history: Vec, + options: EncodeOptions, + /// The code-length table a reader holds once everything coded so far has + /// been read. A solid chain carries it from one member to the next, which + /// is what lets a member say "same table as before" instead of spelling one + /// out. A reader clears it on a member that does not continue a chain, and + /// so does a fresh encoder. + levels: [u8; TABLE_COUNT], + /// The PPMd model a reader holds, once some member in the chain has built + /// one. A reader keeps it across every block that does not ask for a reset, + /// LZ blocks included, so a member can go PPMd against everything the chain + /// has read even when the member before it went LZ. + ppmd: Option, +} + +impl Default for Unpack29Encoder { + fn default() -> Self { + Self::with_options(EncodeOptions::default()) + } +} + +impl Unpack29Encoder { + pub fn new() -> Self { + Self::default() + } + + pub fn with_options(options: EncodeOptions) -> Self { + Self { + history: Vec::new(), + options: options.constrained(), + levels: [0; TABLE_COUNT], + ppmd: None, + } + } + + pub fn encode_member(&mut self, input: &[u8]) -> Result> { + let packed = encode_member_with_options_impl( + input, + &self.history, + self.options, + &mut self.levels, + None, + )?; + self.remember(input); + Ok(packed) + } + + /// Codes one member of a solid chain with the engine the caller asked for. + /// + /// Both engines carry state across the chain and only one of them can be + /// right for a given member, so the loser's state is thrown away: an LZ + /// member must not advance the reader's PPMd model, and a PPMd member must + /// not advance its code-length table. What both advance is the window, + /// since the reader's window is fed by whichever engine wrote the bytes. + /// + /// `candidates` is the filter lists to choose between, each measured + /// against the chain as it stands. An empty slice means code it plainly, + /// which is what a caller with no search to offer passes. + pub(crate) fn encode_member_with_engine( + &mut self, + input: &[u8], + engine: ChainEngine, + candidates: &[Vec], + progress: &mut dyn FnMut(usize) -> bool, + ) -> Result> { + if engine == ChainEngine::Ppmd { + let (packed, model) = self.encode_ppmd_member(input, progress)?; + self.ppmd = Some(model); + self.remember(input); + return Ok(packed); + } + + let lz = self.best_lz_candidate(input, candidates, progress)?; + + let ppmd = match engine { + ChainEngine::Smaller => { + Some(self.encode_ppmd_member(input, &mut |_| progress(input.len()))?) + } + _ => None, + }; + + match ppmd.filter(|(packed, _)| packed.len() < lz.packed.len()) { + Some((packed, model)) => { + self.ppmd = Some(model); + self.remember(input); + Ok(packed) + } + None => { + self.levels = lz.levels; + // The LZ layer coded the filtered bytes, so those are what a + // decoder's window holds and what the next member can match + // against. A member that took no filter coded its input. + self.remember(lz.coded.as_deref().unwrap_or(input)); + Ok(lz.packed) + } + } + } + + /// Codes the member under every candidate filter list and keeps the + /// smallest, along with the chain state that candidate would leave behind. + /// + /// This is the whole reason a chain can search for a filter at all. Which + /// filter suits a member is decided elsewhere, on the member's own bytes; + /// what cannot be decided there is whether the winner still pays once the + /// history behind it is doing some of the same work, and that is what the + /// encodes here measure. They are the real thing, against the real history, + /// at the caller's real settings. + /// + /// Only the first candidate advances progress; later candidates poll at + /// the completed byte count for cancellation. Every candidate walks the + /// whole member, and the bar counts bytes coded rather than work done, so + /// reporting each one over again would run it past the end of the member + /// and back. The plain candidate comes first, so what the bar shows is one + /// member's worth of the pass that always happens. + fn best_lz_candidate( + &self, + input: &[u8], + candidates: &[Vec], + progress: &mut dyn FnMut(usize) -> bool, + ) -> Result { + let plain_only = [Vec::new()]; + let candidates = if candidates.is_empty() { + &plain_only[..] + } else { + candidates + }; + let mut best: Option = None; + for (index, filters) in candidates.iter().enumerate() { + let mut report = + |position: usize| progress(if index == 0 { position } else { input.len() }); + let mut levels = self.levels; + let candidate = if filters.is_empty() { + LzCandidate { + packed: encode_member_with_options_and_progress( + input, + &self.history, + self.options, + &mut levels, + &mut report, + )?, + coded: None, + levels, + } + } else { + let mut split = Vec::new(); + for filter in filters { + split.extend(split_large_filter(input.len(), filter.clone())?); + } + let filtered = filtered_members_with_progress(input, &split, Some(&mut report))?; + let packed = encode_filtered_member_blocks( + &filtered.data, + &self.history, + &filtered.records, + self.options, + &mut levels, + Some(&mut report), + )?; + LzCandidate { + packed, + coded: Some(filtered.data), + levels, + } + }; + if best + .as_ref() + .is_none_or(|best| candidate.packed.len() < best.packed.len()) + { + best = Some(candidate); + } + // Check again after committing the candidate, without moving the + // byte count backwards for repeated work on the same member. + if index > 0 && !progress(input.len()) { + return Err(Error::Cancelled); + } + } + Ok(best.expect("a chain always has at least the plain candidate")) + } + + /// Codes the member against a copy of the chain's model, so a trial that + /// loses leaves the reader's model where the winning member expects it. + fn encode_ppmd_member( + &self, + input: &[u8], + progress: &mut dyn FnMut(usize) -> bool, + ) -> Result<(Vec, PpmdDecoder)> { + encode_ppmd_block_with_model( + input, + true, + &[], + self.options.max_match_distance, + self.ppmd.clone(), + Some(progress), + ) + } + + pub fn encode_member_with_filter( + &mut self, + input: &[u8], + filter: crate::FilterSpec, + ) -> Result> { + let filters = split_large_filter(input.len(), filter)?; + let filtered = filtered_members(input, &filters)?; + let mut levels = self.levels; + let packed = encode_filtered_member_blocks( + &filtered.data, + &self.history, + &filtered.records, + self.options, + &mut levels, + None, + )?; + // The LZ layer coded the filtered bytes, so that is what a decoder's + // window holds and what the next member in a solid chain can match + // against. Remembering the caller's input instead leaves every member + // after this one referring to bytes no decoder ever had. + self.levels = levels; + self.remember(&filtered.data); + Ok(packed) + } + + pub fn encode_member_with_filters( + &mut self, + input: &[u8], + filters: &[crate::FilterSpec], + ) -> Result> { + self.encode_member_with_filters_and_progress(input, filters, None) + } + + pub(crate) fn encode_member_with_filters_and_progress( + &mut self, + input: &[u8], + filters: &[crate::FilterSpec], + mut progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + if progress.as_mut().is_some_and(|report| !report(0)) { + return Err(Error::Cancelled); + } + let mut split_filters = Vec::new(); + for filter in filters { + split_filters.extend(split_large_filter(input.len(), filter.clone())?); + } + let filtered = match progress.as_mut() { + Some(report) => { + filtered_members_with_progress(input, &split_filters, Some(&mut **report))? + } + None => filtered_members(input, &split_filters)?, + }; + let mut levels = self.levels; + let packed = encode_filtered_member_blocks( + &filtered.data, + &self.history, + &filtered.records, + self.options, + &mut levels, + progress, + )?; + // The LZ layer coded the filtered bytes, so that is what a decoder's + // window holds and what the next member in a solid chain can match + // against. Remembering the caller's input instead leaves every member + // after this one referring to bytes no decoder ever had. + self.levels = levels; + self.remember(&filtered.data); + Ok(packed) + } + + fn remember(&mut self, input: &[u8]) { + self.history.extend_from_slice(input); + let keep_from = self.history.len().saturating_sub(MAX_HISTORY); + if keep_from != 0 { + self.history.drain(..keep_from); + } + } +} + +fn encode_member(input: &[u8], history: &[u8]) -> Result> { + encode_member_with_options(input, history, EncodeOptions::default()) +} + +fn encode_member_with_options( + input: &[u8], + history: &[u8], + options: EncodeOptions, +) -> Result> { + encode_member_with_options_impl(input, history, options, &mut [0; TABLE_COUNT], None) +} + +fn encode_member_with_options_and_progress( + input: &[u8], + history: &[u8], + options: EncodeOptions, + levels: &mut [u8; TABLE_COUNT], + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + encode_member_with_options_impl(input, history, options, levels, Some(progress)) +} + +fn encode_member_with_options_impl( + input: &[u8], + history: &[u8], + options: EncodeOptions, + levels: &mut [u8; TABLE_COUNT], + progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let options = options.constrained(); + if let Some(block_size) = options.block_size.filter(|&size| size != 0) { + if input.len() > block_size { + return encode_member_blocks(input, history, options, block_size, levels, progress); + } + } + encode_member_inner(input, history, &[], options, false, levels, progress) +} + +fn encode_member_blocks( + input: &[u8], + history: &[u8], + mut options: EncodeOptions, + block_size: usize, + levels: &mut [u8; TABLE_COUNT], + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + options.block_size = None; + let mut out = Vec::new(); + let mut local_history = history[history.len().saturating_sub(MAX_HISTORY)..].to_vec(); + let mut completed = 0usize; + let block_count = input.chunks(block_size).count(); + for (index, chunk) in input.chunks(block_size).enumerate() { + let mut chunk_progress = |position: usize| { + progress + .as_deref_mut() + .is_none_or(|report| report(completed.saturating_add(position))) + }; + out.extend_from_slice(&encode_member_inner( + chunk, + &local_history, + &[], + options, + index + 1 < block_count, + levels, + Some(&mut chunk_progress), + )?); + completed = completed.saturating_add(chunk.len()); + local_history.extend_from_slice(chunk); + let keep_from = local_history.len().saturating_sub(MAX_HISTORY); + if keep_from != 0 { + local_history.drain(..keep_from); + } + } + Ok(out) +} + +/// `more_blocks_follow` is what the block's terminator says. +/// +/// The end-of-block symbol is followed by a bit meaning "another table comes +/// next". A member split across blocks needs that bit set on every block but +/// the last, and clear on the last so the member ends. Getting either one +/// wrong leaves a reader parsing whatever comes after as the wrong thing. +/// +/// `previous_levels` is the code-length table the reader holds when this block +/// starts, and is left holding this block's. A reader clears it between members +/// unless the archive is solid, so a caller that is not chaining members hands +/// over a table of zeroes and gets one block's worth of state back it can throw +/// away. +fn encode_member_inner( + input: &[u8], + history: &[u8], + initial_filters: &[Vec], + options: EncodeOptions, + more_blocks_follow: bool, + previous_levels: &mut [u8; TABLE_COUNT], + progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let tokens = encode_tokens_with_progress(input, history, options, progress)?; + let mut main_frequencies = vec![0usize; MAIN_COUNT]; + let mut offset_frequencies = vec![0usize; OFFSET_COUNT]; + let mut low_offset_frequencies = vec![0usize; LOW_OFFSET_COUNT]; + let mut length_frequencies = vec![0usize; LENGTH_COUNT]; + main_frequencies[257] += initial_filters.len(); + let mut match_state = EncoderMatchState::default(); + for token in &tokens { + match *token { + EncodeToken::Literal(byte) => { + main_frequencies[byte as usize] += 1; + } + EncodeToken::Match { length, offset } => { + match match_state.encode_match(length, offset)? { + EncodedMatch::LastLengthRepeat => { + main_frequencies[258] += 1; + } + EncodedMatch::RepeatOffset { + index, length_slot, .. + } => { + main_frequencies[259 + index] += 1; + length_frequencies[length_slot] += 1; + } + EncodedMatch::Fresh { + length_slot, + offset_slot, + offset_extra, + .. + } => { + main_frequencies[271 + length_slot] += 1; + offset_frequencies[offset_slot] += 1; + if offset_slot > 9 { + low_offset_frequencies[offset_extra & 0x0f] += 1; + } + } + } + match_state.remember(length, offset); + } + } + } + main_frequencies[256] += 1; + + let mut table_lengths = [0u8; TABLE_COUNT]; + if low_offset_frequencies + .iter() + .all(|&frequency| frequency == 0) + { + low_offset_frequencies[0] = 1; + } + let main_lengths = huffman::lengths_for_frequencies(&main_frequencies, 15); + let offset_lengths = huffman::lengths_for_frequencies(&offset_frequencies, 15); + let low_offset_lengths = huffman::lengths_for_frequencies(&low_offset_frequencies, 15); + let length_lengths = huffman::lengths_for_frequencies(&length_frequencies, 15); + table_lengths[..MAIN_COUNT].copy_from_slice(&main_lengths); + table_lengths[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT].copy_from_slice(&offset_lengths); + table_lengths[MAIN_COUNT + OFFSET_COUNT..MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT] + .copy_from_slice(&low_offset_lengths); + table_lengths[MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT..].copy_from_slice(&length_lengths); + + // Two ways to say the same table: outright, or as a delta against the one + // the reader already holds. Neither wins everywhere, so code both and take + // the shorter. Coding a table costs nothing next to parsing the block it + // describes, and picking by size means the keep-tables bit can only help. + let outright = encode_table_level_tokens(&table_lengths); + let against_previous = encode_level_tokens_against(&table_lengths, previous_levels); + let keep_previous_tables = + level_tokens_bit_cost(&against_previous) < level_tokens_bit_cost(&outright); + let level_tokens = match keep_previous_tables { + true => against_previous, + false => outright, + }; + *previous_levels = table_lengths; + + let level_lengths = level_code_lengths(&level_tokens); + let level_codes = canonical_codes(&level_lengths); + let main_codes = canonical_codes(&table_lengths[..MAIN_COUNT]); + + let mut bits = BitWriter::default(); + bits.write_bit(false); // LZ block. + bits.write_bit(keep_previous_tables); + for &len in &level_lengths { + bits.write_bits(len as u32, 4); + } + // Both passes replay the same tokens from the same initial match state. + // Every emitted symbol was counted, and positive frequencies receive codes. + for token in level_tokens { + let code = level_codes[token.symbol].expect("counted level Huffman code"); + bits.write_bits(code.code as u32, code.len); + if token.extra_bits != 0 { + bits.write_bits(token.extra_value as u32, token.extra_bits); + } + } + let offset_codes = canonical_codes(&table_lengths[MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT]); + let low_offset_codes = canonical_codes( + &table_lengths[MAIN_COUNT + OFFSET_COUNT..MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT], + ); + let length_codes = + canonical_codes(&table_lengths[MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT..]); + for filter in initial_filters { + let code = main_codes[257].expect("counted VM filter Huffman code"); + bits.write_bits(code.code as u32, code.len); + for &byte in filter { + bits.write_bits(u32::from(byte), 8); + } + } + let mut match_state = EncoderMatchState::default(); + for token in tokens { + match token { + EncodeToken::Literal(byte) => { + let code = main_codes[byte as usize].expect("counted literal Huffman code"); + bits.write_bits(code.code as u32, code.len); + } + EncodeToken::Match { length, offset } => { + match match_state.encode_match(length, offset)? { + EncodedMatch::LastLengthRepeat => { + let code = + main_codes[258].expect("counted last-length repeat Huffman code"); + bits.write_bits(code.code as u32, code.len); + } + EncodedMatch::RepeatOffset { + index, + length_slot, + length_extra, + } => { + let code = + main_codes[259 + index].expect("counted repeat-offset Huffman code"); + bits.write_bits(code.code as u32, code.len); + let length_code = + length_codes[length_slot].expect("counted repeat length Huffman code"); + bits.write_bits(length_code.code as u32, length_code.len); + if LENGTH_BITS[length_slot] != 0 { + bits.write_bits(length_extra as u32, LENGTH_BITS[length_slot]); + } + } + EncodedMatch::Fresh { + length_slot, + length_extra, + offset_slot, + offset_extra, + } => { + let code = + main_codes[271 + length_slot].expect("counted match Huffman code"); + bits.write_bits(code.code as u32, code.len); + if LENGTH_BITS[length_slot] != 0 { + bits.write_bits(length_extra as u32, LENGTH_BITS[length_slot]); + } + let offset = + offset_codes[offset_slot].expect("counted offset Huffman code"); + bits.write_bits(offset.code as u32, offset.len); + if offset_slot > 9 { + let offset_bits = OFFSET_BITS[offset_slot]; + if offset_bits > 4 { + bits.write_bits((offset_extra >> 4) as u32, offset_bits - 4); + } + let low_offset = low_offset_codes[offset_extra & 0x0f] + .expect("counted low-offset Huffman code"); + bits.write_bits(low_offset.code as u32, low_offset.len); + } else if OFFSET_BITS[offset_slot] != 0 { + bits.write_bits(offset_extra as u32, OFFSET_BITS[offset_slot]); + } + } + } + match_state.remember(length, offset); + } + } + } + let end = main_codes[256].expect("counted end-of-block Huffman code"); + bits.write_bits(end.code as u32, end.len); + // The end-of-block symbol on its own does not end the member: the next bit + // says whether another table follows. A block in the middle of a member + // sets it, because one does. The last block clears it and then writes a + // second bit, which is what the reader carries into a solid follower: a one + // tells it to read its own tables. + if more_blocks_follow { + bits.write_bit(true); + } else { + bits.write_bit(false); + bits.write_bit(true); + } + Ok(bits.finish()) +} + +/// Encodes the filter records for one LZ block. +/// +/// `base` is where the block starts in the member, because a record's block +/// start is read relative to the decoder's current output position, and that +/// position is the head of the block the record is declared in. Writing the +/// member-absolute offset instead makes the decoder mask it against the window +/// and apply the filter in the wrong place. +/// +/// `programs` carries across blocks so a program declared once is referenced by +/// index afterwards, which is what the decoder expects. +fn encoded_filter_records_at( + filters: &[&OwnedVmFilterRecord], + base: usize, + window: usize, + programs: &mut Vec<&'static [u8]>, +) -> Result>> { + let mut records = Vec::with_capacity(filters.len()); + for filter in filters { + let existing = (filter.code != RAR3_AUDIO_FILTER_BYTECODE) + .then(|| programs.iter().position(|&code| code == filter.code)) + .flatten(); + let (program_selector, include_code) = match existing { + Some(index) => ( + u32::try_from(index + 1) + .map_err(|_| Error::InvalidData("RAR 2.9 VM program index overflows"))?, + false, + ), + None => { + let selector = if programs.is_empty() { + 0 + } else { + u32::try_from(programs.len() + 1) + .map_err(|_| Error::InvalidData("RAR 2.9 VM program index overflows"))? + }; + programs.push(filter.code); + (selector, true) + } + }; + let block_start = filter + .block_start + .checked_sub(base) + .ok_or(Error::InvalidData( + "RAR 2.9 VM filter starts before its block", + ))?; + // The decoder masks this against its window, so a record reaching past + // one lands somewhere else entirely and the member decodes to the wrong + // bytes. Our own decoder reads it back the way it was written and so + // agrees, which is why this is checked here rather than left to a round + // trip to notice. + if block_start >= window { + return Err(Error::InvalidData( + "RAR 2.9 VM filter starts further past its block than the window can express", + )); + } + records.push(encode_vm_filter_record_inner( + VmFilterRecord { + block_start, + block_size: filter.block_size, + init_regs: &filter.init_regs, + code: filter.code, + global_data: &filter.global_data, + }, + program_selector, + include_code, + )?); + } + Ok(records) +} + +/// Codes filtered bytes a block at a time, so that no filter is declared +/// further ahead of its block than the decoder's window can express, and so a +/// large member does not need a whole LZ pass in memory at once. +fn encode_filtered_member_blocks( + data: &[u8], + history: &[u8], + filters: &[OwnedVmFilterRecord], + options: EncodeOptions, + levels: &mut [u8; TABLE_COUNT], + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + // A record's offset is relative to the head of its own block, so a block no + // larger than the window keeps every offset inside it. + let block_size = options + .block_size + .filter(|&size| size != 0) + .unwrap_or(data.len().max(1)) + .min(options.max_match_distance.max(1)) + .max(1); + let window = options.max_match_distance.max(1); + let mut inner = options; + inner.block_size = None; + let mut programs: Vec<&'static [u8]> = Vec::new(); + let mut out = Vec::new(); + let mut local_history = history[history.len().saturating_sub(MAX_HISTORY)..].to_vec(); + let mut base = 0usize; + while base < data.len().max(1) { + let end = (base + block_size).min(data.len()); + let chunk = &data[base..end]; + let in_block: Vec<&OwnedVmFilterRecord> = filters + .iter() + .filter(|record| record.block_start >= base && record.block_start < end.max(base + 1)) + .collect(); + let records = encoded_filter_records_at(&in_block, base, window, &mut programs)?; + let mut chunk_progress = |position: usize| { + progress + .as_deref_mut() + .is_none_or(|report| report(base.saturating_add(position))) + }; + out.extend_from_slice(&encode_member_inner( + chunk, + &local_history, + &records, + inner, + end < data.len(), + levels, + Some(&mut chunk_progress), + )?); + local_history.extend_from_slice(chunk); + let keep_from = local_history.len().saturating_sub(MAX_HISTORY); + if keep_from != 0 { + local_history.drain(..keep_from); + } + base = end; + if chunk.is_empty() { + break; + } + } + Ok(out) +} + +#[derive(Debug, Clone, Copy)] +struct VmFilterRecord<'a> { + block_start: usize, + block_size: usize, + init_regs: &'a [(usize, u32)], + code: &'a [u8], + global_data: &'a [u8], +} + +fn encode_vm_filter_record_inner( + record: VmFilterRecord<'_>, + program_selector: u32, + include_code: bool, +) -> Result> { + if record.block_size == 0 { + return Err(Error::InvalidData("RAR 2.9 VM filter block is empty")); + } + if include_code && record.code.is_empty() { + return Err(Error::InvalidData("RAR 2.9 VM filter bytecode is empty")); + } + + let mut body = BitWriter::default(); + body.write_encoded_u32(program_selector); + body.write_encoded_u32( + u32::try_from(record.block_start) + .map_err(|_| Error::InvalidData("RAR 2.9 VM block start overflows"))?, + ); + body.write_encoded_u32( + u32::try_from(record.block_size) + .map_err(|_| Error::InvalidData("RAR 2.9 VM block size overflows"))?, + ); + if !record.init_regs.is_empty() { + let mut mask = 0u32; + for &(index, _) in record.init_regs { + if index >= 7 { + return Err(Error::InvalidData( + "RAR 2.9 VM init register index is invalid", + )); + } + mask |= 1 << index; + } + body.write_bits(mask, 7); + for index in 0..7 { + if let Some((_, value)) = record.init_regs.iter().find(|(reg, _)| *reg == index) { + body.write_encoded_u32(*value); + } + } + } + if include_code { + body.write_encoded_u32( + u32::try_from(record.code.len()) + .map_err(|_| Error::InvalidData("RAR 2.9 VM code size overflows"))?, + ); + for &byte in record.code { + body.write_bits(u32::from(byte), 8); + } + } + if !record.global_data.is_empty() { + body.write_encoded_u32( + u32::try_from(record.global_data.len()) + .map_err(|_| Error::InvalidData("RAR 2.9 VM global data size overflows"))?, + ); + for &byte in record.global_data { + body.write_bits(u32::from(byte), 8); + } + } + let body = body.finish(); + + let mut out = Vec::new(); + let mut first = 0x80 | 0x20; + if !record.init_regs.is_empty() { + first |= 0x10; + } + if !record.global_data.is_empty() { + first |= 0x08; + } + match body.len() { + 1..=6 => first |= (body.len() as u8) - 1, + 7..=262 => { + first |= 6; + out.push((body.len() - 7) as u8); + } + 263..=65535 => { + first |= 7; + out.extend_from_slice(&(body.len() as u16).to_be_bytes()); + } + _ => return Err(Error::InvalidData("RAR 2.9 VM filter record is too large")), + } + out.insert(0, first); + out.extend_from_slice(&body); + Ok(out) +} + +fn rgb_encode(data: &[u8], width: usize, pos_r: usize) -> Result> { + if data.len() < 3 || width == 0 || !width.is_multiple_of(3) || width > data.len() || pos_r > 2 { + return Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid", + )); + } + let mut work = data.to_vec(); + for i in (pos_r..work.len().saturating_sub(2)).step_by(3) { + let green = work[i + 1]; + work[i] = work[i].wrapping_sub(green); + work[i + 2] = work[i + 2].wrapping_sub(green); + } + + let mut out = Vec::with_capacity(data.len()); + for channel in 0..3 { + let mut prev = 0u8; + let mut i = channel; + while i < work.len() { + let predicted = if i >= width + 3 { + rgb_predict(prev, work[i - width], work[i - width - 3]) + } else { + prev + }; + let byte = work[i]; + out.push(predicted.wrapping_sub(byte)); + prev = byte; + i += 3; + } + } + Ok(out) +} + +fn audio_encode(data: &[u8], channels: usize) -> Result> { + if channels == 0 || channels > MAX_AUDIO_CHANNELS { + return Err(Error::InvalidData( + "RAR 2.9 AUDIO filter channel count is invalid", + )); + } + let mut out = Vec::with_capacity(data.len()); + for channel in 0..channels { + let mut prev_byte = 0u32; + let mut prev_delta = 0i32; + let mut d1 = 0i32; + let mut d2 = 0i32; + let mut k1 = 0i32; + let mut k2 = 0i32; + let mut k3 = 0i32; + let mut dif = [0u32; 7]; + let mut byte_count = 0usize; + let mut i = channel; + while i < data.len() { + let d3 = d2; + d2 = prev_delta - d1; + d1 = prev_delta; + let predicted = ((8 * prev_byte as i32 + k1 * d1 + k2 * d2 + k3 * d3) >> 3) & 0xff; + let decoded = data[i]; + let encoded = (predicted as u8).wrapping_sub(decoded); + out.push(encoded); + prev_delta = decoded.wrapping_sub(prev_byte as u8) as i8 as i32; + prev_byte = decoded as u32; + let d = (encoded as i8 as i32) << 3; + dif[0] += d.unsigned_abs(); + dif[1] += (d - d1).unsigned_abs(); + dif[2] += (d + d1).unsigned_abs(); + dif[3] += (d - d2).unsigned_abs(); + dif[4] += (d + d2).unsigned_abs(); + dif[5] += (d - d3).unsigned_abs(); + dif[6] += (d + d3).unsigned_abs(); + if byte_count & 0x1f == 0 { + let mut min = dif[0]; + let mut min_index = 0usize; + dif[0] = 0; + for (index, value) in dif.iter_mut().enumerate().skip(1) { + if *value < min { + min = *value; + min_index = index; + } + *value = 0; + } + match min_index { + 1 if k1 >= -16 => k1 -= 1, + 2 if k1 < 16 => k1 += 1, + 3 if k2 >= -16 => k2 -= 1, + 4 if k2 < 16 => k2 += 1, + 5 if k3 >= -16 => k3 -= 1, + 6 if k3 < 16 => k3 += 1, + _ => {} + } + } + byte_count += 1; + i += channels; + } + } + Ok(out) +} + +fn itanium_encode(data: &mut [u8], file_offset: u32) { + if data.len() <= 21 { + return; + } + let base_offset = file_offset >> 4; + let block_count = (data.len() - 21).div_ceil(16); + for block in 0..block_count { + let pos = block * 16; + let file_offset = base_offset.wrapping_add(block as u32); + let mut mask = (0x334b_0000u32 >> (data[pos] & 0x1e)) & 3; + if mask != 0 { + mask += 1; + while mask <= 4 { + let p = pos + (mask as usize * 5 - 8); + if ((data[p + 3] >> mask) & 15) == 5 { + let raw = u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]); + let mut value = raw >> mask; + value = value.wrapping_add(file_offset) & 0x000f_ffff; + let raw = (raw & !(0x000f_ffff << mask)) | (value << mask); + data[p..p + 4].copy_from_slice(&raw.to_le_bytes()); + } + mask += 1; + } + } + } +} + +#[derive(Debug, Clone, Copy)] +enum EncodeToken { + Literal(u8), + Match { length: usize, offset: usize }, +} + +#[derive(Debug, Clone, Copy, Default)] +struct EncoderMatchState { + old_offsets: [usize; 4], + last_offset: usize, + last_length: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EncodedMatch { + LastLengthRepeat, + RepeatOffset { + index: usize, + length_slot: usize, + length_extra: usize, + }, + Fresh { + length_slot: usize, + length_extra: usize, + offset_slot: usize, + offset_extra: usize, + }, +} + +impl EncoderMatchState { + fn encode_match(&self, length: usize, offset: usize) -> Result { + if self.last_length != 0 && offset == self.last_offset && length == self.last_length { + return Ok(EncodedMatch::LastLengthRepeat); + } + if let Some(index) = self + .old_offsets + .iter() + .position(|&old_offset| old_offset == offset && old_offset != 0) + { + // The repeat-distance table stops at 257 bytes, one byte before + // the fresh-match table. A 258-byte match at a remembered + // distance is still representable, just not with the shorter + // repeat token. + if let Ok((length_slot, length_extra)) = length_slot_for_repeat_match(length) { + return Ok(EncodedMatch::RepeatOffset { + index, + length_slot, + length_extra, + }); + } + } + let encoded_length = + length + .checked_sub(match_length_adjustment(offset)) + .ok_or(Error::InvalidData( + "RAR 2.9 adjusted match length underflows", + ))?; + let (length_slot, length_extra) = length_slot_for_match(encoded_length)?; + let (offset_slot, offset_extra) = offset_slot_for_match(offset)?; + Ok(EncodedMatch::Fresh { + length_slot, + length_extra, + offset_slot, + offset_extra, + }) + } + + fn remember(&mut self, length: usize, offset: usize) { + if self.last_length != 0 && offset == self.last_offset && length == self.last_length { + return; + } + if let Some(index) = self + .old_offsets + .iter() + .position(|&old_offset| old_offset == offset) + .filter(|_| length_slot_for_repeat_match(length).is_ok()) + { + self.old_offsets[..=index].rotate_right(1); + } else { + // This is a fresh token even when its distance already occurs in + // the repeat ring. The decoder shifts it in and keeps the older + // occurrence, so the encoder must retain the duplicate too. + self.old_offsets.rotate_right(1); + self.old_offsets[0] = offset; + } + self.last_offset = offset; + self.last_length = length; + } +} + +fn encode_tokens_with_progress( + input: &[u8], + history: &[u8], + options: EncodeOptions, + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let mut tokens = Vec::new(); + let history = &history[history.len().saturating_sub(options.max_match_distance)..]; + let mut combined = Vec::with_capacity(history.len() + input.len()); + combined.extend_from_slice(history); + combined.extend_from_slice(input); + let mut finder = Rar29MatchFinder::new(combined.len()); + for history_pos in 0..history.len() { + finder.insert(&combined, history_pos); + } + + let mut pos = history.len(); + let end = combined.len(); + let mut state = EncoderMatchState::default(); + let mut next_report = 0usize; + let mut pending_match: Option = None; + while pos < end { + let candidate = pending_match + .take() + .or_else(|| best_match(&combined, pos, end, &finder, options, &state)); + if let Some(candidate) = candidate { + let (emit_literal, cached_next) = + lazy_match_decision(&combined, pos, &finder, options, &state, candidate); + if emit_literal { + tokens.push(EncodeToken::Literal(combined[pos])); + finder.insert(&combined, pos); + pos += 1; + pending_match = cached_next; + continue; + } + let MatchCandidate { length, offset, .. } = candidate; + tokens.push(EncodeToken::Match { length, offset }); + state.remember(length, offset); + for history_pos in pos..pos + length { + finder.insert(&combined, history_pos); + } + pos += length; + } else { + tokens.push(EncodeToken::Literal(combined[pos])); + finder.insert(&combined, pos); + pos += 1; + } + let consumed = pos.saturating_sub(history.len()); + if consumed >= next_report { + if progress + .as_deref_mut() + .is_some_and(|report| !report(consumed)) + { + return Err(Error::Cancelled); + } + next_report = consumed.saturating_add(1024 * 1024); + } + } + if progress.is_some_and(|report| !report(input.len())) { + return Err(Error::Cancelled); + } + Ok(tokens) +} + +/// Decides whether a literal should be emitted instead of `current` because a +/// better match starts within the lazy lookahead window. Also returns the +/// match found one byte ahead (when computed) so the caller can reuse it for +/// the next position instead of searching again. +fn lazy_match_decision( + input: &[u8], + pos: usize, + finder: &Rar29MatchFinder, + options: EncodeOptions, + state: &EncoderMatchState, + current: MatchCandidate, +) -> (bool, Option) { + let end = input.len(); + if !options.lazy_matching { + return (false, None); + } + let lookahead = options.lazy_lookahead.max(1); + let mut cached_next = None; + for offset in 1..=lookahead { + if pos + offset >= end { + break; + } + let next = best_match(input, pos + offset, end, finder, options, state); + if offset == 1 { + cached_next = next; + } + let skipped_literal_score = offset as isize * 8; + if next.is_some_and(|next| next.score > current.score + skipped_literal_score) { + return (true, cached_next); + } + } + (false, None) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct MatchCandidate { + length: usize, + offset: usize, + score: isize, +} + +/// The running price of each token kind, measured off the range coder as the +/// member is encoded. An escape token is far from free: a match is six model +/// symbols (escape, 4, three offset bytes, a length) and a repeat is three, +/// coded through contexts where the escape byte is rare, and the copied bytes +/// never enter the model. On text the model predicts a repeated span for a +/// fraction of a bit per byte, so a minimum-length match can cost several +/// times the literals it replaces. Emitting every legal match is what left +/// PPMd members 30-45% behind rar 3.00 on log-shaped text. +/// +/// The literal average answers "what does a byte cost as a literal right +/// now", which is the opportunity cost of a match, so it moves slowly. The +/// token averages track a near-fixed overhead, so they move fast. All three +/// are only ever compared against each other, so the model self-corrects: a +/// token kind that keeps losing keeps its measured price and stays rejected. +struct PpmdTokenCosts { + literal_bits: f64, + match_bits: f64, + repeat_bits: f64, +} + +impl PpmdTokenCosts { + fn new() -> Self { + Self { + literal_bits: PPMD_LITERAL_BITS_SEED, + match_bits: PPMD_MATCH_BITS_SEED, + repeat_bits: PPMD_REPEAT_BITS_SEED, + } + } + + fn match_pays(&self, length: usize) -> bool { + length as f64 * self.literal_bits > self.match_bits + PPMD_CONTEXT_BREAK_BITS + } + + fn repeat_pays(&self, length: usize) -> bool { + length as f64 * self.literal_bits > self.repeat_bits + PPMD_CONTEXT_BREAK_BITS + } + + fn record_literal(&mut self, bits: f64) { + ema(&mut self.literal_bits, bits, PPMD_LITERAL_EMA_WEIGHT); + } + + fn record_match(&mut self, bits: f64) { + ema(&mut self.match_bits, bits, PPMD_TOKEN_EMA_WEIGHT); + } + + fn record_repeat(&mut self, bits: f64) { + ema(&mut self.repeat_bits, bits, PPMD_TOKEN_EMA_WEIGHT); + } +} + +fn ema(slot: &mut f64, sample: f64, weight: f64) { + *slot += weight * (sample - *slot); +} + +/// Feed the member through PPMd, escaping to an LZ token only where the token +/// prices in cheaper than letting the model code the same bytes as literals. +/// Tokenising and encoding are one loop so each decision can read the cost of +/// the last one straight off the range coder; a separate tokenising pass has +/// no way to know what the model would have charged. +/// +/// `on_token` sees every emitted token, in order. Production passes a no-op; +/// the tests collect them. +#[cfg(all(test, feature = "write"))] +fn encode_ppmd_hybrid( + input: &[u8], + max_match_distance: usize, + encoder: &mut PpmdEncoder, + on_token: impl FnMut(PpmdEncodeToken), +) -> Result<()> { + encode_ppmd_hybrid_with_progress(input, max_match_distance, encoder, on_token, None) +} + +fn encode_ppmd_hybrid_with_progress( + input: &[u8], + max_match_distance: usize, + encoder: &mut PpmdEncoder, + mut on_token: impl FnMut(PpmdEncodeToken), + mut progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result<()> { + let mut costs = PpmdTokenCosts::new(); + let mut finder = Rar29MatchFinder::new(input.len()); + let mut pos = 0usize; + let mut search_from = 0usize; + let mut next_check = 0usize; + while pos < input.len() { + if pos >= next_check { + if progress.as_mut().is_some_and(|report| !report(pos)) { + return Err(Error::Cancelled); + } + next_check = pos.saturating_add(4096); + } + if let Some(length) = ppmd_offset_one_repeat(input, pos) { + if costs.repeat_pays(length) { + let before = encoder.spent_bits(); + encoder.encode_repeat_offset_one(length)?; + costs.record_repeat(encoder.spent_bits() - before); + on_token(PpmdEncodeToken::RepeatOffsetOne { length }); + for history_pos in pos..pos + length { + finder.insert(input, history_pos); + } + pos += length; + continue; + } + } + + if pos >= search_from { + if let Some((length, offset)) = best_ppmd_match(input, pos, &finder, max_match_distance) + { + if costs.match_pays(length) { + let before = encoder.spent_bits(); + encoder.encode_match(offset, length)?; + costs.record_match(encoder.spent_bits() - before); + on_token(PpmdEncodeToken::Match { offset, length }); + for history_pos in pos..pos + length { + finder.insert(input, history_pos); + } + pos += length; + continue; + } + search_from = pos + PPMD_REJECT_SEARCH_COOLDOWN; + } + } + + let before = encoder.spent_bits(); + encoder.encode_literal(input[pos])?; + costs.record_literal(encoder.spent_bits() - before); + on_token(PpmdEncodeToken::Literal(input[pos])); + finder.insert(input, pos); + pos += 1; + } + Ok(()) +} + +fn ppmd_offset_one_repeat(input: &[u8], pos: usize) -> Option { + if pos == 0 || input[pos] != input[pos - 1] { + return None; + } + let mut length = 0usize; + while pos + length < input.len() + && input[pos + length] == input[pos - 1] + && length < MAX_PPMD_REPEAT_LENGTH + { + length += 1; + } + (length >= 4).then_some(length) +} + +fn best_ppmd_match( + input: &[u8], + pos: usize, + finder: &Rar29MatchFinder, + max_match_distance: usize, +) -> Option<(usize, usize)> { + let max_offset = pos.min(0x1000001).min(MAX_HISTORY).min(max_match_distance); + let max_length = (input.len() - pos).min(MAX_PPMD_MATCH_LENGTH); + if max_offset < 2 || max_length < MIN_PPMD_MATCH_LENGTH { + return None; + } + let mut best = None; + let mut checked = 0usize; + let mut candidate = finder.first(input, pos); + while candidate != match_finder::NO_POSITION { + // MatchFinder chains contain only previously inserted positions and + // each link moves strictly backwards. + let offset = pos - candidate; + if offset > max_offset { + break; + } + if offset < 2 { + candidate = finder.previous(candidate); + continue; + } + checked += 1; + let length = match_length(input, pos, offset, max_length); + if length >= MIN_PPMD_MATCH_LENGTH + && best.is_none_or(|(best_length, best_offset)| { + length > best_length || (length == best_length && offset < best_offset) + }) + { + best = Some((length, offset)); + if length == max_length { + break; + } + } + if checked >= MAX_MATCH_CANDIDATES { + break; + } + candidate = finder.previous(candidate); + } + best +} + +fn best_match( + input: &[u8], + pos: usize, + end: usize, + finder: &Rar29MatchFinder, + options: EncodeOptions, + state: &EncoderMatchState, +) -> Option { + let max_offset = pos.min(options.max_match_distance).min(MAX_HISTORY); + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if options.max_match_candidates == 0 || max_offset == 0 || max_length < 4 { + return None; + } + let mut best = None; + let mut checked = 0usize; + for offset in state.old_offsets { + if offset == 0 || offset > max_offset { + continue; + } + let length = match_length(input, pos, offset, max_length); + consider_match_candidate(&mut best, state, length, offset); + } + if let Some(best) = best { + if best.length == max_length { + return Some(best); + } + } + let mut candidate = finder.first(input, pos); + while candidate != match_finder::NO_POSITION { + // MatchFinder chains contain only previously inserted positions and + // each link moves strictly backwards. + let offset = pos - candidate; + if offset > max_offset { + break; + } + checked += 1; + // A candidate can only improve on the current best when it matches at + // least one byte past the best length, so probe that byte first. + let best_length = best.map_or(0, |best: MatchCandidate| best.length); + if best_length == 0 || input[candidate + best_length] == input[pos + best_length] { + let length = match_length(input, pos, offset, max_length); + consider_match_candidate(&mut best, state, length, offset); + } + if best.is_some_and(|candidate| candidate.length == max_length) { + break; + } + if checked >= options.max_match_candidates { + break; + } + candidate = finder.previous(candidate); + } + best +} + +fn match_length(input: &[u8], pos: usize, offset: usize, max_length: usize) -> usize { + super::super::fast::match_length(input, pos, offset, max_length) +} + +fn consider_match_candidate( + best: &mut Option, + state: &EncoderMatchState, + length: usize, + offset: usize, +) { + if length < 4 { + return; + } + let Ok(cost) = estimated_match_cost(state, length, offset) else { + return; + }; + let score = (length as isize * 8) - cost as isize; + let candidate = MatchCandidate { + length, + offset, + score, + }; + if best.is_none_or(|best| { + candidate.score > best.score + || (candidate.score == best.score + && (candidate.length > best.length + || (candidate.length == best.length && candidate.offset < best.offset))) + }) { + *best = Some(candidate); + } +} + +fn estimated_match_cost(state: &EncoderMatchState, length: usize, offset: usize) -> Result { + match state.encode_match(length, offset)? { + EncodedMatch::LastLengthRepeat => Ok(2), + EncodedMatch::RepeatOffset { length_slot, .. } => { + Ok(5 + usize::from(LENGTH_BITS[length_slot])) + } + EncodedMatch::Fresh { + length_slot, + offset_slot, + .. + } => { + let low_offset_cost = usize::from(offset_slot > 9) * 4; + Ok(8 + usize::from(LENGTH_BITS[length_slot]) + + usize::from(OFFSET_BITS[offset_slot]) + + low_offset_cost) + } + } +} + +fn match_length_adjustment(offset: usize) -> usize { + usize::from(offset >= 0x2000) + usize::from(offset >= 0x40000) +} + +fn length_slot_for_match(length: usize) -> Result<(usize, usize)> { + if length < 3 { + return Err(Error::InvalidData("RAR 2.9 match length is too short")); + } + let adjusted = length - 3; + for (slot, &base) in LENGTH_BASES.iter().enumerate() { + let extra_bits = LENGTH_BITS[slot]; + let max = base + + if extra_bits == 0 { + 0 + } else { + (1usize << extra_bits) - 1 + }; + if adjusted <= max { + return Ok((slot, adjusted - base)); + } + } + Err(Error::InvalidData("RAR 2.9 match length is too long")) +} + +fn length_slot_for_repeat_match(length: usize) -> Result<(usize, usize)> { + if length < 2 { + return Err(Error::InvalidData( + "RAR 2.9 repeat match length is too short", + )); + } + let adjusted = length - 2; + for (slot, &base) in LENGTH_BASES.iter().enumerate() { + let extra_bits = LENGTH_BITS[slot]; + let max = base + + if extra_bits == 0 { + 0 + } else { + (1usize << extra_bits) - 1 + }; + if adjusted <= max { + return Ok((slot, adjusted - base)); + } + } + Err(Error::InvalidData( + "RAR 2.9 repeat match length is too long", + )) +} + +fn offset_slot_for_match(offset: usize) -> Result<(usize, usize)> { + if offset == 0 { + return Err(Error::InvalidData("RAR 2.9 match offset is zero")); + } + let adjusted = offset - 1; + for (slot, &base) in OFFSET_BASES.iter().enumerate() { + let extra_bits = OFFSET_BITS[slot]; + let max = base + + if extra_bits == 0 { + 0 + } else { + (1usize << extra_bits) - 1 + }; + if adjusted <= max { + return Ok((slot, adjusted - base)); + } + } + Err(Error::InvalidData("RAR 2.9 match offset is too large")) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct LevelToken { + symbol: usize, + extra_bits: u8, + extra_value: u8, +} + +impl LevelToken { + const fn plain(symbol: usize) -> Self { + Self { + symbol, + extra_bits: 0, + extra_value: 0, + } + } + + const fn repeat_previous_short(count: usize) -> Self { + Self { + symbol: 16, + extra_bits: 3, + extra_value: (count - 3) as u8, + } + } + + const fn repeat_previous_long(count: usize) -> Self { + Self { + symbol: 17, + extra_bits: 7, + extra_value: (count - 11) as u8, + } + } + + const fn zero_run_short(count: usize) -> Self { + Self { + symbol: 18, + extra_bits: 3, + extra_value: (count - 3) as u8, + } + } + + const fn zero_run_long(count: usize) -> Self { + Self { + symbol: 19, + extra_bits: 7, + extra_value: (count - 11) as u8, + } + } +} + +fn encode_table_level_tokens(lengths: &[u8; TABLE_COUNT]) -> Vec { + encode_level_tokens_against(lengths, &[0; TABLE_COUNT]) +} + +/// Codes one code-length table as level tokens, against the table the reader +/// already holds. +/// +/// Symbols 0 to 15 are read as a delta: the reader adds one to what it has at +/// that position, modulo 16. So a table close to the previous one spends the +/// cheap end of the alphabet, which is what the block header's keep-tables bit +/// buys. Pass a table of zeroes to code the lengths outright, which is the same +/// arithmetic with nothing to add to. +/// +/// The run symbols do not take part. 16 and 17 repeat the length just decoded +/// and 18 and 19 write zeroes, both regardless of `base`, so runs are found in +/// the lengths themselves either way. +fn encode_level_tokens_against(lengths: &[u8], base: &[u8]) -> Vec { + let delta = |pos: usize, value: u8| (value.wrapping_sub(base[pos]) & 0x0f) as usize; + let mut tokens = Vec::new(); + let mut pos = 0usize; + let mut previous = None; + while pos < lengths.len() { + let value = lengths[pos]; + let mut run = 1usize; + while pos + run < lengths.len() && lengths[pos + run] == value { + run += 1; + } + + if value == 0 { + emit_zero_level_run(&mut tokens, pos, run, &delta); + previous = Some(0); + pos += run; + continue; + } + + if previous == Some(value) && run >= 3 { + emit_repeat_level_run(&mut tokens, run); + pos += run; + continue; + } + + tokens.push(LevelToken::plain(delta(pos, value))); + previous = Some(value); + pos += 1; + } + tokens +} + +/// What the level tokens cost, so two codings of a table can be compared. +/// +/// The 20 four-bit code lengths at the head of the table are the same either +/// way and are left out. +fn level_tokens_bit_cost(tokens: &[LevelToken]) -> usize { + let lengths = level_code_lengths(tokens); + tokens + .iter() + .map(|token| usize::from(lengths[token.symbol]) + usize::from(token.extra_bits)) + .sum() +} + +fn emit_repeat_level_run(tokens: &mut Vec, mut run: usize) { + while run >= 11 { + let mut chunk = run.min(138); + if matches!(run - chunk, 1 | 2) { + chunk -= 3; + } + tokens.push(LevelToken::repeat_previous_long(chunk)); + run -= chunk; + } + if run >= 3 { + tokens.push(LevelToken::repeat_previous_short(run)); + } +} + +fn emit_zero_level_run( + tokens: &mut Vec, + start: usize, + mut run: usize, + delta: &dyn Fn(usize, u8) -> usize, +) { + let mut pos = start; + while run != 0 { + if run >= 11 { + let mut chunk = run.min(138); + if matches!(run - chunk, 1 | 2) { + chunk -= 3; + } + tokens.push(LevelToken::zero_run_long(chunk)); + run -= chunk; + pos += chunk; + } else if run >= 3 { + let chunk = run.min(10); + tokens.push(LevelToken::zero_run_short(chunk)); + run -= chunk; + pos += chunk; + } else { + // A run too short for its own symbol is written out position by + // position, and each of those is a delta like any other. + tokens.extend((pos..pos + run).map(|pos| LevelToken::plain(delta(pos, 0)))); + break; + } + } +} + +/// Codes the level alphabet by how often each symbol is used, not by how many +/// of them appear. +/// +/// A flat code charges the same for every symbol in play, so a table whose +/// tokens are mostly one symbol pays as if they were spread evenly. That is +/// what the keep-tables bit produces, and against a flat code it saved almost +/// nothing. Weighting by frequency is what makes it pay. +/// +/// The 20 lengths are written four bits each, hence the cap of 15. +fn level_code_lengths(tokens: &[LevelToken]) -> [u8; LEVEL_COUNT] { + let mut frequencies = [0usize; LEVEL_COUNT]; + for token in tokens { + frequencies[token.symbol] += 1; + } + // One symbol in play gives a code with one branch and an empty slot beside + // it, which a strict reader rejects. Only the flat assignment pads it. + if frequencies.iter().filter(|&&count| count != 0).count() <= 1 { + let mut lengths = [0u8; LEVEL_COUNT]; + for (symbol, &count) in frequencies.iter().enumerate() { + lengths[symbol] = u8::from(count != 0); + } + huffman::assign_flat_complete_code(&mut lengths); + return lengths; + } + huffman::lengths_for_frequency_array(&frequencies, 15) +} + +#[derive(Debug, Clone, Copy)] +struct HuffmanCode { + code: u16, + len: u8, +} + +// Only encoder-generated tables reach this helper; archive tables use Huffman. +fn canonical_codes(lengths: &[u8]) -> Vec> { + debug_assert!(lengths.iter().all(|&len| len <= 15)); + let mut count = [0u16; 16]; + for &len in lengths { + if len != 0 { + count[len as usize] += 1; + } + } + debug_assert!(validate_huffman_counts(&count).is_ok()); + + let mut next_code = [0u16; 16]; + let mut code = 0u16; + for len in 1..=15 { + code = (code + count[len - 1]) << 1; + next_code[len] = code; + } + + let mut codes = vec![None; lengths.len()]; + for (symbol, &len) in lengths.iter().enumerate() { + if len == 0 { + continue; + } + let code = next_code[len as usize]; + next_code[len as usize] += 1; + codes[symbol] = Some(HuffmanCode { code, len }); + } + codes +} + +#[derive(Default)] +struct BitWriter { + bytes: Vec, + bit_pos: usize, +} + +impl BitWriter { + fn write_bits(&mut self, value: u32, count: u8) { + super::super::fast::write_msb_bits( + &mut self.bytes, + &mut self.bit_pos, + u64::from(value), + usize::from(count), + ); + } + + fn write_encoded_u32(&mut self, value: u32) { + if value < 16 { + self.write_bits(0, 2); + self.write_bits(value, 4); + } else if value < 256 { + self.write_bits(1, 2); + self.write_bits(value, 8); + } else if value <= 0xffff { + self.write_bits(2, 2); + self.write_bits(value, 16); + } else { + self.write_bits(3, 2); + self.write_bits(value >> 16, 16); + self.write_bits(value & 0xffff, 16); + } + } + + fn write_bit(&mut self, bit: bool) { + if self.bit_pos.is_multiple_of(8) { + self.bytes.push(0); + } + if bit { + let shift = 7 - (self.bit_pos % 8); + *self.bytes.last_mut().unwrap() |= 1 << shift; + } + self.bit_pos += 1; + } + + fn finish(self) -> Vec { + self.bytes + } +} + +// Round-trip and malformed-input tests use private encoder helpers and ancestor +// decoder state, without exposing those internals to sibling production code. +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/arca-rar/vendor/rars/src/codec/rar29/standard_filters.rs b/arca-rar/vendor/rars/src/codec/rar29/standard_filters.rs new file mode 100644 index 00000000..91828309 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar29/standard_filters.rs @@ -0,0 +1,302 @@ +//! RAR29 standard VM filter recognition and execution. +use super::super::filters; +#[cfg(all(test, feature = "write"))] +use super::Allowance; +use super::{ + rar29_delta_messages, Budget, Buffer, Error, Result, StandardFilter, MAX_AUDIO_CHANNELS, +}; +use crate::crc32::crc32; + +pub(super) fn identify_standard_filter(code: &[u8]) -> Option { + if code.iter().fold(0u8, |acc, &byte| acc ^ byte) != 0 { + return None; + } + match (code.len(), crc32(code)) { + (53, 0xad57_6887) => Some(StandardFilter::E8), + (57, 0x3cd7_e57e) => Some(StandardFilter::E8E9), + (120, 0x3769_893f) => Some(StandardFilter::Itanium), + (29, 0x0e06_077d) => Some(StandardFilter::Delta), + (149, 0x1c2c_5dc8) => Some(StandardFilter::Rgb), + (216, 0xbc85_e701) => Some(StandardFilter::Audio), + _ => None, + } +} + +#[cfg(all(test, feature = "write"))] +pub(super) fn apply_standard_filter_with_control( + filter: StandardFilter, + data: &mut Vec, + file_offset: u32, + regs: &[u32; 7], + control: &crate::read_control::ReadControl, +) -> Result<()> { + let mut owned = Buffer::from_vec(std::mem::take(data)); + let result = + apply_standard_filter_with_allowance(filter, &mut owned, file_offset, regs, control); + *data = owned.into_vec(); + result +} +#[cfg(all(test, feature = "write"))] +pub(super) fn apply_standard_filter( + filter: StandardFilter, + data: &mut Vec, + file_offset: u32, + regs: &[u32; 7], +) -> Result<()> { + apply_standard_filter_with_control( + filter, + data, + file_offset, + regs, + &crate::read_control::ReadControl::default(), + ) +} + +pub(super) fn apply_standard_filter_with_allowance( + filter: StandardFilter, + data: &mut Buffer, + file_offset: u32, + regs: &[u32; 7], + control: &crate::read_control::ReadControl, +) -> Result<()> { + control.check_codec()?; + + match filter { + StandardFilter::E8 => filters::e8e9_decode_with_control(data, file_offset, false, control)?, + StandardFilter::E8E9 => { + filters::e8e9_decode_with_control(data, file_offset, true, control)? + } + StandardFilter::Itanium => itanium_decode_with_control(data, file_offset, control)?, + StandardFilter::Delta => { + let channels = regs[0] as usize; + // Validate once in the shared decoder, retaining the register + // diagnostic for zero as well as excessive channel counts. + let mut messages = rar29_delta_messages(); + messages.zero_channels = messages.invalid_channels; + *data = filters::delta_decode_with_allowance( + data, + channels, + messages, + control, + &data.allowance(), + )?; + } + StandardFilter::Rgb => { + if regs[0] < 3 || regs[1] > 2 { + return Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid", + )); + } + let width = regs[0] as usize - 3; + let pos_r = regs[1] as usize; + *data = rgb_decode_with_allowance(data, width, pos_r, control, &data.allowance())?; + } + StandardFilter::Audio => { + let channels = regs[0] as usize; + if channels == 0 || channels > MAX_AUDIO_CHANNELS { + return Err(Error::InvalidData( + "RAR 2.9 AUDIO filter channel count is invalid", + )); + } + *data = audio_decode_with_allowance(data, channels, control, &data.allowance())?; + } + } + Ok(()) +} + +#[cfg(all(test, feature = "write"))] +pub(super) fn itanium_decode(data: &mut [u8], file_offset: u32) { + itanium_decode_with_control( + data, + file_offset, + &crate::read_control::ReadControl::default(), + ) + .expect("uncancelled filter"); +} + +pub(super) fn itanium_decode_with_control( + data: &mut [u8], + file_offset: u32, + control: &crate::read_control::ReadControl, +) -> Result<()> { + control.check_codec()?; + let mut poller = control.poller(); + if data.len() <= 21 { + return Ok(()); + } + let base_offset = file_offset >> 4; + // Each 16-byte Itanium bundle can inspect a 4-byte instruction field that + // starts up to 13 bytes into the bundle. Keeping a 21-byte tail prevents + // decoding a partial final bundle. + let block_count = (data.len() - 21).div_ceil(16); + for block in 0..block_count { + let pos = block * 16; + poller.check_codec(pos)?; + let file_offset = base_offset.wrapping_add(block as u32); + let mut mask = (0x334b_0000u32 >> (data[pos] & 0x1e)) & 3; + if mask != 0 { + mask += 1; + while mask <= 4 { + let p = pos + (mask as usize * 5 - 8); + if ((data[p + 3] >> mask) & 15) == 5 { + let raw = u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]); + let mut value = raw >> mask; + value = value.wrapping_sub(file_offset) & 0x000f_ffff; + let raw = (raw & !(0x000f_ffff << mask)) | (value << mask); + data[p..p + 4].copy_from_slice(&raw.to_le_bytes()); + } + mask += 1; + } + } + } + + Ok(()) +} + +#[cfg(all(test, feature = "write"))] +pub(super) fn rgb_decode_with_control( + data: &[u8], + width: usize, + pos_r: usize, + control: &crate::read_control::ReadControl, +) -> Result> { + rgb_decode_with_allowance(data, width, pos_r, control, &Allowance::default()) + .map(Buffer::into_vec) +} +fn rgb_decode_with_allowance( + data: &[u8], + width: usize, + pos_r: usize, + control: &crate::read_control::ReadControl, + allowance: &B, +) -> Result> { + control.check_codec()?; + let mut poller = control.poller(); + if data.len() < 3 || width == 0 || !width.is_multiple_of(3) || width > data.len() || pos_r > 2 { + return Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid", + )); + } + let mut out = Buffer::filled(data.len(), 0, allowance)?; + let mut src = 0usize; + for channel in 0..3 { + let mut prev = 0u8; + let mut i = channel; + while i < data.len() { + poller.check_codec(src)?; + let predicted = if i >= width + 3 { + rgb_predict(prev, out[i - width], out[i - width - 3]) + } else { + prev + }; + let encoded = *data + .get(src) + .ok_or(Error::InvalidData("RAR 2.9 RGB filter source is truncated"))?; + prev = predicted.wrapping_sub(encoded); + out[i] = prev; + src += 1; + i += 3; + } + } + for i in (pos_r..data.len().saturating_sub(2)).step_by(3) { + poller.check_codec(i)?; + let green = out[i + 1]; + out[i] = out[i].wrapping_add(green); + out[i + 2] = out[i + 2].wrapping_add(green); + } + Ok(out) +} + +pub(super) fn rgb_predict(prev: u8, upper: u8, upper_left: u8) -> u8 { + let predicted = i32::from(prev) + i32::from(upper) - i32::from(upper_left); + let pa = (predicted - i32::from(prev)).abs(); + let pb = (predicted - i32::from(upper)).abs(); + let pc = (predicted - i32::from(upper_left)).abs(); + if pa <= pb && pa <= pc { + prev + } else if pb <= pc { + upper + } else { + upper_left + } +} + +#[cfg(all(test, feature = "write"))] +pub(super) fn audio_decode_with_control( + data: &[u8], + channels: usize, + control: &crate::read_control::ReadControl, +) -> Result> { + audio_decode_with_allowance(data, channels, control, &Allowance::default()) + .map(Buffer::into_vec) +} +fn audio_decode_with_allowance( + data: &[u8], + channels: usize, + control: &crate::read_control::ReadControl, + allowance: &B, +) -> Result> { + control.check_codec()?; + let mut poller = control.poller(); + let mut out = Buffer::filled(data.len(), 0, allowance)?; + let mut src = 0usize; + for channel in 0..channels { + let mut prev_byte = 0u32; + let mut prev_delta = 0i32; + let mut d1 = 0i32; + let mut d2 = 0i32; + let mut k1 = 0i32; + let mut k2 = 0i32; + let mut k3 = 0i32; + let mut dif = [0u32; 7]; + let mut byte_count = 0usize; + let mut i = channel; + while i < data.len() { + poller.check_codec(src)?; + let d3 = d2; + d2 = prev_delta - d1; + d1 = prev_delta; + let predicted = ((8 * prev_byte as i32 + k1 * d1 + k2 * d2 + k3 * d3) >> 3) & 0xff; + let encoded = *data.get(src).ok_or(Error::InvalidData( + "RAR 2.9 AUDIO filter source is truncated", + ))?; + src += 1; + let decoded = (predicted as u8).wrapping_sub(encoded); + out[i] = decoded; + prev_delta = decoded.wrapping_sub(prev_byte as u8) as i8 as i32; + prev_byte = decoded as u32; + let d = (encoded as i8 as i32) << 3; + dif[0] += d.unsigned_abs(); + dif[1] += (d - d1).unsigned_abs(); + dif[2] += (d + d1).unsigned_abs(); + dif[3] += (d - d2).unsigned_abs(); + dif[4] += (d + d2).unsigned_abs(); + dif[5] += (d - d3).unsigned_abs(); + dif[6] += (d + d3).unsigned_abs(); + if byte_count & 0x1f == 0 { + let mut min = dif[0]; + let mut min_index = 0usize; + dif[0] = 0; + for (index, value) in dif.iter_mut().enumerate().skip(1) { + if *value < min { + min = *value; + min_index = index; + } + *value = 0; + } + match min_index { + 1 if k1 >= -16 => k1 -= 1, + 2 if k1 < 16 => k1 += 1, + 3 if k2 >= -16 => k2 -= 1, + 4 if k2 < 16 => k2 += 1, + 5 if k3 >= -16 => k3 -= 1, + 6 if k3 < 16 => k3 += 1, + _ => {} + } + } + byte_count += 1; + i += channels; + } + } + Ok(out) +} diff --git a/arca-rar/vendor/rars/src/codec/rar29/tests.rs b/arca-rar/vendor/rars/src/codec/rar29/tests.rs new file mode 100644 index 00000000..0091fcc2 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar29/tests.rs @@ -0,0 +1,4223 @@ +#[test] +fn public_decoder_clone_keeps_independent_lz_and_ppmd_solid_state() { + let first = b"RAR29 model and window history ".repeat(32); + let second = b"RAR29 model and window history ".repeat(8); + for engine in [super::ChainEngine::Lz, super::ChainEngine::Ppmd] { + let mut encoder = super::Unpack29Encoder::new(); + let packed_first = encoder + .encode_member_with_engine(&first, engine, &[], &mut |_| true) + .unwrap(); + let packed_second = encoder + .encode_member_with_engine(&second, engine, &[], &mut |_| true) + .unwrap(); + let mut original = super::Unpack29::new(); + assert_eq!( + original + .decode_non_solid_member(&packed_first, first.len()) + .unwrap(), + first + ); + let mut copied = original.clone(); + original.reset_non_solid(); + drop(original); + assert_eq!( + copied.decode_member(&packed_second, second.len()).unwrap(), + second + ); + } +} + +fn refuse_each_rar29_allocation( + mut run: impl FnMut(&crate::codec::workspace::RefusingBudget) -> Result<()>, +) { + use crate::codec::workspace::RefusingBudget; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + let attempts = baseline.attempts(); + assert!(attempts > 0); + assert_eq!(baseline.used(), 0); + for index in 0..attempts { + let budget = RefusingBudget::new(index); + assert!( + matches!(run(&budget), Err(Error::Cancelled)), + "allocation {index}" + ); + assert_eq!(budget.used(), 0, "allocation {index}"); + } +} + +#[test] +fn reader_rar29_refusals_release_input_huffman_history_results_and_checkpoints() { + refuse_each_rar29_allocation(|budget| { + let mut state = super::Reader29State::with_allowance(budget); + let decoded = state.decode_member_owned(COMPRESSED_TEXT, 2400)?; + assert_eq!(&decoded[..], expected_text()); + let checkpoint = state.try_clone()?; + assert_eq!(checkpoint.output, state.output); + assert_eq!(checkpoint.main.state.len(), state.main.state.len()); + let mut follower = COMPRESSED_TEXT; + let mut out = super::Buffer::new(budget); + state.decode_non_solid_member_from_reader(&mut follower, 2400, &mut out)?; + assert_eq!(&out[..], expected_text()); + Ok(()) + }); +} + +#[test] +fn reader_rar29_refusals_release_vm_code_records_globals_execution_and_filters() { + const COUNTER: &[u8] = &[ + 0x0d, 0x05, 0xc0, 0x7c, 0x00, 0x0f, 0x01, 0x01, 0xaf, 0x80, 0x01, 0xe0, 0x20, 0x01, 0xf0, + 0x00, 0x3c, 0x03, 0x00, 0x1b, 0x80, + ]; + let records = [ + OwnedVmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: Vec::new(), + code: COUNTER, + global_data: vec![b'A'], + }, + OwnedVmFilterRecord { + block_start: 1, + block_size: 1, + init_regs: Vec::new(), + code: COUNTER, + global_data: Vec::new(), + }, + ]; + let refs = records.iter().collect::>(); + let records = encoded_filter_records_at(&refs, 0, usize::MAX, &mut Vec::new()).unwrap(); + let packed = super::encode_member_inner( + b"xx", + &[], + &records, + EncodeOptions::default(), + false, + &mut [0; TABLE_COUNT], + None, + ) + .unwrap(); + refuse_each_rar29_allocation(|budget| { + let mut state = super::Reader29State::with_allowance(budget); + let decoded = state.decode_member_owned(&packed, 2)?; + assert_eq!(&decoded[..], b"AB"); + let checkpoint = state.try_clone()?; + assert_eq!(checkpoint.programs[0].globals, state.programs[0].globals); + Ok(()) + }); +} + +#[test] +fn reader_rar29_refusals_release_standard_filter_scratch_and_output() { + for (filter, regs) in [ + (StandardFilter::Delta, [2, 0, 0, 0, 0, 0, 0]), + (StandardFilter::Rgb, [9, 0, 0, 0, 0, 0, 0]), + (StandardFilter::Audio, [2, 0, 0, 0, 0, 0, 0]), + ] { + refuse_each_rar29_allocation(|budget| { + let mut state = super::Reader29State::with_allowance(budget); + state.output = super::Buffer::copied(&[0; 96], budget)?; + state.programs.try_push(VmProgram { + kind: VmProgramKind::Standard(filter), + block_size: 96, + exec_count: 0, + globals: super::Buffer::new(budget), + })?; + state.filters.try_push(VmFilter { + program: 0, + start: 0, + size: 96, + regs, + global_data: super::Buffer::new(budget), + })?; + let checkpoint = state.try_clone()?; + assert_eq!(checkpoint.filters.len(), 1); + let out = state.filtered_range_owned(0, 96, 0)?; + assert_eq!(&out[..], &[0; 96]); + assert!(state.filters.is_empty()); + Ok(()) + }); + } +} + +#[test] +fn reader_rar29_returned_output_keeps_reservation_after_decoder_drop() { + use crate::codec::workspace::Allowance; + let ledger = Allowance::limited(128 * 1024); + let mut reservation = ledger.reserve(120 * 1024).unwrap(); + reservation.start(); + let budget = reservation.allowance(); + let mut state = super::Reader29State::with_allowance(&budget); + let out = state.decode_member_owned(COMPRESSED_TEXT, 2400).unwrap(); + drop(state); + drop(budget); + reservation.retire(); + assert!(ledger.used() >= out.capacity() as u64); + assert_eq!(&out[..], expected_text()); + drop(out); + assert_eq!(ledger.used(), 0); +} + +type Unpack29 = super::Reader29State; +#[test] +fn ppmd_progress_preserves_bytes_and_interrupts_both_engines() { + let input = b"PPMd cooperative cancellation payload\n".repeat(400); + for escapes in [false, true] { + let expected = if escapes { + super::unpack29_encode_ppmd(&input, 1 << 20).unwrap() + } else { + super::unpack29_encode_ppmd_literals(&input).unwrap() + }; + let actual = + super::unpack29_encode_ppmd_with_progress(&input, escapes, None, 1 << 20, &mut |_| { + true + }) + .unwrap(); + assert_eq!(actual, expected); + let mut stopped_at = 0; + let result = super::unpack29_encode_ppmd_with_progress( + &input, + escapes, + None, + 1 << 20, + &mut |position| { + stopped_at = position; + position < 4096 + }, + ); + assert!(matches!(result, Err(super::Error::Cancelled))); + assert!(stopped_at >= 4096 && stopped_at < input.len()); + } +} + +#[test] +fn ppmd_progress_covers_entry_filtered_preprocessing_and_completion() { + let input = b"PPMd cancellation boundary payload\n".repeat(400); + for escapes in [false, true] { + let mut calls = Vec::new(); + assert_eq!( + super::unpack29_encode_ppmd_with_progress( + &input, + escapes, + None, + 1 << 20, + &mut |position| { + calls.push(position); + false + }, + ), + Err(super::Error::Cancelled) + ); + assert_eq!(calls, [0]); + + let mut completions = 0; + assert_eq!( + super::unpack29_encode_ppmd_with_progress( + &input, + escapes, + None, + 1 << 20, + &mut |position| { + if position == input.len() { + completions += 1; + return false; + } + true + }, + ), + Err(super::Error::Cancelled) + ); + assert_eq!(completions, 1); + } + + let mut polls = 0; + let filter = crate::FilterSpec::whole(crate::FilterKind::E8); + assert_eq!( + super::unpack29_encode_ppmd_with_progress(&input, true, Some(filter), 1 << 20, &mut |_| { + polls += 1; + polls < 3 + },), + Err(super::Error::Cancelled) + ); + // Entry, the bounded filter record, then the PPMd block itself. + assert_eq!(polls, 3); + + let mut polls = 0; + assert_eq!( + super::unpack29_encode_ppmd_with_progress( + &input, + true, + Some(crate::FilterSpec::whole(crate::FilterKind::E8)), + 1 << 20, + &mut |_| { + polls += 1; + polls < 2 + }, + ), + Err(super::Error::Cancelled) + ); + assert_eq!(polls, 2); +} + +#[test] +fn lz_progress_can_cancel_its_final_report() { + let input = b"RAR29 final LZ progress boundary\n".repeat(400); + let mut completions = 0; + let result = super::unpack29_encode_literals_with_options_and_progress( + &input, + EncodeOptions::default(), + &mut |position| { + if position == input.len() { + completions += 1; + return false; + } + true + }, + ); + + assert_eq!(result, Err(super::Error::Cancelled)); + assert_eq!(completions, 1); +} + +use super::{audio_decode_with_control, itanium_decode_with_control, rgb_decode_with_control}; + +#[test] +fn cancellation_interrupts_lz_and_ppmd_after_non_solid_reset() { + let data = b"cancellable legacy symbols ".repeat(16384); + for (ppmd, packed) in [ + (false, unpack29_encode_literals(&data).unwrap()), + (true, unpack29_encode_ppmd_literals(&data).unwrap()), + ] { + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack29::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + // PPMd checks cancellation before allocating the initial model. + decoder + .read_control + .cancel_after_checks(4 + usize::from(ppmd)); + assert_eq!( + decoder + .decode_non_solid_member(&packed, data.len()) + .unwrap_err(), + Error::Cancelled + ); + assert!(decoder.current_pos() > 0 && decoder.current_pos() < data.len()); + } +} + +#[test] +fn cancellation_interrupts_standard_filter_passes() { + for kind in 0..3 { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(2); + let mut data = vec![0; 384 * 1024]; + let result = match kind { + 0 => itanium_decode_with_control(&mut data, 0, &control), + 1 => rgb_decode_with_control(&data, 96, 0, &control).map(|_| ()), + _ => audio_decode_with_control(&data, 2, &control).map(|_| ()), + }; + assert_eq!(result.unwrap_err(), Error::Cancelled); + } +} + +#[test] +fn cancellation_propagates_from_shared_standard_filter_decoders() { + for (filter, regs) in [ + (StandardFilter::E8, [0; 7]), + (StandardFilter::E8E9, [0; 7]), + (StandardFilter::Delta, [1, 0, 0, 0, 0, 0, 0]), + ] { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(2); + let mut data = vec![0; 384 * 1024]; + assert_eq!( + super::apply_standard_filter_with_control(filter, &mut data, 0, ®s, &control,), + Err(Error::Cancelled) + ); + } +} +use super::rarvm::{Instruction, Opcode, Operand, Program}; +use std::ops::Range; + +fn encode_tokens(input: &[u8], history: &[u8], options: EncodeOptions) -> Vec { + encode_tokens_with_progress(input, history, options, None) + .expect("encoding without cancellation cannot be cancelled") +} + +fn should_lazy_emit_literal( + input: &[u8], + pos: usize, + finder: &Rar29MatchFinder, + options: EncodeOptions, + state: &EncoderMatchState, + current: MatchCandidate, +) -> bool { + lazy_match_decision(input, pos, finder, options, state, current).0 +} + +use super::{ + apply_standard_filter, audio_encode, best_match, best_ppmd_match, canonical_codes, + encode_level_tokens_against, encode_ppmd_hybrid, encode_table_level_tokens, + encode_tokens_with_progress, encoded_filter_records_at, itanium_decode, itanium_encode, + lazy_match_decision, level_code_lengths, split_large_filter, unpack29_decode, + unpack29_encode_literals, unpack29_encode_ppmd, unpack29_encode_ppmd_literals, + unpack29_encode_ppmd_with_filter, BitReader, BitWriter, ChainEngine, EncodeOptions, + EncodeToken, EncoderMatchState, Error, Huffman, LevelToken, MatchCandidate, + OwnedVmFilterRecord, PpmdEncodeToken, PpmdEncoder, Rar29MatchFinder, Result, StandardFilter, + Unpack29Encoder, VmFilter, VmProgram, VmProgramKind, LENGTH_COUNT, LOW_OFFSET_COUNT, + MAIN_COUNT, MAX_ENCODER_MATCH_LENGTH, MAX_ENCODER_MATCH_OFFSET, MAX_HISTORY, + MAX_MATCH_CANDIDATES, MAX_VM_AUDIO_FILTER_BLOCK_SIZE, MAX_VM_DELTA_FILTER_BLOCK_SIZE, + MAX_VM_FILTER_BLOCK_SIZE, OFFSET_COUNT, PPMD_DICTIONARY_MB, PPMD_ESC, PPMD_ORDER, + RAR3_AUDIO_FILTER_BYTECODE, RAR3_DELTA_FILTER_BYTECODE, RAR3_ITANIUM_FILTER_BYTECODE, + RAR3_RGB_FILTER_BYTECODE, STREAM_CHUNK, TABLE_COUNT, +}; + +/// A flat code charges the same for every symbol in play. The keep-tables +/// bit works by pushing the tokens onto one symbol, which buys nothing +/// unless the level code notices. +#[test] +fn the_level_code_spends_fewer_bits_on_the_common_symbol() { + let mut tokens = vec![LevelToken::plain(0); 200]; + tokens.push(LevelToken::plain(7)); + tokens.push(LevelToken::plain(9)); + tokens.push(LevelToken::plain(11)); + let lengths = level_code_lengths(&tokens); + assert!( + lengths[0] < lengths[7], + "the symbol used 200 times costs {} bits and the one used once costs {}", + lengths[0], + lengths[7] + ); +} + +/// Symbols 0 to 15 are read as a delta against the table the reader holds, +/// so a table that has not changed is a run of zeroes. Runs of zero-length +/// codes and repeats stay as they are, since the reader takes those without +/// reference to what it holds. +#[test] +fn a_table_matching_the_previous_one_codes_as_deltas_of_zero() { + let mut lengths = [0u8; TABLE_COUNT]; + for (position, slot) in lengths.iter_mut().enumerate().take(120) { + *slot = (position % 13 + 1) as u8; + } + let against_itself = encode_level_tokens_against(&lengths, &lengths); + assert!( + against_itself + .iter() + .all(|token| token.symbol == 0 || token.symbol >= 16), + "a table coded against itself should spend only zero deltas and runs" + ); + + let outright = encode_table_level_tokens(&lengths); + assert!( + outright.iter().any(|token| (1..16).contains(&token.symbol)), + "the same table coded outright has to name its lengths" + ); +} + +/// Members that share a shape and a vocabulary without repeating each +/// other. Every line carries a number that appears nowhere else, so there +/// is little for the chain to match and the PPMd model is what carries +/// between one member and the next. +fn related_text_member(seed: u64, lines: usize) -> Vec { + const WORDS: [&str; 12] = [ + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", + "juliet", "kilo", "lima", + ]; + let mut state = seed | 1; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let mut out = Vec::new(); + for line in 0..lines { + for _ in 0..3 { + out.extend_from_slice(WORDS[(next() % 12) as usize].as_bytes()); + out.push(b'.'); + } + out.extend_from_slice( + format!( + "{} = {}\n", + line + seed as usize * lines, + next() % 1_000_000_007 + ) + .as_bytes(), + ); + } + out +} + +/// WinRAR asks for a PPMd reset on 1 of the 68 PPMd blocks in a solid +/// archive of 70 small files and carries the model through the other 67. +/// Building a model per member instead is most of why that archive used to +/// be 18% smaller than ours. +#[test] +fn a_solid_chain_carries_one_ppmd_model_across_its_members() { + // Members small enough that a model built for one alone never warms up, + // which is the shape the win is really about: 70 manpages, not one book. + let members: Vec> = (0..12).map(|i| related_text_member(i + 1, 40)).collect(); + let mut chain = Unpack29Encoder::with_options(EncodeOptions::default()); + let packed: Vec> = members + .iter() + .map(|member| { + chain + .encode_member_with_engine(member, ChainEngine::Smaller, &[], &mut |_| true) + .unwrap() + }) + .collect(); + + // Which engine wins a given member is a size question and not the + // point here. The point is that once one member has built a model, no + // later member throws it away. + let ppmd: Vec<&Vec> = packed.iter().filter(|block| block[0] & 0x80 != 0).collect(); + assert!( + ppmd.len() > 2, + "this chain was meant to go PPMd more than twice" + ); + assert_eq!( + ppmd[0][0] & 0x20, + 0x20, + "the first PPMd member builds a model" + ); + assert!( + ppmd[1..].iter().all(|block| block[0] & 0x20 == 0), + "no member after the first should throw the model away" + ); + + // The same member, coded by a chain that has read nothing. + let alone = Unpack29Encoder::with_options(EncodeOptions::default()) + .encode_member_with_engine( + members.last().unwrap(), + ChainEngine::Smaller, + &[], + &mut |_| true, + ) + .unwrap(); + let last = ppmd.last().unwrap(); + assert!( + last.len() * 6 < alone.len() * 5, + "a member coded against the chain's model cost {} bytes against {} on its own", + last.len(), + alone.len() + ); +} + +/// Both engines carry state and only the winner's may advance, or the +/// reader is left holding something the writer never wrote. +#[test] +fn a_solid_chain_mixing_engines_round_trips() { + let mut members: Vec> = Vec::new(); + for index in 0..3u64 { + members.push(related_text_member(index + 1, 200)); + // Bytes PPMd loses on, so the chain has to switch engines and back. + let mut state = 0x9e37_79b9_7f4a_7c15u64 ^ index; + let mut noise = Vec::new(); + while noise.len() < 20_000 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + noise.extend_from_slice(&state.to_le_bytes()); + } + members.push(noise); + } + + let mut chain = Unpack29Encoder::with_options(EncodeOptions::default()); + let mut stream = Vec::new(); + let mut engines = Vec::new(); + for member in &members { + let packed = chain + .encode_member_with_engine(member, ChainEngine::Smaller, &[], &mut |_| true) + .unwrap(); + engines.push(packed[0] & 0x80 != 0); + stream.push(packed); + } + assert!( + engines.contains(&true) && engines.contains(&false), + "this chain was meant to use both engines, got {engines:?}" + ); + + let mut decoder = Unpack29::new(); + for (packed, member) in stream.iter().zip(&members) { + assert_eq!( + decoder.decode_member(packed, member.len()).unwrap(), + *member + ); + } +} + +/// Calls to fixed addresses, which the x86 filter flattens into three +/// repeated values. See the writer's own copy of this for why the operand +/// is relative. +fn call_heavy_x86(seed: u32, calls: usize) -> Vec { + const TARGETS: [u32; 3] = [0x1000, 0x2400, 0x3800]; + let mut data = Vec::with_capacity(calls * 16); + let mut state = seed | 1; + for index in 0..calls { + let target = TARGETS[index % TARGETS.len()]; + let relative = target.wrapping_sub(data.len() as u32 + 5); + data.push(0xe8); + data.extend_from_slice(&relative.to_le_bytes()); + for _ in 0..11 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + data.push((state >> 24) as u8 | 0x40); + } + } + data +} + +/// Only the winning candidate may move the chain. +/// +/// Every candidate codes the member for real, and each one leaves a +/// code-length table and a window behind it. A reader rebuilds both from +/// the bytes it actually reads, so committing a loser's leaves the next +/// member coded against a table and a history no decoder holds. That does +/// not show up as an error: the member after it decodes to plausible +/// rubbish, which is why this decodes the whole chain rather than checking +/// sizes. +#[test] +fn a_solid_chain_keeps_only_the_winning_candidates_state() { + // One member per outcome. The x86 member takes the x86 filter, the + // counters take the delta, and the text takes neither, so every + // candidate both wins somewhere and loses somewhere. Text is the + // member that matters: delta rewrites it into something with a very + // different table, and that table is exactly what must not survive + // losing. + let x86 = call_heavy_x86(0x1234_5678, 400); + let counters: Vec = (0..1600u32).flat_map(|n| (n * 7).to_le_bytes()).collect(); + let mut text = Vec::new(); + for index in 0..400u32 { + text.extend_from_slice(format!("field_{index:04} = value {index:04}\n").as_bytes()); + } + let members = [x86, counters, text]; + + let candidates = vec![ + Vec::new(), + vec![crate::FilterSpec::whole(crate::FilterKind::E8)], + vec![crate::FilterSpec::whole(crate::FilterKind::Delta { + channels: 4, + })], + ]; + let mut chain = Unpack29Encoder::with_options(EncodeOptions::default()); + let stream: Vec> = members + .iter() + .map(|member| { + chain + .encode_member_with_engine(member, ChainEngine::Lz, &candidates, &mut |_| true) + .unwrap() + }) + .collect(); + + // A chain offered no candidates at all, to prove the filter was + // chosen rather than never reached. Only the first member is + // comparable member-for-member: once the chains disagree about what + // they coded, they disagree about their histories too. + let unfiltered_only = { + let mut chain = Unpack29Encoder::with_options(EncodeOptions::default()); + members + .iter() + .map(|member| { + chain + .encode_member_with_engine(member, ChainEngine::Lz, &[], &mut |_| true) + .unwrap() + }) + .collect::>() + }; + assert!( + stream[0].len() < unfiltered_only[0].len(), + "the x86 member should have taken the filter, got {} bytes against {}", + stream[0].len(), + unfiltered_only[0].len() + ); + let chosen: usize = stream.iter().map(Vec::len).sum(); + let never: usize = unfiltered_only.iter().map(Vec::len).sum(); + assert!( + chosen < never, + "the chain that could choose wrote {chosen} bytes against {never}" + ); + + // Coding each member again for every candidate is only safe if the + // losers leave nothing behind, so the chain is decoded a member at a + // time and its table compared with the reader's after each one. The + // bytes alone would not catch a wrong table: a member whose deltas are + // applied to the wrong base still decodes, to rubbish, and only the + // member after it notices. + let mut decoder = Unpack29::new(); + let mut chain = Unpack29Encoder::with_options(EncodeOptions::default()); + for (index, member) in members.iter().enumerate() { + let packed = chain + .encode_member_with_engine(member, ChainEngine::Lz, &candidates, &mut |_| true) + .unwrap(); + assert_eq!(packed, stream[index]); + assert_eq!( + decoder.decode_member(&packed, member.len()).unwrap(), + *member + ); + assert_eq!( + chain.levels, decoder.levels, + "member {index} left the writer holding a table the reader does not" + ); + // The reader's window holds what the LZ layer coded, filters not + // yet applied, which is the same thing the writer remembers. + let common = chain.history.len().min(decoder.output.len()); + assert_eq!( + chain.history[chain.history.len() - common..], + decoder.output[decoder.output.len() - common..], + "member {index} left the writer remembering bytes the reader never held" + ); + } +} + +/// The reader carries its table across the members of a solid chain, so +/// members that look alike stop paying to describe the same table again. +#[test] +fn a_solid_chain_stops_repaying_for_the_same_table() { + let mut member = Vec::new(); + for index in 0..400u32 { + member.extend_from_slice(format!("field_{index:04} = value {index:04}\n").as_bytes()); + } + + let mut chained = Unpack29Encoder::with_options(EncodeOptions::default()); + let first = chained.encode_member(&member).unwrap(); + let second = chained.encode_member(&member).unwrap(); + + let mut alone = Unpack29Encoder::with_options(EncodeOptions::default()); + alone.encode_member(&member).unwrap(); + let restated = Unpack29Encoder::with_options(EncodeOptions::default()) + .encode_member(&member) + .unwrap(); + + assert_eq!(first.len(), restated.len()); + assert!( + second.len() < restated.len(), + "the second member of the chain cost {} bytes against {} on its own", + second.len(), + restated.len() + ); +} + +/// The tokens the hybrid encoder emits for `input`, decided against the +/// same live model the production path prices against. +fn hybrid_tokens(input: &[u8], max_match_distance: usize) -> Vec { + let mut encoder = + PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, usize::from(PPMD_DICTIONARY_MB)).unwrap(); + let mut tokens = Vec::new(); + encode_ppmd_hybrid(input, max_match_distance, &mut encoder, |token| { + tokens.push(token) + }) + .unwrap(); + tokens +} + +const COMPRESSED_TEXT: &[u8] = &[ + 0x09, 0x10, 0x10, 0x93, 0xe4, 0xce, 0x7f, 0xa2, 0xba, 0x80, 0x46, 0x16, 0x82, 0x63, 0xe9, 0x9a, + 0x19, 0xe4, 0x10, 0xe0, 0x41, 0x3d, 0x16, 0xfc, 0x4d, 0xfa, 0x6f, 0xf2, 0x5c, 0xae, 0x32, 0x86, + 0xc9, 0x95, 0x9d, 0xf1, 0x04, 0xa4, 0xe8, 0x92, 0x8f, 0x12, 0xd7, 0xe7, 0xba, 0xcb, 0x26, 0xf1, + 0x97, 0xac, 0x7c, 0x5f, 0xfd, 0xa0, 0x00, 0x1f, 0x77, 0x50, +]; + +#[test] +fn decodes_rar29_lz_member() { + assert_eq!( + unpack29_decode(COMPRESSED_TEXT, 2400).unwrap(), + expected_text() + ); +} + +#[test] +fn rejects_oversubscribed_rar29_huffman_tables() { + assert!(matches!( + Huffman::from_lengths(&[1, 1, 1]), + Err(Error::InvalidData("RAR 2.9 oversubscribed Huffman table")) + )); +} + +#[test] +fn codec_helpers_reject_out_of_contract_reads_and_history() { + assert_eq!( + BitReader::from_bytes(&[0; 4]).peek_bits(25), + Err(Error::InvalidData("RAR 2.9 bit read is too wide")) + ); + + let mut decoder = Unpack29::new(); + decoder.base_offset = 10; + decoder.output.extend_from_slice(b"retained").unwrap(); + assert_eq!( + decoder.raw_range(9, 10), + Err(Error::InvalidData( + "RAR 2.9 retained history is unavailable" + )) + ); + assert_eq!( + decoder.raw_range(12, 11), + Err(Error::InvalidData( + "RAR 2.9 retained history is unavailable" + )) + ); + assert_eq!( + decoder.raw_range(10, 19), + Err(Error::InvalidData( + "RAR 2.9 retained history is unavailable" + )) + ); +} + +#[test] +fn canonical_assignment_matches_pinned_codes_and_generated_alphabets() { + let pinned = canonical_codes(&[1, 2, 3, 3, 0]); + assert_eq!( + pinned + .iter() + .map(|code| code.map(|code| (code.code, code.len))) + .collect::>(), + [Some((0, 1)), Some((2, 2)), Some((6, 3)), Some((7, 3)), None] + ); + for size in [ + super::LEVEL_COUNT, + MAIN_COUNT, + OFFSET_COUNT, + LOW_OFFSET_COUNT, + LENGTH_COUNT, + ] { + let mut single = vec![0; size]; + single[size - 1] = 1; + let mut deep = vec![0; size]; + let (mut a, mut b) = (1, 1); + for frequency in deep.iter_mut().take(40) { + *frequency = a; + (a, b) = (b, a + b); + } + for frequencies in [ + vec![0; size], + single, + vec![1; size], + (0..size).map(|i| usize::from(i % 3 == 0)).collect(), + deep, + ] { + let lengths = crate::codec::huffman::lengths_for_frequencies(&frequencies, 15); + assert!(lengths.iter().all(|&length| length <= 15)); + let codes = canonical_codes(&lengths); + let table = Huffman::from_lengths(&lengths).unwrap(); + for (symbol, code) in codes.iter().enumerate() { + assert_eq!(code.is_some(), frequencies[symbol] != 0); + if let Some(code) = code { + let mut bits = BitWriter::default(); + bits.write_bits(u32::from(code.code), code.len); + assert_eq!( + table + .decode(&mut BitReader::from_bytes(&bits.finish())) + .unwrap(), + symbol + ); + } + } + } + } +} + +#[test] +fn huffman_decoding_returns_the_index_from_its_constructor_alphabet() { + for size in [20, MAIN_COUNT, OFFSET_COUNT, LOW_OFFSET_COUNT, LENGTH_COUNT] { + let lengths = crate::codec::huffman::complete_lengths_for_frequencies(&vec![1; size], 15); + let codes = canonical_codes(&lengths); + let table = Huffman::from_lengths(&lengths).unwrap(); + + for (expected, code) in codes.iter().enumerate() { + let code = code.unwrap(); + let mut bits = BitWriter::default(); + bits.write_bits(u32::from(code.code), code.len); + let mut bits = BitReader::from_bytes(&bits.finish()); + let decoded = table.decode(&mut bits).unwrap(); + assert_eq!(decoded, expected); + assert!(decoded < size); + } + } +} + +fn table_description(level_lengths: &[u8; 20], tokens: &[LevelToken]) -> Vec { + let codes = canonical_codes(level_lengths); + let mut bits = BitWriter::default(); + bits.write_bit(false); + bits.write_bit(false); + for &length in level_lengths { + bits.write_bits(u32::from(length), 4); + } + for token in tokens { + let code = codes[token.symbol].unwrap(); + bits.write_bits(u32::from(code.code), code.len); + bits.write_bits(u32::from(token.extra_value), token.extra_bits); + } + bits.finish() +} + +fn encoded_table_description(levels: &[u8; TABLE_COUNT]) -> Vec { + let tokens = encode_table_level_tokens(levels); + let level_lengths = level_code_lengths(&tokens); + table_description(&level_lengths, &tokens) +} + +#[test] +fn table_repeats_at_position_zero_are_rejected() { + for symbol in [16, 17] { + let mut level_lengths = [0; 20]; + level_lengths[0] = 1; + level_lengths[symbol] = 1; + let token = if symbol == 16 { + LevelToken::repeat_previous_short(3) + } else { + LevelToken::repeat_previous_long(11) + }; + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&table_description(&level_lengths, &[token])); + + let expected = if symbol == 16 { + Error::InvalidData("RAR 2.9 table repeat at start") + } else { + Error::InvalidData("RAR 2.9 long table repeat at start") + }; + assert_eq!(decoder.read_tables(), Err(expected)); + } +} + +#[test] +fn table_run_past_the_destination_is_truncated_for_compatibility() { + let mut level_lengths = [0; 20]; + level_lengths[0] = 1; + level_lengths[19] = 1; + let tokens = [ + LevelToken::zero_run_long(138), + LevelToken::zero_run_long(138), + LevelToken::zero_run_long(138), + ]; + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&table_description(&level_lengths, &tokens)); + + decoder.read_tables().unwrap(); + + assert_eq!(decoder.levels, [0; TABLE_COUNT]); +} + +#[test] +fn empty_level_and_main_tables_fail_when_used() { + let mut empty_level_decoder = Unpack29::new(); + empty_level_decoder.bits = BitReader::from_bytes(&table_description(&[0; 20], &[])); + assert_eq!( + empty_level_decoder.read_tables(), + Err(Error::InvalidData("RAR 2.9 empty Huffman table")) + ); + + let mut empty_main_decoder = Unpack29::new(); + empty_main_decoder.bits = BitReader::from_bytes(&encoded_table_description(&[0; TABLE_COUNT])); + empty_main_decoder.read_tables().unwrap(); + assert_eq!( + empty_main_decoder.main.decode(&mut empty_main_decoder.bits), + Err(Error::InvalidData("RAR 2.9 empty Huffman table")) + ); +} + +#[test] +fn incomplete_huffman_table_accepts_assigned_and_rejects_unassigned_prefixes() { + let table = Huffman::from_lengths(&[2]).unwrap(); + let mut assigned = BitReader::from_bytes(&[0]); + assert_eq!(table.decode(&mut assigned), Ok(0)); + + let mut unassigned = BitReader::from_bytes(&[0x40, 0]); + assert_eq!( + table.decode(&mut unassigned), + Err(Error::InvalidData("RAR 2.9 invalid Huffman code")) + ); +} + +#[test] +fn truncated_table_headers_and_descriptions_need_more_input() { + let mut truncated_header = BitWriter::default(); + truncated_header.write_bit(false); + truncated_header.write_bit(false); + truncated_header.write_bits(1, 4); + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&truncated_header.finish()); + assert_eq!(decoder.read_tables(), Err(Error::NeedMoreInput)); + + let mut level_lengths = [0; 20]; + level_lengths[0] = 1; + level_lengths[19] = 1; + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&table_description( + &level_lengths, + &[LevelToken::zero_run_long(138)], + )); + assert_eq!(decoder.read_tables(), Err(Error::NeedMoreInput)); +} + +#[test] +fn level_and_final_tables_reject_oversubscription() { + let mut oversubscribed_level = BitWriter::default(); + oversubscribed_level.write_bit(false); + oversubscribed_level.write_bit(false); + for length in [1, 1, 1].into_iter().chain(std::iter::repeat_n(0, 17)) { + oversubscribed_level.write_bits(length, 4); + } + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&oversubscribed_level.finish()); + assert_eq!( + decoder.read_tables(), + Err(Error::InvalidData("RAR 2.9 oversubscribed Huffman table")) + ); + + let mut level_lengths = [0; 20]; + level_lengths[1] = 1; + level_lengths[19] = 1; + let tokens = [ + LevelToken::plain(1), + LevelToken::plain(1), + LevelToken::plain(1), + LevelToken::zero_run_long(138), + LevelToken::zero_run_long(138), + LevelToken::zero_run_long(125), + ]; + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&table_description(&level_lengths, &tokens)); + assert_eq!( + decoder.read_tables(), + Err(Error::InvalidData("RAR 2.9 oversubscribed Huffman table")) + ); +} + +#[test] +fn every_final_table_slice_rejects_oversubscription() { + let slices = [ + MAIN_COUNT..MAIN_COUNT + OFFSET_COUNT, + MAIN_COUNT + OFFSET_COUNT..MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT, + MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT..TABLE_COUNT, + ]; + for slice in slices { + let mut levels = [0; TABLE_COUNT]; + levels[b'A' as usize] = 1; + levels[256] = 1; + levels[slice.start] = 1; + levels[slice.start + 1] = 1; + levels[slice.start + 2] = 1; + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&encoded_table_description(&levels)); + + assert_eq!( + decoder.read_tables(), + Err(Error::InvalidData("RAR 2.9 oversubscribed Huffman table")) + ); + } +} + +#[test] +fn level_length_header_accepts_literal_fifteen_and_clips_zero_runs() { + let mut bits = BitWriter::default(); + bits.write_bits(15, 4); + bits.write_bits(0, 4); + bits.write_bits(15, 4); + bits.write_bits(15, 4); + bits.write_bits(15, 4); + bits.write_bits(1, 4); + let mut bits = BitReader::from_bytes(&bits.finish()); + + let lengths = Unpack29::read_level_lengths(&mut bits).unwrap(); + + assert_eq!(lengths[0], 15); + assert_eq!(lengths[1..], [0; 19]); +} + +#[test] +fn unused_auxiliary_huffman_tables_may_be_empty() { + let mut levels = [0; TABLE_COUNT]; + levels[b'A' as usize] = 1; + levels[256] = 1; + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&encoded_table_description(&levels)); + + decoder.read_tables().unwrap(); + + assert!(!decoder.main.state.is_empty()); + assert!(decoder.offsets.state.is_empty()); + assert!(decoder.low_offsets.state.is_empty()); + assert!(decoder.lengths.state.is_empty()); + assert_eq!( + decoder.main.state.len() + + decoder.offsets.state.len() + + decoder.low_offsets.state.len() + + decoder.lengths.state.len(), + 2 + ); + assert_eq!( + MAIN_COUNT + OFFSET_COUNT + LOW_OFFSET_COUNT + LENGTH_COUNT, + TABLE_COUNT + ); +} + +#[test] +fn literal_encoder_round_trips_rar29_lz_blocks() { + let input = b"literal-only RAR 2.9 baseline\nwith repeated text literal-only\n"; + let packed = unpack29_encode_literals(input).unwrap(); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn an_lz_member_ends_with_a_new_file_marker() { + let input = b"RAR 2.9 terminator check, with repeated text to force a match: \ +RAR 2.9 terminator check\n"; + let packed = unpack29_encode_literals(input).unwrap(); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn rejects_a_final_lz_marker_that_promises_a_missing_table() { + let input = b"RAR 2.9 missing final table check\n".repeat(8); + let mut packed = unpack29_encode_literals(&input).unwrap(); + let last_nonzero = packed.iter().rposition(|&byte| byte != 0).unwrap(); + let final_one = 1 << packed[last_nonzero].trailing_zeros(); + let preceding_bit = final_one << 1; + assert_eq!(packed[last_nonzero] & preceding_bit, 0); + packed[last_nonzero] ^= final_one | preceding_bit; + + assert!(matches!( + unpack29_decode(&packed, input.len()), + Err(Error::InvalidData("RAR 2.9 bitstream is truncated")) + )); +} + +/// A member split across LZ blocks marks every block but the last as having +/// another table after it. +#[test] +fn every_block_but_the_last_says_another_table_follows() { + let input = b"rar29 multi block terminator check with repeated filler text\n".repeat(400); + let packed = super::encode_member_with_options( + &input, + &[], + EncodeOptions::new(96).with_block_size(4096), + ) + .unwrap(); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn rejects_a_new_file_marker_before_the_declared_member_size() { + let first = b"first block ends too soon\n".repeat(64); + let second = b"second block must not be decoded as the same member\n".repeat(64); + let mut levels = [0; TABLE_COUNT]; + let mut packed = super::encode_member_inner( + &first, + &[], + &[], + EncodeOptions::new(96), + false, + &mut levels, + None, + ) + .unwrap(); + packed.extend_from_slice( + &super::encode_member_inner( + &second, + &first, + &[], + EncodeOptions::new(96), + false, + &mut levels, + None, + ) + .unwrap(), + ); + + assert!(matches!( + unpack29_decode(&packed, first.len() + second.len()), + Err(Error::InvalidData( + "RAR 2.9 member ended before its declared size" + )) + )); +} + +#[test] +fn rejects_an_lz_literal_after_the_declared_member_size() { + let input = b"literal-only RAR 2.9 member"; + let packed = unpack29_encode_literals(input).unwrap(); + + assert!(matches!( + unpack29_decode(&packed, input.len() - 1), + Err(Error::InvalidData("RAR 2.9 LZ member has trailing data")) + )); +} + +#[test] +fn rejects_an_lz_match_crossing_the_declared_member_size() { + let input = b"repeated tail ".repeat(256); + let packed = Unpack29Encoder::new().encode_member(&input).unwrap(); + + assert!(matches!( + unpack29_decode(&packed, input.len() - 1), + Err(Error::InvalidData( + "RAR 2.9 member produces more output than its declared size" + )) + )); +} + +#[test] +fn multi_block_lz_encoding_round_trips_large_repeated_documents() { + let seed = b"\n\ +

RAR29 repeated document body with enough structured text to \ +exercise LZSS block table selection.

\n" + .repeat(96); + let input = seed.repeat(180); + let single = super::encode_member_with_options(&input, &[], EncodeOptions::new(96)).unwrap(); + let blocked = super::encode_member_with_options( + &input, + &[], + EncodeOptions::new(96).with_block_size(1024 * 1024), + ) + .unwrap(); + + assert_eq!(unpack29_decode(&single, input.len()).unwrap(), input); + assert_eq!(unpack29_decode(&blocked, input.len()).unwrap(), input); + assert!(blocked.len() < input.len()); +} + +#[test] +fn matched_member_streams_across_flush_and_history_boundaries() { + struct RecordingSink { + data: Vec, + writes: Vec, + } + + impl std::io::Write for RecordingSink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + self.writes.push(data.len()); + self.data.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + // After the first literal, runs are encoded as 258-byte matches. The + // match starting at 1,048,513 therefore crosses the first 1 MiB flush + // boundary and has to be resumed by the next decode batch. + let input = vec![b'Z'; MAX_HISTORY + STREAM_CHUNK + 513]; + let mut encoder = Unpack29Encoder::new(); + let packed = encoder.encode_member(&input).unwrap(); + assert_eq!(encoder.history.len(), MAX_HISTORY); + + let mut decoder = Unpack29::new(); + let mut sink = RecordingSink { + data: Vec::new(), + writes: Vec::new(), + }; + decoder + .decode_member_to(&packed, input.len(), &mut sink) + .unwrap(); + + assert_eq!(sink.data, input); + assert_eq!( + sink.writes, + [STREAM_CHUNK; 5] + .into_iter() + .chain([513]) + .collect::>() + ); + assert_eq!(decoder.base_offset, input.len() - MAX_HISTORY); + assert_eq!(decoder.output.len(), MAX_HISTORY); + assert!(decoder.pending_match.is_none()); +} + +#[test] +fn solid_follower_can_match_the_oldest_retained_history() { + let mut state = 0x6d2b_79f5u32; + let marker: Vec = (0..MAX_ENCODER_MATCH_LENGTH) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let mut first = vec![b'X'; 17]; + first.extend_from_slice(&marker); + first.resize(MAX_HISTORY + 17, b'Z'); + + let options = EncodeOptions::default().with_max_match_distance(MAX_HISTORY); + let mut encoder = Unpack29Encoder::with_options(options); + let first_packed = encoder.encode_member(&first).unwrap(); + assert_eq!(encoder.history.len(), MAX_HISTORY); + assert_eq!(&encoder.history[..marker.len()], marker); + + let follower_tokens = encode_tokens(&marker, &encoder.history, options); + assert!(follower_tokens.iter().any(|token| matches!( + token, + EncodeToken::Match { offset, .. } if *offset == MAX_HISTORY + ))); + let follower_packed = encoder.encode_member(&marker).unwrap(); + + let mut decoder = Unpack29::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!(decoder.base_offset, 17); + assert_eq!( + decoder + .decode_member(&follower_packed, marker.len()) + .unwrap(), + marker + ); + assert_eq!(decoder.output.len(), MAX_HISTORY); + assert_eq!(decoder.base_offset, 17 + marker.len()); +} + +#[test] +fn block_encoder_trims_local_history_at_the_dictionary_boundary() { + let history = vec![b'Z'; MAX_HISTORY]; + let options = EncodeOptions::new(0).with_block_size(1); + let mut encoder = Unpack29Encoder::with_options(options); + encoder.history.clone_from(&history); + let packed = encoder.encode_member(b"AB").unwrap(); + + let mut decoder = Unpack29::new(); + decoder.output = history.into(); + assert_eq!(decoder.decode_member(&packed, 2).unwrap(), b"AB"); + assert_eq!(encoder.history.len(), MAX_HISTORY); + assert_eq!(&encoder.history[MAX_HISTORY - 2..], b"AB"); + assert_eq!(&decoder.output[MAX_HISTORY - 2..], b"AB"); +} + +#[test] +fn filter_spanning_a_flush_waits_for_complete_input_and_is_retired() { + struct RecordingSink { + data: Vec, + writes: Vec, + } + + impl std::io::Write for RecordingSink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + self.writes.push(data.len()); + self.data.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let filter_range = STREAM_CHUNK - 512..STREAM_CHUNK + 512; + let mut input = vec![b'Z'; MAX_HISTORY + STREAM_CHUNK + 513]; + for position in (filter_range.start..filter_range.end - 4).step_by(16) { + input[position] = 0xe8; + input[position + 1..position + 5].copy_from_slice(&0x1234u32.to_le_bytes()); + } + // Literal coding stops exactly at the streaming target. Match coding + // can legally overshoot it and happen to complete this small filter in + // the same decode pass, which would not exercise the wait. + let packed = Unpack29Encoder::with_options(EncodeOptions::new(0)) + .encode_member_with_filter( + &input, + crate::FilterSpec::range(crate::FilterKind::E8, filter_range.clone()), + ) + .unwrap(); + + let mut decoder = Unpack29::new(); + let mut sink = RecordingSink { + data: Vec::new(), + writes: Vec::new(), + }; + decoder + .decode_member_to(&packed, input.len(), &mut sink) + .unwrap(); + + assert_eq!(sink.data, input); + assert_eq!(sink.writes[0], filter_range.start); + assert_eq!(sink.writes[1], STREAM_CHUNK + 512); + assert!(decoder.filters.is_empty()); + + let mut decoder = Unpack29::new(); + let mut prematurely_emitted = Vec::new(); + assert_eq!( + decoder + .decode_member_to(&packed, filter_range.end - 1, &mut prematurely_emitted) + .unwrap_err(), + Error::InvalidData("RAR 2.9 VM filter extends beyond output") + ); + assert!(prematurely_emitted.is_empty()); +} + +#[test] +fn a_future_filter_stays_scheduled_until_its_range_is_published() { + let mut decoder = Unpack29::new(); + decoder.output.resize(128, 0).unwrap(); + decoder + .programs + .push(VmProgram { + kind: VmProgramKind::Standard(StandardFilter::E8), + block_size: 8, + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 64, + size: 8, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + + assert_eq!(decoder.safe_flush_end(0, 32, 128).unwrap(), 32); + assert_eq!(decoder.filtered_range(0, 32, 0).unwrap(), vec![0; 32]); + assert_eq!(decoder.filters.len(), 1); + + assert_eq!(decoder.filtered_range(32, 72, 0).unwrap(), vec![0; 40]); + assert!(decoder.filters.is_empty()); +} + +#[test] +fn history_trimming_discards_stale_filters_and_keeps_future_filters() { + let mut decoder = Unpack29::new(); + decoder.output.resize(MAX_HISTORY + 64, 0).unwrap(); + decoder + .programs + .push(VmProgram { + kind: VmProgramKind::Standard(StandardFilter::E8), + block_size: 8, + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 0, + size: 8, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 128, + size: 8, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + + decoder.trim_history(MAX_HISTORY + 64, MAX_HISTORY + 64); + + assert_eq!(decoder.base_offset, 64); + assert_eq!(decoder.output.len(), MAX_HISTORY); + assert_eq!(decoder.filters.len(), 1); + assert_eq!(decoder.filters[0].start, 128); + assert_eq!(decoder.filtered_range(64, 136, 0).unwrap(), vec![0; 72]); + assert!(decoder.filters.is_empty()); +} + +#[test] +fn solid_ppmd_match_rejects_distance_beyond_retained_history() { + let input = vec![b'Z'; MAX_HISTORY + 64]; + let packed = unpack29_encode_literals(&input).unwrap(); + let mut decoder = Unpack29::new(); + decoder + .decode_non_solid_member_to(&packed, input.len(), &mut std::io::sink()) + .unwrap(); + assert_eq!(decoder.base_offset, 64); + + let mut encoder = PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, 1).unwrap(); + encoder.encode_match(MAX_HISTORY + 1, 32).unwrap(); + let (body, _) = encoder.finish_keeping_model().unwrap(); + let mut packed = vec![0x80 | 0x20 | (PPMD_ORDER as u8 - 1), 0]; + packed.extend_from_slice(&body); + assert_eq!( + decoder.decode_member_to(&packed, 32, &mut std::io::sink()), + Err(Error::InvalidData("RAR 2.9 match distance is out of range")) + ); +} + +#[test] +fn ppmd_member_rejects_a_vm_filter_range_beyond_native_output() { + let record = super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: u32::MAX as usize, + init_regs: &[], + code: super::RAR3_E8_FILTER_BYTECODE, + global_data: &[], + }, + 0, + true, + ) + .unwrap(); + let mut encoder = PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, 1).unwrap(); + encoder.encode_literal(b'A').unwrap(); + encoder.encode_vm_filter_record(&record).unwrap(); + encoder.encode_literal(b'B').unwrap(); + let (body, _) = encoder.finish_keeping_model().unwrap(); + let mut packed = vec![0x80 | 0x20 | (PPMD_ORDER as u8 - 1), 0]; + packed.extend_from_slice(&body); + let message = if cfg!(target_pointer_width = "32") { + "RAR 2.9 VM filter size overflows" + } else { + "RAR 2.9 VM filter extends beyond output" + }; + assert_eq!( + unpack29_decode(&packed, 2), + Err(Error::InvalidData(message)) + ); +} + +#[test] +fn stale_filter_ranges_do_not_change_later_published_bytes() { + let mut decoder = Unpack29::new(); + decoder.output.resize(32, 0x5a).unwrap(); + decoder + .filters + .push(VmFilter { + program: usize::MAX, + start: 0, + size: 8, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + + assert_eq!(decoder.safe_flush_end(16, 32, 32).unwrap(), 32); + assert_eq!(decoder.filtered_range(16, 32, 0).unwrap(), vec![0x5a; 16]); +} + +#[test] +fn table_level_encoder_uses_rar29_run_symbols() { + let mut lengths = [0u8; TABLE_COUNT]; + lengths[..4].fill(5); + lengths[8..21].fill(0); + + let tokens = encode_table_level_tokens(&lengths); + + assert!(tokens.contains(&LevelToken::repeat_previous_short(3))); + assert!(tokens.iter().any(|token| token.symbol == 19)); +} + +#[test] +fn lazy_lz_parser_defers_short_match_for_longer_next_match() { + let input = b"abcdXbcdYYYYYYYYYYYYabcdYYYYYYYYYYYY"; + let greedy = encode_tokens(input, &[], EncodeOptions::new(MAX_MATCH_CANDIDATES)); + let lazy = encode_tokens( + input, + &[], + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_lazy_matching(true), + ); + let packed = Unpack29Encoder::with_options( + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_lazy_matching(true), + ) + .encode_member(input) + .unwrap(); + + assert!(greedy + .iter() + .any(|token| matches!(token, EncodeToken::Match { length: 4, .. }))); + assert!(lazy + .iter() + .any(|token| matches!(token, EncodeToken::Match { length, .. } if *length > 8))); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn lazy_lz_parser_uses_match_cost_not_only_match_length() { + let pos = 300_000usize; + let mut input = vec![0u8; pos + 16]; + input[100..106].copy_from_slice(b"BCDEFG"); + input[106] = b'!'; + input[pos - 10..pos - 5].copy_from_slice(b"ABCD!"); + input[pos..pos + 7].copy_from_slice(b"ABCDEFG"); + let mut finder = Rar29MatchFinder::new(input.len()); + finder.insert(&input, 100); + finder.insert(&input, pos - 10); + + let current = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::new(MAX_MATCH_CANDIDATES), + &EncoderMatchState::default(), + ) + .unwrap(); + let next = best_match( + &input, + pos + 1, + input.len(), + &finder, + EncodeOptions::new(MAX_MATCH_CANDIDATES), + &EncoderMatchState::default(), + ) + .unwrap(); + + assert_eq!(current.length, 4); + assert_eq!(current.offset, 10); + assert_eq!(next.length, 6); + assert!(next.offset > 0x40000); + assert!(!should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_lazy_matching(true), + &EncoderMatchState::default(), + current, + )); +} + +#[test] +fn lazy_lz_parser_uses_bounded_cost_lookahead() { + let pos = 160; + let mut input: Vec = (0..240u16) + .map(|value| value.wrapping_mul(91) as u8) + .collect(); + input[pos - 30..pos - 22].copy_from_slice(b"ABCDEFGH"); + input[pos - 80..pos - 64].copy_from_slice(b"CDEFGHIJKLMNOPQR"); + input[pos..pos + 18].copy_from_slice(b"ABCDEFGHIJKLMNOPQR"); + + let mut finder = Rar29MatchFinder::new(input.len()); + for candidate in 0..pos { + finder.insert(&input, candidate); + } + let current = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &EncoderMatchState::default(), + ) + .unwrap(); + + assert_eq!((current.length, current.offset), (8, 30)); + assert!(!should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(1), + &EncoderMatchState::default(), + current, + )); + assert!(should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(2), + &EncoderMatchState::default(), + current, + )); +} + +#[test] +fn lazy_lz_parser_stops_lookahead_at_member_end() { + let input = b"aaaaaa"; + let mut finder = Rar29MatchFinder::new(input.len()); + finder.insert(input, 0); + let state = EncoderMatchState::default(); + let options = EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(16); + let current = best_match(input, 2, input.len(), &finder, options, &state).unwrap(); + + assert_eq!((current.length, current.offset), (4, 2)); + assert!(!should_lazy_emit_literal( + input, 2, &finder, options, &state, current, + )); +} + +#[test] +fn match_state_encodes_last_length_and_repeat_offset_symbols() { + let mut state = EncoderMatchState::default(); + assert!(matches!( + state.encode_match(12, 64).unwrap(), + super::EncodedMatch::Fresh { .. } + )); + state.remember(12, 64); + + assert_eq!( + state.encode_match(12, 64).unwrap(), + super::EncodedMatch::LastLengthRepeat + ); + assert!(matches!( + state.encode_match(9, 64).unwrap(), + super::EncodedMatch::RepeatOffset { index: 0, .. } + )); + + assert!(matches!( + state.encode_match(MAX_ENCODER_MATCH_LENGTH, 64).unwrap(), + super::EncodedMatch::Fresh { .. } + )); + state.remember(MAX_ENCODER_MATCH_LENGTH, 64); + assert_eq!(state.old_offsets, [64, 64, 0, 0]); +} + +#[test] +fn match_field_encoders_reject_values_outside_the_wire_ranges() { + assert_eq!( + super::length_slot_for_match(2), + Err(Error::InvalidData("RAR 2.9 match length is too short")) + ); + assert_eq!( + super::length_slot_for_match(MAX_ENCODER_MATCH_LENGTH + 1), + Err(Error::InvalidData("RAR 2.9 match length is too long")) + ); + assert_eq!( + super::length_slot_for_repeat_match(1), + Err(Error::InvalidData( + "RAR 2.9 repeat match length is too short" + )) + ); + assert_eq!( + super::length_slot_for_repeat_match(MAX_ENCODER_MATCH_LENGTH), + Err(Error::InvalidData( + "RAR 2.9 repeat match length is too long" + )) + ); + assert_eq!( + super::offset_slot_for_match(0), + Err(Error::InvalidData("RAR 2.9 match offset is zero")) + ); + let last_slot = OFFSET_COUNT - 1; + let largest_offset = + super::OFFSET_BASES[last_slot] + (1usize << super::OFFSET_BITS[last_slot]) - 1 + 1; + assert_eq!( + super::offset_slot_for_match(largest_offset + 1), + Err(Error::InvalidData("RAR 2.9 match offset is too large")) + ); + let state = EncoderMatchState::default(); + for (length, offset, message) in [ + (0, 0x40000, "RAR 2.9 adjusted match length underflows"), + (4, 0, "RAR 2.9 match offset is zero"), + (5, largest_offset + 1, "RAR 2.9 match offset is too large"), + ] { + assert_eq!( + state.encode_match(length, offset), + Err(Error::InvalidData(message)) + ); + assert_eq!( + super::estimated_match_cost(&state, length, offset), + Err(Error::InvalidData(message)) + ); + let mut candidate = None; + super::consider_match_candidate(&mut candidate, &state, length, offset); + assert_eq!(candidate, None); + } +} + +#[test] +fn match_length_and_offset_slots_have_no_gaps() { + for (bases, bits) in [ + (&super::LENGTH_BASES[..], &super::LENGTH_BITS[..]), + (&super::OFFSET_BASES[..], &super::OFFSET_BITS[..]), + ] { + for index in 1..bases.len() { + assert_eq!(bases[index], bases[index - 1] + (1usize << bits[index - 1])); + } + } +} + +#[test] +fn match_search_respects_fresh_distance_length_adjustments() { + for distance in [0x1fff, 0x2000, 0x3ffff, 0x40000] { + let mut input = vec![0xff; distance + 4]; + input[..4].copy_from_slice(b"ABCD"); + input[distance..].copy_from_slice(b"ABCD"); + let mut finder = Rar29MatchFinder::new(input.len()); + finder.insert(&input, 0); + let options = EncodeOptions::default(); + let fresh = best_match( + &input, + distance, + input.len(), + &finder, + options, + &EncoderMatchState::default(), + ); + if distance == 0x40000 { + assert_eq!(fresh, None); + } else { + let candidate = fresh.unwrap(); + assert_eq!((candidate.length, candidate.offset), (4, distance)); + } + + // A prior legal five-byte fresh match establishes this distance. + // Repeat-distance lengths do not receive fresh-distance additions. + let mut state = EncoderMatchState::default(); + assert!(matches!( + state.encode_match(5, distance).unwrap(), + super::EncodedMatch::Fresh { .. } + )); + state.remember(5, distance); + let repeated = best_match(&input, distance, input.len(), &finder, options, &state).unwrap(); + assert_eq!((repeated.length, repeated.offset), (4, distance)); + assert!(matches!( + state + .encode_match(repeated.length, repeated.offset) + .unwrap(), + super::EncodedMatch::RepeatOffset { index: 0, .. } + )); + } +} + +#[test] +fn match_search_ignores_a_remembered_offset_outside_its_window() { + let input = b"abcdefghijklmnop"; + let finder = Rar29MatchFinder::new(input.len()); + let mut state = EncoderMatchState::default(); + state.old_offsets[0] = 9; + let options = EncodeOptions::default().with_max_match_distance(8); + + assert_eq!( + best_match(input, 8, input.len(), &finder, options, &state), + None + ); +} + +#[test] +fn cost_aware_match_selection_prefers_repeat_offset_token() { + let pos = 600usize; + let mut input: Vec = (0..pos + 16) + .map(|index| (index as u8).wrapping_mul(37)) + .collect(); + input[pos - 30..pos - 22].copy_from_slice(b"ABCDEFGH"); + input[pos - 512..pos - 503].copy_from_slice(b"ABCDEFGHI"); + input[pos..pos + 9].copy_from_slice(b"ABCDEFGHI"); + input[pos - 22] = 0x11; + input[pos - 503] = 0x22; + input[pos + 9] = 0x33; + let mut finder = Rar29MatchFinder::new(input.len()); + finder.insert(&input, pos - 30); + finder.insert(&input, pos - 512); + + let fresh = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &EncoderMatchState::default(), + ) + .unwrap(); + let repeat = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &EncoderMatchState { + old_offsets: [30, 0, 0, 0], + last_offset: 0, + last_length: 0, + }, + ) + .unwrap(); + + assert_eq!((fresh.length, fresh.offset), (9, 512)); + assert_eq!((repeat.length, repeat.offset), (8, 30)); +} + +#[test] +fn match_finder_respects_configured_maximum_distance() { + let phrase = b"rar29 bounded dictionary phrase"; + let mut input = Vec::new(); + input.extend_from_slice(phrase); + input.extend(std::iter::repeat_n(0u8, 256 * 1024)); + input.extend_from_slice(phrase); + + let bounded = encode_tokens( + &input, + &[], + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_max_match_distance(128 * 1024), + ); + let unbounded = encode_tokens( + &input, + &[], + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_max_match_distance(1024 * 1024), + ); + + assert!(!bounded + .iter() + .any(|token| matches!(token, EncodeToken::Match { offset, .. } if *offset > 128 * 1024))); + assert!(unbounded + .iter() + .any(|token| matches!(token, EncodeToken::Match { offset, .. } if *offset > 128 * 1024))); +} + +#[test] +fn encode_options_cap_match_distance_at_the_rar29_window() { + assert_eq!( + EncodeOptions::default() + .with_max_match_distance(usize::MAX) + .max_match_distance, + MAX_HISTORY + ); + + let options = EncodeOptions { + max_match_distance: usize::MAX, + ..EncodeOptions::default() + }; + assert_eq!( + Unpack29Encoder::with_options(options) + .options + .max_match_distance, + MAX_HISTORY + ); +} + +#[test] +fn public_encoders_handle_zero_match_distance_and_one_byte_blocks() { + let input = b"small RAR29 blocks"; + let options = EncodeOptions::default() + .with_max_match_distance(0) + .with_block_size(1); + let packed = Unpack29Encoder::with_options(options) + .encode_member(input) + .unwrap(); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); + + let packed = Unpack29Encoder::with_options(options) + .encode_member_with_filter(input, crate::FilterSpec::whole(crate::FilterKind::E8)) + .unwrap(); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn solid_member_rejects_declared_output_beyond_native_range() { + let mut decoder = super::Unpack29::new(); + decoder + .decode_non_solid_member(COMPRESSED_TEXT, 2400) + .unwrap(); + assert_eq!( + decoder.decode_member(&[], usize::MAX), + Err(Error::InvalidData("RAR 2.9 output size overflows")) + ); +} + +#[test] +fn ppmd_match_finder_uses_the_declared_dictionary_past_one_megabyte() { + let distance = MAX_ENCODER_MATCH_OFFSET + 4096; + let phrase = b"RAR29 PPMd match beyond the old one-megabyte ceiling"; + let mut input = vec![0u8; distance + phrase.len()]; + input[..phrase.len()].copy_from_slice(phrase); + input[distance..].copy_from_slice(phrase); + let mut finder = Rar29MatchFinder::new(input.len()); + finder.insert(&input, 0); + + assert_eq!( + best_ppmd_match(&input, distance, &finder, MAX_HISTORY), + Some((phrase.len(), distance)) + ); +} + +#[test] +fn lz_encoder_uses_weighted_rar29_huffman_tables() { + let mut input = Vec::new(); + for byte in 0u8..120 { + input.push(b'A'); + input.push(byte); + } + let packed = Unpack29Encoder::new().encode_member(&input).unwrap(); + let mut decoder = Unpack29::new(); + decoder.bits.append(&packed); + decoder.read_tables().unwrap(); + let main_lengths = &decoder.levels[..MAIN_COUNT]; + let nonzero_lengths = main_lengths + .iter() + .copied() + .filter(|&length| length != 0) + .collect::>(); + + assert!(nonzero_lengths.len() > 1); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn copy_match_zero_fills_an_offset_that_reaches_past_the_stream() { + let mut decoder = Unpack29::new(); + decoder.output.extend_from_slice(b"AB").unwrap(); + + decoder.copy_match(4, 9, 6).unwrap(); + + assert_eq!(&decoder.output[..], b"AB\0\0\0\0"); +} + +#[test] +fn an_undefined_repeat_distance_zero_fills_like_reference_readers() { + let mut decoder = Unpack29::new(); + let mut main_lengths = vec![0; MAIN_COUNT]; + main_lengths[b'Z' as usize] = 1; + main_lengths[259] = 1; + decoder.main = Huffman::from_lengths(&main_lengths).unwrap(); + let mut repeat_lengths = vec![0; LENGTH_COUNT]; + repeat_lengths[2] = 1; + decoder.lengths = Huffman::from_lengths(&repeat_lengths).unwrap(); + + let main_codes = canonical_codes(&main_lengths); + let repeat_codes = canonical_codes(&repeat_lengths); + let mut bits = BitWriter::default(); + for code in [ + main_codes[b'Z' as usize].unwrap(), + main_codes[259].unwrap(), + repeat_codes[2].unwrap(), + ] { + bits.write_bits(u32::from(code.code), code.len); + } + decoder.bits = BitReader::from_bytes(&bits.finish()); + + decoder.decode_lz(5).unwrap(); + + assert_eq!(&decoder.output[..], b"Z\0\0\0\0"); +} + +#[test] +fn an_undefined_last_match_repeat_is_a_noop_like_reference_readers() { + let mut decoder = Unpack29::new(); + let mut main_lengths = vec![0; MAIN_COUNT]; + main_lengths[258] = 1; + main_lengths[b'X' as usize] = 2; + main_lengths[b'Y' as usize] = 2; + decoder.main = Huffman::from_lengths(&main_lengths).unwrap(); + + let main_codes = canonical_codes(&main_lengths); + let mut bits = BitWriter::default(); + for symbol in [b'X' as usize, 258, b'Y' as usize] { + let code = main_codes[symbol].unwrap(); + bits.write_bits(u32::from(code.code), code.len); + } + decoder.bits = BitReader::from_bytes(&bits.finish()); + + decoder.decode_lz(2).unwrap(); + + assert_eq!(&decoder.output[..], b"XY"); + assert_eq!(decoder.last_length, 0); +} + +#[test] +fn ppmd_literal_encoder_round_trips_rar29_ppmd_blocks() { + let mut input = b"rar29 ppmd literal text payload alpha beta gamma\n".repeat(64); + input.extend_from_slice(&[2, 2, 2, b'e', b's', b'c']); + let packed = unpack29_encode_ppmd_literals(&input).unwrap(); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); + assert_ne!(packed.first().copied(), Some(0)); +} + +#[test] +fn ppmd_end_block_command_reads_the_next_block() { + let first = b"first PPMd block "; + let second = b"and its continuation"; + let mut packed = vec![ + 0x80 | 0x20 | ((PPMD_ORDER as u8) - 1), + PPMD_DICTIONARY_MB - 1, + ]; + let mut encoder = + PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, usize::from(PPMD_DICTIONARY_MB)).unwrap(); + for &byte in first { + encoder.encode_literal(byte).unwrap(); + } + let (block, model) = encoder.finish_block_keeping_model().unwrap(); + packed.extend_from_slice(&block); + + packed.push(0x80 | ((PPMD_ORDER as u8) - 1)); + let mut encoder = PpmdEncoder::continuing(model, PPMD_ESC); + for &byte in second { + encoder.encode_literal(byte).unwrap(); + } + let (block, _) = encoder.finish_keeping_model().unwrap(); + packed.extend_from_slice(&block); + + let expected = [first.as_slice(), second.as_slice()].concat(); + assert_eq!(unpack29_decode(&packed, expected.len()).unwrap(), expected); +} + +#[test] +fn ppmd_member_can_end_in_an_empty_following_block() { + let input = b"PPMd output ends before its final empty block"; + let mut packed = vec![ + 0x80 | 0x20 | ((PPMD_ORDER as u8) - 1), + PPMD_DICTIONARY_MB - 1, + ]; + let mut encoder = + PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, usize::from(PPMD_DICTIONARY_MB)).unwrap(); + for &byte in input { + encoder.encode_literal(byte).unwrap(); + } + let (block, model) = encoder.finish_block_keeping_model().unwrap(); + packed.extend_from_slice(&block); + + packed.push(0x80 | ((PPMD_ORDER as u8) - 1)); + let encoder = PpmdEncoder::continuing(model, PPMD_ESC); + let (block, _) = encoder.finish_keeping_model().unwrap(); + packed.extend_from_slice(&block); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn lz_member_can_end_in_an_empty_following_block() { + let input = b"LZ output ends before its final empty block"; + let mut levels = [0; TABLE_COUNT]; + let options = EncodeOptions::default(); + let mut packed = + super::encode_member_inner(input, &[], &[], options, true, &mut levels, None).unwrap(); + packed.extend_from_slice( + &super::encode_member_inner(&[], input, &[], options, false, &mut levels, None).unwrap(), + ); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn rejects_ppmd_eof_before_the_declared_member_size() { + let input = b"PPMd member with an inflated declared size"; + let packed = unpack29_encode_ppmd_literals(input).unwrap(); + + assert!(matches!( + unpack29_decode(&packed, input.len() + 1), + Err(Error::InvalidData( + "RAR 2.9 member ended before its declared size" + )) + )); +} + +#[test] +fn rejects_ppmd_output_after_the_declared_member_size() { + let input = b"PPMd member with a shortened declared size"; + let packed = unpack29_encode_ppmd_literals(input).unwrap(); + + assert!(matches!( + unpack29_decode(&packed, input.len() - 1), + Err(Error::InvalidData("RAR 2.9 PPMd member has trailing data")) + )); +} + +#[test] +fn rejects_a_truncated_ppmd_end_marker() { + let input = b"PPMd member whose final range-coder byte is missing"; + let mut packed = unpack29_encode_ppmd_literals(input).unwrap(); + packed.pop(); + + assert!(matches!( + unpack29_decode(&packed, input.len()), + Err(Error::InvalidData("RAR 2.9 bitstream is truncated")) + )); +} + +#[test] +fn ppmd_model_exhaustion_is_corruption() { + assert!(matches!( + super::require_ppmd_symbol(None), + Err(Error::InvalidData("RAR 2.9 PPMd model is corrupt")) + )); +} + +#[test] +fn rejects_a_reserved_ppmd_command() { + let input = b"PPMd stream followed by a reserved command"; + let mut packed = vec![ + 0x80 | 0x20 | ((PPMD_ORDER as u8) - 1), + PPMD_DICTIONARY_MB - 1, + ]; + let mut encoder = + PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, usize::from(PPMD_DICTIONARY_MB)).unwrap(); + for &byte in input { + encoder.encode_literal(byte).unwrap(); + } + packed.extend_from_slice(&encoder.finish_with_command(6).unwrap()); + + assert!(matches!( + unpack29_decode(&packed, input.len()), + Err(Error::InvalidData("RAR 2.9 PPMd member has trailing data")) + )); + assert!(matches!( + unpack29_decode(&packed, input.len() + 1), + Err(Error::InvalidData("RAR 2.9 PPMd command is invalid")) + )); +} + +fn incomplete_ppmd_command(command: u8, parameters: &[u8]) -> Result> { + let mut packed = vec![ + 0x80 | 0x20 | ((PPMD_ORDER as u8) - 1), + PPMD_DICTIONARY_MB - 1, + ]; + let encoder = PpmdEncoder::new(PPMD_ORDER, PPMD_ESC, usize::from(PPMD_DICTIONARY_MB)).unwrap(); + packed.extend_from_slice( + &encoder + .finish_with_command_prefix(command, parameters) + .unwrap(), + ); + unpack29_decode(&packed, 1) +} + +#[test] +fn rejects_truncated_ppmd_match_parameters() { + let cases = [ + (&[][..], "first offset byte"), + (&[0][..], "second offset byte"), + (&[0, 0][..], "third offset byte"), + (&[0, 0, 0][..], "length byte"), + ]; + for (parameters, missing) in cases { + assert!( + incomplete_ppmd_command(4, parameters).is_err(), + "accepted a match missing its {missing}" + ); + } +} + +#[test] +fn rejects_a_truncated_ppmd_repeat_parameter() { + assert!(matches!( + incomplete_ppmd_command(5, &[]), + Err(Error::InvalidData("RAR 2.9 bitstream is truncated")) + )); +} + +#[test] +fn rejects_truncated_ppmd_vm_parameters() { + let cases = [ + (&[][..], "first record byte"), + (&[6][..], "one-byte extended length"), + (&[7][..], "two-byte length high byte"), + (&[7, 0][..], "two-byte length low byte"), + (&[0][..], "one-byte record body"), + (&[6, 0][..], "extended-length record body"), + (&[7, 0, 1][..], "two-byte-length record body"), + ]; + for (parameters, missing) in cases { + assert!( + matches!( + incomplete_ppmd_command(3, parameters), + Err(Error::InvalidData("RAR 2.9 bitstream is truncated")) + ), + "accepted a VM command missing its {missing}" + ); + } +} + +#[test] +fn ppmd_encoder_advertises_period_compatible_model_for_external_decoders() { + let packed = + unpack29_encode_ppmd(b"rar29 ppmd dictionary header", MAX_ENCODER_MATCH_OFFSET).unwrap(); + + assert_eq!(packed[0], 0xa7); + assert_eq!(packed[1], 24); +} + +#[test] +fn ppmd_encoder_emits_offset_one_repeat_escapes() { + let input = b"seed " + .iter() + .copied() + .chain(std::iter::repeat_n(b'Z', 512)) + .collect::>(); + let tokens = hybrid_tokens(&input, MAX_ENCODER_MATCH_OFFSET); + let packed = unpack29_encode_ppmd(&input, MAX_ENCODER_MATCH_OFFSET).unwrap(); + + assert!(tokens + .iter() + .any(|token| matches!(token, PpmdEncodeToken::RepeatOffsetOne { length } if *length >= 4))); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +/// PPMd's escape-4 matches copy out of the same window the LZ decoder +/// keeps, so a match reaching further back than the dictionary the file +/// header declares lands on whatever the decoder still happens to hold. +/// Nothing bounded them, so every RAR 3.0 and 4.0 PPMd member of a large +/// enough file failed its checksum in unrar; RAR 2.9 escaped only because +/// it declares a dictionary eight times larger. +#[test] +fn ppmd_matches_stay_inside_the_declared_dictionary() { + let dictionary = 16 * 1024; + // A distinctive block, repeated once inside the dictionary and once + // well outside it. The near copy is the match the encoder should still + // take, and it is what stops this passing merely because the bound + // suppressed every match there was. + let block: Vec = (0..2048u32) + .map(|index| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + let mut input = block.clone(); + let filler = |input: &mut Vec, until: usize| { + while input.len() < until { + input.extend_from_slice( + format!("filler line {:06} for the gap\n", input.len()).as_bytes(), + ); + } + }; + filler(&mut input, dictionary / 2); + input.extend_from_slice(&block); + filler(&mut input, 3 * dictionary); + input.extend_from_slice(&block); + let tokens = hybrid_tokens(&input, dictionary); + + let furthest = tokens + .iter() + .filter_map(|token| match token { + PpmdEncodeToken::Match { offset, .. } => Some(*offset), + _ => None, + }) + .max() + .expect("the payload has to produce matches for this to mean anything"); + assert!( + furthest <= dictionary, + "a match reached {furthest} bytes back, past the {dictionary} byte dictionary" + ); +} + +#[test] +fn ppmd_encoder_emits_distance_match_escapes() { + let phrase = b"repeated phrase for rar29 ppmd distance escape 4 "; + let mut input = Vec::new(); + input.extend_from_slice(phrase); + input.extend_from_slice(b"middle bytes make the repeat distance greater than one "); + input.extend_from_slice(phrase); + input.extend_from_slice(phrase); + input.extend_from_slice(b"tail"); + let tokens = hybrid_tokens(&input, MAX_ENCODER_MATCH_OFFSET); + let packed = unpack29_encode_ppmd(&input, MAX_ENCODER_MATCH_OFFSET).unwrap(); + + assert!(tokens + .iter() + .any(|token| matches!(token, PpmdEncodeToken::Match { offset, length } if *offset > 1 && *length >= 32))); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn ppmd_distance_match_lengths_stay_period_decoder_compatible() { + let phrase = b"RAR PPMd LZSS conversion phrase\n"; + let mut input = Vec::new(); + for _ in 0..200 { + input.extend_from_slice(phrase); + } + let tokens = hybrid_tokens(&input, MAX_ENCODER_MATCH_OFFSET); + + assert!(tokens.iter().any( + |token| matches!(token, PpmdEncodeToken::Match { offset, length } if *offset > 1 && *length >= 32) + )); + assert!(!tokens + .iter() + .any(|token| matches!(token, PpmdEncodeToken::Match { length, .. } if *length > 255))); +} + +#[test] +fn ppmd_encoder_emits_embedded_vm_filter_escape() { + let input = b"\xe8\0\0\0\0rar29 ppmd embedded e8 filter payload\n".repeat(16); + let packed = unpack29_encode_ppmd_with_filter( + &input, + crate::FilterSpec::whole(crate::FilterKind::E8), + MAX_ENCODER_MATCH_OFFSET, + ) + .unwrap(); + let plain_ppmd = unpack29_encode_ppmd(&input, MAX_ENCODER_MATCH_OFFSET).unwrap(); + let filtered_lz = Unpack29Encoder::new() + .encode_member_with_filter(&input, crate::FilterSpec::whole(crate::FilterKind::E8)) + .unwrap(); + + assert!(packed.len() != plain_ppmd.len() || packed.len() != filtered_lz.len()); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn same_block_filters_chain_in_wire_order() { + let mut input = Vec::new(); + for index in 0..256u32 { + input.push(0xe8); + input.extend_from_slice(&index.wrapping_mul(97).to_le_bytes()); + input.extend_from_slice(b"chained-filter-payload"); + } + let filters = [ + crate::FilterSpec::whole(crate::FilterKind::Delta { channels: 1 }), + crate::FilterSpec::whole(crate::FilterKind::E8), + ]; + let packed = Unpack29Encoder::new() + .encode_member_with_filters(&input, &filters) + .unwrap(); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn archive_decoder_rejects_partially_overlapping_filters() { + let input = vec![b'Z'; 96]; + for (second_start, second_size) in [(32, 64), (0, 63)] { + let filters = [ + OwnedVmFilterRecord { + block_start: 0, + block_size: 64, + init_regs: vec![(0, 1)], + code: RAR3_DELTA_FILTER_BYTECODE, + global_data: Vec::new(), + }, + OwnedVmFilterRecord { + block_start: second_start, + block_size: second_size, + init_regs: Vec::new(), + code: super::RAR3_E8_FILTER_BYTECODE, + global_data: Vec::new(), + }, + ]; + let refs = filters.iter().collect::>(); + let records = encoded_filter_records_at(&refs, 0, usize::MAX, &mut Vec::new()).unwrap(); + let packed = super::encode_member_inner( + &input, + &[], + &records, + EncodeOptions::default(), + false, + &mut [0; TABLE_COUNT], + None, + ) + .unwrap(); + + assert_eq!( + unpack29_decode(&packed, input.len()).unwrap_err(), + Error::InvalidData("RAR 2.9 VM filters partially overlap") + ); + } +} + +#[test] +fn writer_rejects_partially_overlapping_filters() { + let input = vec![b'Z'; 96]; + for second_range in [32..96, 0..63] { + let filters = [ + crate::FilterSpec::range(crate::FilterKind::Delta { channels: 1 }, 0..64), + crate::FilterSpec::range(crate::FilterKind::E8, second_range), + ]; + + assert_eq!( + Unpack29Encoder::new() + .encode_member_with_filters(&input, &filters) + .unwrap_err(), + Error::InvalidData("RAR 2.9 VM filters partially overlap") + ); + } +} + +#[test] +fn public_encoders_reject_invalid_filter_ranges() { + let input = vec![b'Z'; 96]; + for (start, end) in [(32, 32), (64, 32), (0, 97)] { + let range = start..end; + let filter = crate::FilterSpec::range(crate::FilterKind::E8, range); + + assert_eq!( + Unpack29Encoder::new() + .encode_member_with_filter(&input, filter.clone()) + .unwrap_err(), + Error::InvalidData("RAR 2.9 VM filter range is invalid") + ); + assert_eq!( + unpack29_encode_ppmd_with_filter(&input, filter, MAX_ENCODER_MATCH_OFFSET).unwrap_err(), + Error::InvalidData("RAR 2.9 VM filter range is invalid") + ); + } +} + +#[test] +fn public_encoder_rejects_invalid_filter_parameters() { + let input = vec![b'Z'; 96]; + let cases = [ + ( + crate::FilterKind::Delta { channels: 0 }, + Error::InvalidData("RAR 2.9 VM filter channel count is invalid"), + ), + ( + crate::FilterKind::Delta { + channels: MAX_VM_DELTA_FILTER_BLOCK_SIZE + 1, + }, + Error::InvalidData("RAR 2.9 VM filter channel count is invalid"), + ), + ( + crate::FilterKind::Audio { channels: 0 }, + Error::InvalidData("RAR 2.9 VM filter channel count is invalid"), + ), + ( + crate::FilterKind::Audio { + channels: super::MAX_AUDIO_CHANNELS + 1, + }, + Error::InvalidData("RAR 2.9 VM filter channel count is invalid"), + ), + ( + crate::FilterKind::Rgb { width: 0, pos_r: 0 }, + Error::InvalidData("RAR 2.9 RGB filter scanline width is invalid"), + ), + ( + crate::FilterKind::Rgb { + width: MAX_VM_FILTER_BLOCK_SIZE + 1, + pos_r: 0, + }, + Error::InvalidData("RAR 2.9 RGB filter scanline width is invalid"), + ), + ( + crate::FilterKind::Rgb { width: 8, pos_r: 0 }, + Error::InvalidData("RAR 2.9 RGB filter parameters are invalid"), + ), + ( + crate::FilterKind::Rgb { + width: 12, + pos_r: 3, + }, + Error::InvalidData("RAR 2.9 RGB filter parameters are invalid"), + ), + ]; + + for (kind, expected) in cases { + assert_eq!( + Unpack29Encoder::new() + .encode_member_with_filter(&input, crate::FilterSpec::whole(kind)) + .unwrap_err(), + expected, + "accepted {kind:?}" + ); + } + + assert_eq!( + Unpack29Encoder::new() + .encode_member_with_filter( + &input, + crate::FilterSpec::whole(crate::FilterKind::Delta { channels: 33 }), + ) + .unwrap_err(), + Error::InvalidData("RAR 2.9 DELTA filter channel count is invalid") + ); +} + +#[test] +fn filtered_entry_point_accepts_empty_input_without_filters() { + let packed = Unpack29Encoder::new() + .encode_member_with_filters(b"", &[]) + .unwrap(); + + assert!(unpack29_decode(&packed, 0).unwrap().is_empty()); +} + +#[test] +fn filtered_progress_polls_preprocessing_and_keeps_cancelled_state_transactional() { + let input = vec![b'Z'; MAX_VM_FILTER_BLOCK_SIZE * 2 + 16]; + let filter = crate::FilterSpec::whole(crate::FilterKind::E8); + let mut at_entry = Unpack29Encoder::new(); + assert_eq!( + at_entry.encode_member_with_filters_and_progress( + &input, + std::slice::from_ref(&filter), + Some(&mut |_| false), + ), + Err(Error::Cancelled) + ); + assert!(at_entry.history.is_empty()); + assert_eq!(at_entry.levels, [0; TABLE_COUNT]); + + let mut preprocessing = Unpack29Encoder::new(); + let mut polls = 0; + let result = preprocessing.encode_member_with_filters_and_progress( + &input, + std::slice::from_ref(&filter), + Some(&mut |_| { + polls += 1; + polls < 3 + }), + ); + assert_eq!(result.unwrap_err(), Error::Cancelled); + assert_eq!(polls, 3); + assert!(preprocessing.history.is_empty()); + assert_eq!(preprocessing.levels, [0; TABLE_COUNT]); + + let options = EncodeOptions::default().with_block_size(4096); + let mut between_blocks = Unpack29Encoder::with_options(options); + between_blocks.levels[0] = 7; + let original_levels = between_blocks.levels; + let result = between_blocks.encode_member_with_filters_and_progress( + &input[..8192], + &[filter], + Some(&mut |position| position <= 4096), + ); + assert_eq!(result.unwrap_err(), Error::Cancelled); + assert!(between_blocks.history.is_empty()); + assert_eq!(between_blocks.levels, original_levels); +} + +#[test] +fn solid_candidate_transitions_are_cancellable_without_committing_state() { + let seed = b"solid seed history and table state\n".repeat(200); + let input = b"candidate transition payload with repeated text\n".repeat(300); + let mut encoder = Unpack29Encoder::new(); + encoder.encode_member(&seed).unwrap(); + let original_history = encoder.history.clone(); + let original_levels = encoder.levels; + let filter = crate::FilterSpec::whole(crate::FilterKind::E8); + let candidates = [Vec::new(), vec![filter]]; + + let mut completions = 0; + let result = + encoder.encode_member_with_engine(&input, ChainEngine::Lz, &candidates, &mut |position| { + if position == input.len() { + completions += 1; + return completions < 3; + } + true + }); + assert_eq!(result.unwrap_err(), Error::Cancelled); + assert_eq!(completions, 3); + assert_eq!(encoder.history, original_history); + assert_eq!(encoder.levels, original_levels); + assert!(encoder.ppmd.is_none()); + + // Candidate encodes can themselves poll at the completed member byte + // count. Measure a successful two-candidate pass, then refuse its last + // poll: that last poll is the explicit boundary after the candidate + // has been evaluated and before its state can be committed. + let candidates = [Vec::new(), Vec::new()]; + let mut successful_polls = 0; + let mut probe = encoder.clone(); + probe + .encode_member_with_engine(&input, ChainEngine::Lz, &candidates, &mut |position| { + if position == input.len() { + successful_polls += 1; + } + true + }) + .unwrap(); + let mut polls = 0; + let result = + encoder.encode_member_with_engine(&input, ChainEngine::Lz, &candidates, &mut |position| { + if position == input.len() { + polls += 1; + return polls < successful_polls; + } + true + }); + assert_eq!(result, Err(Error::Cancelled)); + assert_eq!(polls, successful_polls); + assert_eq!(encoder.history, original_history); + assert_eq!(encoder.levels, original_levels); + + completions = 0; + let result = + encoder.encode_member_with_engine(&input, ChainEngine::Smaller, &[], &mut |position| { + if position == input.len() { + completions += 1; + return completions < 3; + } + true + }); + assert_eq!(result.unwrap_err(), Error::Cancelled); + assert_eq!(completions, 3); + assert_eq!(encoder.history, original_history); + assert_eq!(encoder.levels, original_levels); + assert!(encoder.ppmd.is_none()); +} + +#[test] +fn plain_solid_candidate_propagates_encoder_cancellation() { + let mut encoder = Unpack29Encoder::new(); + let input = b"plain candidate cancellation"; + let result = encoder.encode_member_with_engine(input, ChainEngine::Lz, &[], &mut |_| false); + + assert_eq!(result, Err(Error::Cancelled)); + assert!(encoder.history.is_empty()); + assert_eq!(encoder.levels, [0; TABLE_COUNT]); +} + +fn encode_with_filter(input: &[u8], kind: crate::FilterKind) -> Result> { + Unpack29Encoder::new().encode_member_with_filter(input, crate::FilterSpec::whole(kind)) +} + +fn encode_with_filter_range( + input: &[u8], + kind: crate::FilterKind, + range: Range, +) -> Result> { + Unpack29Encoder::new().encode_member_with_filter(input, crate::FilterSpec::range(kind, range)) +} + +fn encode_with_filter_ranges( + input: &[u8], + kind: crate::FilterKind, + ranges: Vec>, +) -> Result> { + let filters: Vec<_> = ranges + .into_iter() + .map(|range| crate::FilterSpec::range(kind, range)) + .collect(); + Unpack29Encoder::new().encode_member_with_filters(input, &filters) +} + +fn decode_with_raw_standard_filter( + data: &[u8], + code: &'static [u8], + init_regs: Vec<(usize, u32)>, +) -> Result> { + let filter = OwnedVmFilterRecord { + block_start: 0, + block_size: data.len(), + init_regs, + code, + global_data: Vec::new(), + }; + let mut levels = [0; TABLE_COUNT]; + let packed = super::encode_filtered_member_blocks( + data, + &[], + &[filter], + EncodeOptions::default(), + &mut levels, + None, + )?; + unpack29_decode(&packed, data.len()) +} + +#[test] +fn encoder_emits_rar29_offset_one_matches_for_repeated_bytes() { + let input = b"Z".repeat(1024); + let packed = unpack29_encode_literals(&input).unwrap(); + + assert!(packed.len() < input.len() / 4); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar29_dictionary_matches_for_repeated_sequences() { + let input = b"abc123xyz-".repeat(128); + let packed = unpack29_encode_literals(&input).unwrap(); + + assert!(packed.len() < input.len() / 2); + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_finds_rar29_matches_beyond_near_offsets() { + let phrase = b"long-distance repeated phrase for rar29 low-offset coding."; + let mut input = Vec::new(); + input.extend_from_slice(phrase); + input.extend(std::iter::repeat_n(0, 300 * 1024)); + input.extend_from_slice(phrase); + input.extend_from_slice(phrase); + let tokens = encode_tokens(&input, &[], EncodeOptions::default()); + let packed = unpack29_encode_literals(&input).unwrap(); + + assert!(tokens.iter().any(|token| matches!( + token, + EncodeToken::Match { offset, .. } if *offset > 0x40000 + ))); + assert!(packed.len() < input.len()); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + assert!( + decoded == input, + "RAR 2.9 long-distance match round-trip failed" + ); +} + +#[test] +fn encoder_emits_rar29_e8_vm_filter_record() { + let input = b"\xe8\0\0\0\0rar29 e8 filter writer payload\n".repeat(8); + let packed = encode_with_filter(&input, crate::FilterKind::E8).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert!( + decoded == input, + "RAR 2.9 multi-filter E8 round-trip failed" + ); +} + +#[test] +fn encoder_emits_rar29_e8e9_vm_filter_record() { + let input = b"\xe9\0\0\0\0rar29 e8e9 filter writer payload\n".repeat(8); + let packed = encode_with_filter(&input, crate::FilterKind::E8E9).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_segmented_e8_vm_filter_record() { + let mut input = b"prefix data that should not be x86 filtered ".to_vec(); + let start = input.len(); + input.extend_from_slice(b"\xe8\0\0\0\0segmented e8 filtered payload\n"); + let end = input.len(); + input.extend_from_slice(b" suffix data that should also remain raw"); + let packed = encode_with_filter_range(&input, crate::FilterKind::E8, start..end).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_multiple_e8_vm_filter_records() { + let mut input = vec![0x41u8; 80_000]; + for cluster_start in [8_000, 60_000] { + for index in 0..8 { + let pos = cluster_start + index * 64; + input[pos] = 0xe8; + input[pos + 1..pos + 5].copy_from_slice(&(0x2000u32 + index as u32).to_le_bytes()); + } + } + + let packed = encode_with_filter_ranges( + &input, + crate::FilterKind::E8, + vec![8_000..8_512, 60_000..60_512], + ) + .unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_segmented_e8e9_vm_filter_record() { + let mut input = b"prefix data that should not be x86 filtered ".to_vec(); + let start = input.len(); + input.extend_from_slice(b"\xe9\0\0\0\0segmented e8e9 filtered payload\n"); + let end = input.len(); + input.extend_from_slice(b" suffix data that should also remain raw"); + let packed = encode_with_filter_range(&input, crate::FilterKind::E8E9, start..end).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_delta_vm_filter_record() { + let input: Vec = (0..192).map(|index| (index * 13 + 7) as u8).collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Delta { channels: 3 }).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_segmented_delta_vm_filter_record() { + let mut input = b"prefix bytes before delta segment ".to_vec(); + let start = input.len(); + input.extend((0..192).map(|index| (index * 13 + 7) as u8)); + let end = input.len(); + input.extend_from_slice(b" suffix bytes after delta segment"); + let packed = + encode_with_filter_range(&input, crate::FilterKind::Delta { channels: 3 }, start..end) + .unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_itanium_vm_filter_record() { + let mut input = vec![0u8; 48]; + input[16] = 22; + input[21] = 20; + input.extend_from_slice(b"rar29 itanium filter writer payload\n"); + let packed = encode_with_filter(&input, crate::FilterKind::Itanium).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_segmented_itanium_vm_filter_record() { + let mut input = b"prefix bytes before itanium segment ".to_vec(); + let start = input.len(); + input.extend_from_slice(&[0; 48]); + input[start + 16] = 22; + input[start + 21] = 20; + input.extend_from_slice(b"rar29 segmented itanium filter writer payload\n"); + let end = input.len(); + input.extend_from_slice(b" suffix bytes after itanium segment"); + let packed = encode_with_filter_range(&input, crate::FilterKind::Itanium, start..end).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_rgb_vm_filter_record() { + let width = 12; + let input: Vec = (0..96).map(|index| (index * 29 + 11) as u8).collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Rgb { width, pos_r: 0 }).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_rar29_rgb_filter_with_nonzero_red_position() { + let width = 12; + let input: Vec = (0..96).map(|index| (index * 29 + 11) as u8).collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Rgb { width, pos_r: 2 }).unwrap(); + + assert_eq!(unpack29_decode(&packed, input.len()).unwrap(), input); +} + +#[test] +fn encoder_emits_rar29_segmented_rgb_vm_filter_record() { + let width = 12; + let mut input = b"prefix bytes before rgb segment ".to_vec(); + let start = input.len(); + input.extend((0..96).map(|index| (index * 29 + 11) as u8)); + let end = input.len(); + input.extend_from_slice(b" suffix bytes after rgb segment"); + let packed = encode_with_filter_range( + &input, + crate::FilterKind::Rgb { width, pos_r: 0 }, + start..end, + ) + .unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_rejects_rar29_rgb_filter_with_unaligned_scanline_width() { + let input: Vec = (0..96).map(|index| (index * 29 + 11) as u8).collect(); + assert!(encode_with_filter(&input, crate::FilterKind::Rgb { width: 8, pos_r: 0 }).is_err()); +} + +#[test] +fn encoder_rejects_rgb_ranges_shorter_than_a_pixel_or_scanline() { + for (input, width) in [(b"ab".as_slice(), 3), (b"abc".as_slice(), 6)] { + assert_eq!( + Unpack29Encoder::new().encode_member_with_filter( + input, + crate::FilterSpec::whole(crate::FilterKind::Rgb { width, pos_r: 0 }), + ), + Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid" + )), + ); + } +} + +#[test] +fn rgb_helpers_reject_invalid_parameters_even_without_caller_prechecks() { + let expected = Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid", + )); + assert_eq!(super::rgb_encode(&[0; 6], 0, 0), expected); + assert_eq!( + super::rgb_decode_with_control(&[0; 6], 3, 3, &crate::read_control::ReadControl::default(),), + expected, + ); +} + +#[test] +fn encoder_emits_rar29_audio_vm_filter_record() { + let input: Vec = (0..160) + .map(|index| (index * 7 + index / 3) as u8) + .collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Audio { channels: 2 }).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn rar29_audio_channel_bounds_match_period_decoders() { + let input: Vec = (0..256).map(|index| (index * 37 + 11) as u8).collect(); + for channels in [1, 32, 33, 128] { + let encoded = audio_encode(&input, channels).unwrap(); + let mut decoded = encoded; + let mut regs = [0; 7]; + regs[0] = channels as u32; + apply_standard_filter(StandardFilter::Audio, &mut decoded, 0, ®s).unwrap(); + assert_eq!(decoded, input, "failed with {channels} channels"); + } + + for channels in [0, 129] { + assert!(matches!( + audio_encode(&input, channels), + Err(Error::InvalidData( + "RAR 2.9 AUDIO filter channel count is invalid" + )) + )); + let mut data = input.clone(); + let mut regs = [0; 7]; + regs[0] = channels as u32; + assert!(matches!( + apply_standard_filter(StandardFilter::Audio, &mut data, 0, ®s), + Err(Error::InvalidData( + "RAR 2.9 AUDIO filter channel count is invalid" + )) + )); + } +} + +#[test] +fn audio_filter_bytecode_matches_builtin_transform() { + let channels = 2; + let input: Vec = (0..MAX_VM_AUDIO_FILTER_BLOCK_SIZE) + .map(|index| (index * 7 + index / channels + index / 257) as u8) + .collect(); + let encoded = audio_encode(&input, channels).unwrap(); + let program = Program::parse(RAR3_AUDIO_FILTER_BYTECODE).unwrap(); + let result = program + .execute(super::rarvm::Invocation { + input: &encoded, + regs: [channels as u32, 0, 0, 0, 0, 0, 0], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, input); +} + +#[test] +fn rgb_filter_matches_the_captured_winrar_bytecode() { + let program = Program::parse(RAR3_RGB_FILTER_BYTECODE).unwrap(); + for (width, pos_r) in [(3, 0), (12, 2), (63, 1)] { + let input: Vec = (0..189) + .map(|index| (index * 43 + index / 7 + 19) as u8) + .collect(); + let encoded = super::rgb_encode(&input, width, pos_r).unwrap(); + let result = program + .execute(super::rarvm::Invocation { + input: &encoded, + regs: [width as u32 + 3, pos_r as u32, 0, 0, 0, 0, 0], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, input, "width {width}, red position {pos_r}"); + } +} + +#[test] +fn itanium_filter_matches_the_captured_winrar_bytecode() { + let mut input = vec![0u8; 96]; + for bundle in 0..5 { + input[bundle * 16] = 0x16; + input[bundle * 16 + 5] = 0x50; + input[bundle * 16 + 8] = (bundle * 29 + 7) as u8; + } + let file_offset = 0x12340; + let mut encoded = input.clone(); + itanium_encode(&mut encoded, file_offset); + let program = Program::parse(RAR3_ITANIUM_FILTER_BYTECODE).unwrap(); + let result = program + .execute(super::rarvm::Invocation { + input: &encoded, + regs: [0; 7], + global_data: &[], + file_offset: u64::from(file_offset), + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, input); +} + +#[test] +fn audio_predictor_extremes_match_the_captured_winrar_bytecode() { + let program = Program::parse(RAR3_AUDIO_FILTER_BYTECODE).unwrap(); + // These xorshift streams independently drive all six coefficient + // choices and both sides of every +/-16 saturation guard. + for seed in [2u32, 3, 4, 17] { + let mut state = seed; + let input: Vec = (0..16_384) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let encoded = audio_encode(&input, 1).unwrap(); + assert_eq!( + audio_decode_with_control(&encoded, 1, &crate::read_control::ReadControl::default(),) + .unwrap(), + input, + "native predictor diverged for seed {seed}" + ); + let result = program + .execute(super::rarvm::Invocation { + input: &encoded, + regs: [1, 0, 0, 0, 0, 0, 0], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, input, "predictor diverged for seed {seed}"); + } +} + +#[test] +fn large_audio_filters_are_split_into_rarvm_safe_blocks() { + let filters = split_large_filter( + MAX_VM_FILTER_BLOCK_SIZE * 2 + 123, + crate::FilterSpec::whole(crate::FilterKind::Audio { channels: 4 }), + ) + .unwrap(); + + assert_eq!(filters.len(), 3); + assert_eq!(filters[0].range, Some(0..MAX_VM_AUDIO_FILTER_BLOCK_SIZE)); + assert_eq!( + filters[1].range, + Some(MAX_VM_AUDIO_FILTER_BLOCK_SIZE..MAX_VM_AUDIO_FILTER_BLOCK_SIZE * 2) + ); + assert_eq!( + filters[2].range, + Some(MAX_VM_AUDIO_FILTER_BLOCK_SIZE * 2..MAX_VM_FILTER_BLOCK_SIZE * 2 + 123) + ); +} + +#[test] +fn large_delta_filters_are_split_into_rarvm_safe_blocks() { + let filters = split_large_filter( + MAX_VM_FILTER_BLOCK_SIZE * 2 + 123, + crate::FilterSpec::whole(crate::FilterKind::Delta { channels: 4 }), + ) + .unwrap(); + + assert_eq!(filters.len(), 3); + assert_eq!(filters[0].range, Some(0..MAX_VM_DELTA_FILTER_BLOCK_SIZE)); + assert_eq!( + filters[1].range, + Some(MAX_VM_DELTA_FILTER_BLOCK_SIZE..MAX_VM_DELTA_FILTER_BLOCK_SIZE * 2) + ); + assert_eq!( + filters[2].range, + Some(MAX_VM_DELTA_FILTER_BLOCK_SIZE * 2..MAX_VM_FILTER_BLOCK_SIZE * 2 + 123) + ); +} + +#[test] +fn segmented_audio_filters_redeclare_program_state() { + let filters = [ + OwnedVmFilterRecord { + block_start: 0, + block_size: MAX_VM_AUDIO_FILTER_BLOCK_SIZE, + init_regs: vec![(0, 4)], + code: RAR3_AUDIO_FILTER_BYTECODE, + global_data: Vec::new(), + }, + OwnedVmFilterRecord { + block_start: MAX_VM_AUDIO_FILTER_BLOCK_SIZE, + block_size: 4096, + init_regs: vec![(0, 4)], + code: RAR3_AUDIO_FILTER_BYTECODE, + global_data: Vec::new(), + }, + ]; + let refs: Vec<&OwnedVmFilterRecord> = filters.iter().collect(); + let records = encoded_filter_records_at(&refs, 0, usize::MAX, &mut Vec::new()).unwrap(); + + assert_vm_filter_declares_program(&records[0], 0); + assert_vm_filter_declares_program(&records[1], 2); +} + +#[test] +fn encoder_emits_rar29_segmented_audio_vm_filter_record() { + let mut input = b"prefix bytes before audio segment ".to_vec(); + let start = input.len(); + input.extend((0..160).map(|index| (index * 7 + index / 3) as u8)); + let end = input.len(); + input.extend_from_slice(b" suffix bytes after audio segment"); + let packed = + encode_with_filter_range(&input, crate::FilterKind::Audio { channels: 2 }, start..end) + .unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_multiple_rar29_audio_vm_filter_records_for_large_ranges() { + let input: Vec = (0..(MAX_VM_AUDIO_FILTER_BLOCK_SIZE * 2 + 64)) + .map(|index| (index * 7 + index / 3 + index / 257) as u8) + .collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Audio { channels: 4 }).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +#[test] +fn encoder_emits_multiple_rar29_delta_vm_filter_records_for_large_ranges() { + let input: Vec = (0..(MAX_VM_DELTA_FILTER_BLOCK_SIZE * 2 + 64)) + .map(|index| (index * 11 + index / 5 + index / 251) as u8) + .collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Delta { channels: 4 }).unwrap(); + let decoded = unpack29_decode(&packed, input.len()).unwrap(); + + assert_eq!(decoded, input); +} + +fn assert_vm_filter_declares_program(record: &[u8], expected_selector: u32) { + let first = record[0]; + assert_ne!(first & 0x80, 0); + assert_ne!(first & 0x20, 0); + assert_ne!(first & 0x10, 0); + let inline_len = match first & 7 { + len @ 0..=5 => len as usize + 1, + 6 => usize::from(record[1]) + 7, + _ => u16::from_be_bytes([record[1], record[2]]) as usize, + }; + let body_start = match first & 7 { + 0..=5 => 1, + 6 => 2, + _ => 3, + }; + let body = &record[body_start..body_start + inline_len]; + let mut bits = BitReader::from_bytes(body); + assert_eq!(bits.read_encoded_u32().unwrap(), expected_selector); + let _block_start = bits.read_encoded_u32().unwrap(); + let _block_size = bits.read_encoded_u32().unwrap(); + let mask = bits.read_bits(7).unwrap(); + for index in 0..7 { + if mask & (1 << index) != 0 { + let _ = bits.read_encoded_u32().unwrap(); + } + } + assert_eq!( + bits.read_encoded_u32().unwrap() as usize, + RAR3_AUDIO_FILTER_BYTECODE.len() + ); +} + +#[test] +fn solid_encoder_emits_rar29_matches_against_previous_member_history() { + let first = b"solid rar29 shared phrase alpha beta gamma ".repeat(4); + let second = b"solid rar29 shared phrase alpha beta gamma ".repeat(2); + let independent = unpack29_encode_literals(&second).unwrap(); + let mut encoder = Unpack29Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + + assert!(second_packed.len() < independent.len()); + let mut decoder = Unpack29::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert_eq!( + decoder.decode_member(&second_packed, second.len()).unwrap(), + second + ); +} + +#[test] +fn decode_member_from_reader_accepts_incremental_input() { + struct TinyReader<'a> { + input: &'a [u8], + } + + impl std::io::Read for TinyReader<'_> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.input.is_empty() { + return Ok(0); + } + let len = self.input.len().min(out.len()).min(3); + out[..len].copy_from_slice(&self.input[..len]); + self.input = &self.input[len..]; + Ok(len) + } + } + + let mut decoder = Unpack29::new(); + let mut reader = TinyReader { + input: COMPRESSED_TEXT, + }; + let mut output = Vec::new(); + decoder + .decode_member_from_reader(&mut reader, 2400, &mut output) + .unwrap(); + + assert_eq!(output, expected_text()); +} + +#[test] +fn member_output_entry_points_share_one_decode_contract() { + let expected = expected_text(); + + let mut decoder = Unpack29::new(); + let returned = decoder + .decode_member(COMPRESSED_TEXT, expected.len()) + .unwrap(); + + let mut decoder = Unpack29::new(); + let mut written = Vec::new(); + decoder + .decode_member_to(COMPRESSED_TEXT, expected.len(), &mut written) + .unwrap(); + + let mut decoder = Unpack29::new(); + let mut reader = COMPRESSED_TEXT; + let mut read_then_written = Vec::new(); + decoder + .decode_member_from_reader(&mut reader, expected.len(), &mut read_then_written) + .unwrap(); + + assert_eq!(returned, expected); + assert_eq!(written, expected); + assert_eq!(read_then_written, expected); +} + +#[test] +fn member_output_entry_points_report_the_same_truncation() { + let truncated = &COMPRESSED_TEXT[..COMPRESSED_TEXT.len() / 2]; + let expected = Error::InvalidData("RAR 2.9 bitstream is truncated"); + + let returned = Unpack29::new().decode_member(truncated, 2400).unwrap_err(); + + let mut decoder = Unpack29::new(); + let mut written = Vec::new(); + let write_error = decoder + .decode_member_to(truncated, 2400, &mut written) + .unwrap_err(); + + let mut decoder = Unpack29::new(); + let mut reader = truncated; + let mut read_then_written = Vec::new(); + let reader_error = decoder + .decode_member_from_reader(&mut reader, 2400, &mut read_then_written) + .unwrap_err(); + + assert_eq!(returned, expected); + assert_eq!(write_error, expected); + assert_eq!(reader_error, expected); +} + +#[test] +fn member_output_entry_points_share_empty_member_handling() { + let packed = unpack29_encode_literals(b"").unwrap(); + + assert!(Unpack29::new() + .decode_member(&packed, 0) + .unwrap() + .is_empty()); + + let mut decoder = Unpack29::new(); + let mut written = Vec::new(); + decoder.decode_member_to(&packed, 0, &mut written).unwrap(); + assert!(written.is_empty()); + + let mut decoder = Unpack29::new(); + let mut reader = packed.as_slice(); + decoder + .decode_member_from_reader(&mut reader, 0, &mut written) + .unwrap(); + assert!(written.is_empty()); +} + +#[test] +fn direct_codec_accepts_an_empty_stream_for_an_empty_member() { + assert!(Unpack29::new().decode_member(&[], 0).unwrap().is_empty()); +} + +#[test] +fn empty_members_still_validate_a_supplied_table_header() { + assert_eq!( + Unpack29::new().decode_member(&[0], 0), + Err(Error::InvalidData("RAR 2.9 bitstream is truncated")) + ); + + let invalid = table_description(&[0; 20], &[]); + assert_eq!( + Unpack29::new().decode_member(&invalid, 0), + Err(Error::InvalidData("RAR 2.9 empty Huffman table")) + ); +} + +#[test] +fn empty_solid_member_can_use_the_previous_members_huffman_tables() { + let first = b"solid member with a retained Huffman table"; + let mut encoder = Unpack29Encoder::new(); + let mut first_packed = encoder.encode_member(first).unwrap(); + + // Locate the last bit of the first member's end marker, then change + // NewFileNewTables (0, 1) to NewFileKeepTables (0, 0). The latter is + // valid legacy wire syntax even though our writer does not emit it. + let mut probe = Unpack29::new(); + assert_eq!( + probe.decode_member(&first_packed, first.len()).unwrap(), + first + ); + let keep_tables_bit = probe.bits.bit_pos - 1; + let mask = 1 << (7 - keep_tables_bit % 8); + assert_ne!(first_packed[keep_tables_bit / 8] & mask, 0); + first_packed[keep_tables_bit / 8] &= !mask; + + let codes = canonical_codes(&encoder.levels[..MAIN_COUNT]); + let end = codes[256].unwrap(); + let mut bits = BitWriter::default(); + bits.write_bits(u32::from(end.code), end.len); + bits.write_bit(false); // new file + bits.write_bit(true); // next member reads new tables + let empty_packed = bits.finish(); + + let mut decoder = Unpack29::new(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + assert!(decoder.in_lz_block); + assert!(decoder.decode_member(&empty_packed, 0).unwrap().is_empty()); + assert!(!decoder.in_lz_block); +} + +#[test] +fn oversized_untrusted_filter_waits_across_a_streaming_batch() { + let expected = vec![0; STREAM_CHUNK + 1]; + let packed = Unpack29Encoder::with_options(EncodeOptions::new(0)) + .encode_member(&expected) + .unwrap(); + let mut decoder = Unpack29::new(); + decoder + .programs + .push(VmProgram { + kind: VmProgramKind::Standard(StandardFilter::E8), + block_size: expected.len(), + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 0, + size: expected.len(), + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + + assert_eq!( + decoder.decode_member(&packed, expected.len()).unwrap(), + expected + ); + assert!(decoder.filters.is_empty()); +} + +#[test] +fn slice_and_reader_entry_points_share_output_failures() { + struct FailingWriter; + impl std::io::Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("deliberate failure")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let expected = Error::from(std::io::Error::other("deliberate failure")); + let mut decoder = Unpack29::new(); + assert_eq!( + decoder + .decode_member_to(COMPRESSED_TEXT, 2400, &mut FailingWriter) + .unwrap_err(), + expected + ); + + let mut decoder = Unpack29::new(); + let mut reader = COMPRESSED_TEXT; + assert_eq!( + decoder + .decode_member_from_reader(&mut reader, 2400, &mut FailingWriter) + .unwrap_err(), + expected + ); + + let mut decoder = Unpack29::new(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 0, + size: 1, + regs: [0; 7], + global_data: vec![1].into(), + }) + .unwrap(); + assert_eq!( + decoder + .decode_non_solid_member_to(COMPRESSED_TEXT, 2400, &mut FailingWriter) + .unwrap_err(), + expected + ); + assert!(decoder.filters.is_empty()); +} + +#[test] +fn decode_non_solid_member_resets_reusable_decoder_state() { + let mut decoder = Unpack29::new(); + decoder.output.extend_from_slice(b"stale history").unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 0, + size: 1, + regs: [0; 7], + global_data: vec![1, 2, 3].into(), + }) + .unwrap(); + + let mut output = Vec::new(); + decoder + .decode_non_solid_member_to(COMPRESSED_TEXT, 2400, &mut output) + .unwrap(); + + assert_eq!(output, expected_text()); + assert!(decoder.filters.is_empty()); +} + +#[test] +fn e8_filter_uses_member_relative_offset_in_solid_stream() { + let mut decoder = Unpack29::new(); + let member_start = 1000usize; + let filter_start = member_start + 100; + decoder.output.resize(filter_start + 8, 0).unwrap(); + decoder.output[filter_start] = 0xe8; + + let call_operand_pos = 1u32; + let member_relative_filter_start = (filter_start - member_start) as u32; + let decoded_addr = 0x2000u32; + let encoded_addr = decoded_addr + .wrapping_add(member_relative_filter_start) + .wrapping_add(call_operand_pos); + decoder.output[filter_start + 1..filter_start + 5].copy_from_slice(&encoded_addr.to_le_bytes()); + decoder + .programs + .push(VmProgram { + kind: VmProgramKind::Standard(StandardFilter::E8), + block_size: 5, + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: filter_start, + size: 5, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + + let filtered = decoder + .filtered_range(member_start, filter_start + 5, member_start) + .unwrap(); + let operand = u32::from_le_bytes([filtered[101], filtered[102], filtered[103], filtered[104]]); + + assert_eq!(operand, decoded_addr); +} + +#[test] +fn generic_vm_filter_executes_from_filtered_range() { + let mut decoder = Unpack29::new(); + decoder + .output + .extend_from_slice(&[0x11, 0x22, 0x33]) + .unwrap(); + decoder + .programs + .push(VmProgram { + kind: VmProgramKind::Generic( + Program { + static_data: Vec::new(), + instructions: vec![ + Instruction { + opcode: Opcode::Mov, + byte_mode: true, + operands: vec![Operand::Absolute(0), Operand::Immediate(0x44)], + }, + Instruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Vec::new(), + }, + ], + } + .into(), + ), + block_size: 3, + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + decoder + .filters + .push(VmFilter { + program: 0, + start: 0, + size: 3, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + + let filtered = decoder.filtered_range(0, 3, 0).unwrap(); + + assert_eq!(filtered, [0x44, 0x22, 0x33]); +} + +#[test] +fn generic_vm_filter_uses_explicit_then_retained_user_globals() { + // Copies user-global byte 0 to the output, increments it, records one + // retained user byte at global offset 0x30, then returns. + const GLOBAL_COUNTER_PROGRAM: &[u8] = &[ + 0x0d, 0x05, 0xc0, 0x7c, 0x00, 0x0f, 0x01, 0x01, 0xaf, 0x80, 0x01, 0xe0, 0x20, 0x01, 0xf0, + 0x00, 0x3c, 0x03, 0x00, 0x1b, 0x80, + ]; + let filters = [ + OwnedVmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: Vec::new(), + code: GLOBAL_COUNTER_PROGRAM, + global_data: vec![b'A'], + }, + OwnedVmFilterRecord { + block_start: 1, + block_size: 1, + init_regs: Vec::new(), + code: GLOBAL_COUNTER_PROGRAM, + global_data: Vec::new(), + }, + ]; + let refs = filters.iter().collect::>(); + let records = encoded_filter_records_at(&refs, 0, usize::MAX, &mut Vec::new()).unwrap(); + let packed = super::encode_member_inner( + &[0, 0], + &[], + &records, + EncodeOptions::default(), + false, + &mut [0; TABLE_COUNT], + None, + ) + .unwrap(); + + assert_eq!(unpack29_decode(&packed, 2).unwrap(), b"AB"); +} + +#[test] +fn standard_filters_reject_malformed_delta_and_rgb_registers() { + let mut delta = vec![0; 32]; + let mut delta_regs = [0; 7]; + // 33 channels is legal here: the count comes from R[0], not from a + // five-bit field, and the reference decoder allows up to 1024. + delta_regs[0] = 33; + assert_eq!( + apply_standard_filter(StandardFilter::Delta, &mut delta, 0, &delta_regs), + Ok(()) + ); + delta_regs[0] = super::MAX_DELTA_CHANNELS as u32 + 1; + assert_eq!( + apply_standard_filter(StandardFilter::Delta, &mut delta, 0, &delta_regs), + Err(Error::InvalidData( + "RAR 2.9 DELTA filter channel count is invalid" + )) + ); + delta_regs[0] = 0; + assert_eq!( + apply_standard_filter(StandardFilter::Delta, &mut delta, 0, &delta_regs), + Err(Error::InvalidData( + "RAR 2.9 DELTA filter channel count is invalid" + )) + ); + + let mut rgb = vec![0; 32]; + let mut rgb_regs = [0; 7]; + rgb_regs[0] = 2; + assert_eq!( + apply_standard_filter(StandardFilter::Rgb, &mut rgb, 0, &rgb_regs), + Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid" + )) + ); + rgb_regs[0] = 15; + rgb_regs[1] = 3; + assert_eq!( + apply_standard_filter(StandardFilter::Rgb, &mut rgb, 0, &rgb_regs), + Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid" + )) + ); +} + +#[test] +fn standard_filter_records_enforce_delta_and_audio_channel_bounds() { + let input = vec![0; 1024]; + assert_eq!( + decode_with_raw_standard_filter(&input, RAR3_DELTA_FILTER_BYTECODE, vec![(0, 0)]), + Err(Error::InvalidData( + "RAR 2.9 DELTA filter channel count is invalid" + )) + ); + assert_eq!( + decode_with_raw_standard_filter( + &input, + RAR3_DELTA_FILTER_BYTECODE, + vec![(0, (super::MAX_DELTA_CHANNELS + 1) as u32)], + ), + Err(Error::InvalidData( + "RAR 2.9 DELTA filter channel count is invalid" + )) + ); + assert_eq!( + decode_with_raw_standard_filter( + &input, + RAR3_DELTA_FILTER_BYTECODE, + vec![(0, super::MAX_DELTA_CHANNELS as u32)], + ) + .unwrap(), + input + ); + + for channels in [0, super::MAX_AUDIO_CHANNELS + 1] { + assert_eq!( + decode_with_raw_standard_filter( + &input, + RAR3_AUDIO_FILTER_BYTECODE, + vec![(0, channels as u32)], + ), + Err(Error::InvalidData( + "RAR 2.9 AUDIO filter channel count is invalid" + )) + ); + } +} + +#[test] +fn standard_rgb_filter_records_reject_nonportable_parameters() { + let cases = [ + (vec![0; 2], 3, 0, "short input"), + (vec![0; 12], 3, 0, "zero width"), + (vec![0; 12], 11, 0, "unaligned width"), + (vec![0; 12], 18, 0, "width beyond the block"), + (vec![0; 12], 15, 3, "red channel beyond RGB"), + ]; + for (input, encoded_width, pos_r, description) in cases { + assert_eq!( + decode_with_raw_standard_filter( + &input, + RAR3_RGB_FILTER_BYTECODE, + vec![(0, encoded_width), (1, pos_r)], + ), + Err(Error::InvalidData( + "RAR 2.9 RGB filter parameters are invalid" + )), + "accepted {description}" + ); + } +} + +#[test] +fn short_itanium_filter_records_are_defined_noops() { + let mut empty = Vec::new(); + itanium_decode(&mut empty, 0); + assert!(empty.is_empty()); + + for len in [1, 20, 21, 22] { + let input: Vec = (0..len).map(|index| (index * 17 + 3) as u8).collect(); + assert_eq!( + decode_with_raw_standard_filter(&input, RAR3_ITANIUM_FILTER_BYTECODE, vec![]).unwrap(), + input, + "changed a {len}-byte non-branching block" + ); + } +} + +#[test] +fn short_itanium_filter_encoding_is_a_defined_noop() { + for len in 4..=21 { + let input: Vec = (0..len).map(|index| (index * 17 + 3) as u8).collect(); + let packed = encode_with_filter(&input, crate::FilterKind::Itanium).unwrap(); + assert_eq!( + unpack29_decode(&packed, input.len()).unwrap(), + input, + "changed a {len}-byte block" + ); + } +} + +#[test] +fn vm_encoded_u32_accepts_32_bit_form() { + let mut bits = super::BitReader::from_bytes(&[0xff; 5]); + + assert_eq!(bits.read_encoded_u32().unwrap(), 0xffff_ffff); +} + +#[test] +fn vm_encoded_u32_accepts_the_signed_constant_form() { + let mut encoded = BitWriter::default(); + encoded.write_bits(1, 2); + encoded.write_bits(0x0a, 8); + encoded.write_bits(0x05, 4); + let mut bits = BitReader::from_bytes(&encoded.finish()); + + assert_eq!(bits.read_encoded_u32().unwrap(), 0xffff_ffa5); +} + +#[test] +fn vm_filter_record_serializer_uses_every_specified_length_form() { + fn declared_payload(record: &[u8]) -> (usize, usize) { + match record[0] & 7 { + len @ 0..=5 => (1, usize::from(len) + 1), + 6 => (2, usize::from(record[1]) + 7), + _ => (3, usize::from(u16::from_be_bytes([record[1], record[2]]))), + } + } + + let medium_code = vec![0; 53]; + let long_code = vec![0; 300]; + let records = [ + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &[], + global_data: &[], + }, + 1, + false, + ) + .unwrap(), + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &medium_code, + global_data: &[], + }, + 0, + true, + ) + .unwrap(), + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &long_code, + global_data: &[], + }, + 0, + true, + ) + .unwrap(), + ]; + + assert!(matches!(records[0][0] & 7, 0..=5)); + assert_eq!(records[1][0] & 7, 6); + assert_eq!(records[2][0] & 7, 7); + for record in records { + let (header_len, payload_len) = declared_payload(&record); + assert_eq!(record.len(), header_len + payload_len); + } +} + +#[test] +fn decoder_reads_a_vm_filter_with_a_16_bit_payload_length() { + let globals = vec![0x5a; 300]; + let record = super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 3, + block_size: 16, + init_regs: &[], + code: super::RAR3_E8_FILTER_BYTECODE, + global_data: &globals, + }, + 0, + true, + ) + .unwrap(); + assert_eq!(record[0] & 7, 7); + + let mut decoder = Unpack29::new(); + decoder.bits = BitReader::from_bytes(&record); + decoder.read_vm_code().unwrap(); + + assert_eq!(decoder.programs.len(), 1); + assert_eq!(decoder.filters.len(), 1); + assert_eq!(decoder.filters[0].start, 3); + assert_eq!(decoder.filters[0].size, 16); + assert_eq!( + &decoder.filters[0].global_data[super::VM_SYSTEM_GLOBAL_SIZE..], + globals + ); +} + +#[test] +fn vm_filter_record_serializer_rejects_invalid_fields() { + assert_eq!( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 0, + init_regs: &[], + code: &[1], + global_data: &[], + }, + 0, + true, + ), + Err(Error::InvalidData("RAR 2.9 VM filter block is empty")) + ); + assert_eq!( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &[], + global_data: &[], + }, + 0, + true, + ), + Err(Error::InvalidData("RAR 2.9 VM filter bytecode is empty")) + ); + assert_eq!( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[(7, 0)], + code: &[1], + global_data: &[], + }, + 0, + true, + ), + Err(Error::InvalidData( + "RAR 2.9 VM init register index is invalid" + )) + ); + + let oversized_global = vec![0; 65_536]; + assert_eq!( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &[], + global_data: &oversized_global, + }, + 1, + false, + ), + Err(Error::InvalidData("RAR 2.9 VM filter record is too large")) + ); +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn vm_filter_record_serializer_rejects_fields_wider_than_the_wire() { + for (block_start, block_size, expected) in [ + ( + usize::MAX, + 1, + Error::InvalidData("RAR 2.9 VM block start overflows"), + ), + ( + 0, + usize::MAX, + Error::InvalidData("RAR 2.9 VM block size overflows"), + ), + ] { + assert_eq!( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start, + block_size, + init_regs: &[], + code: &[], + global_data: &[], + }, + 1, + false, + ), + Err(expected) + ); + } +} + +#[test] +fn vm_filter_records_must_belong_to_their_encoding_block() { + let filter = OwnedVmFilterRecord { + block_start: 10, + block_size: 1, + init_regs: Vec::new(), + code: super::RAR3_E8_FILTER_BYTECODE, + global_data: Vec::new(), + }; + + assert_eq!( + encoded_filter_records_at(&[&filter], 11, 32, &mut Vec::new()), + Err(Error::InvalidData( + "RAR 2.9 VM filter starts before its block" + )) + ); + assert_eq!( + encoded_filter_records_at(&[&filter], 0, 10, &mut Vec::new()), + Err(Error::InvalidData( + "RAR 2.9 VM filter starts further past its block than the window can express" + )) + ); +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn member_relative_vm_filter_offsets_obey_the_u32_wire_boundary() { + // PPMd callers declare filters relative to the whole member (base 0), + // unlike LZ callers, which rebase them into dictionary-sized blocks. + let mut filter = OwnedVmFilterRecord { + block_start: u32::MAX as usize, + block_size: 4, + init_regs: Vec::new(), + code: super::RAR3_E8_FILTER_BYTECODE, + global_data: Vec::new(), + }; + let records = encoded_filter_records_at(&[&filter], 0, usize::MAX, &mut Vec::new()).unwrap(); + assert_eq!(records.len(), 1); + let record = &records[0]; + let header_len = match record[0] & 7 { + 0..=5 => 1, + 6 => 2, + _ => 3, + }; + let mut body = BitReader::from_bytes(&record[header_len..]); + assert_eq!(body.read_encoded_u32().unwrap(), 0); + assert_eq!(body.read_encoded_u32().unwrap(), u32::MAX); + assert_eq!(body.read_encoded_u32().unwrap(), 4); + + filter.block_start += 1; + assert_eq!( + encoded_filter_records_at(&[&filter], 0, usize::MAX, &mut Vec::new()), + Err(Error::InvalidData("RAR 2.9 VM block start overflows")) + ); + // The same absolute position is representable after an LZ block rebase. + assert!(encoded_filter_records_at(&[&filter], filter.block_start, 4, &mut Vec::new(),).is_ok()); +} + +#[test] +fn archive_decoder_rejects_invalid_vm_program_records() { + let cases = [ + ( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &[], + global_data: &[], + }, + 2, + false, + ) + .unwrap(), + Error::InvalidData("RAR 2.9 VM program index is invalid"), + ), + ( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &[], + global_data: &[], + }, + 0, + false, + ) + .unwrap(), + Error::InvalidData("RAR 2.9 VM code is empty"), + ), + ( + super::encode_vm_filter_record_inner( + super::VmFilterRecord { + block_start: 0, + block_size: 1, + init_regs: &[], + code: &[1, 0], + global_data: &[], + }, + 0, + true, + ) + .unwrap(), + Error::InvalidData("RARVM program checksum mismatch"), + ), + ]; + + for (record, expected) in cases { + let packed = super::encode_member_inner( + &[0], + &[], + &[record], + EncodeOptions::default(), + false, + &mut [0; TABLE_COUNT], + None, + ) + .unwrap(); + assert_eq!(unpack29_decode(&packed, 1), Err(expected)); + } +} + +#[test] +fn archive_decoder_rejects_generic_vm_block_larger_than_work_memory() { + // A checksum-valid nonstandard program with an implicit RET. + const GENERIC_RET: &[u8] = &[0x5c, 0x5c]; + const TOO_LARGE: usize = 0x3c001; + let filter = OwnedVmFilterRecord { + block_start: 0, + block_size: TOO_LARGE, + init_regs: Vec::new(), + code: GENERIC_RET, + global_data: Vec::new(), + }; + let records = encoded_filter_records_at(&[&filter], 0, usize::MAX, &mut Vec::new()).unwrap(); + let packed = super::encode_member_inner( + &vec![0; TOO_LARGE], + &[], + &records, + EncodeOptions::default(), + false, + &mut [0; TABLE_COUNT], + None, + ) + .unwrap(); + + assert_eq!( + unpack29_decode(&packed, TOO_LARGE), + Err(Error::InvalidData("RARVM filter input is too large")) + ); +} + +#[test] +fn vm_global_data_size_is_capped_before_reading_or_allocation() { + let mut decoder = Unpack29::new(); + decoder + .programs + .push(VmProgram { + kind: VmProgramKind::Standard(StandardFilter::E8), + block_size: 1, + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + + let mut data = BitWriter::default(); + data.write_encoded_u32(1); + data.write_encoded_u32(0); + data.write_encoded_u32(u32::MAX); + + assert_eq!( + decoder.parse_vm_code(0x80 | 0x08, data.finish()), + Err(Error::InvalidData("RAR 2.9 VM global data is too large")) + ); +} + +#[test] +fn vm_code_size_is_capped_before_allocation() { + let mut decoder = Unpack29::new(); + let mut data = BitWriter::default(); + data.write_encoded_u32(0); + data.write_encoded_u32(1); + data.write_encoded_u32(super::MAX_VM_CODE_SIZE as u32); + + assert_eq!( + decoder.parse_vm_code(0x80, data.finish()), + Err(Error::InvalidData("RAR 2.9 VM code is too large")) + ); +} + +#[test] +fn vm_program_and_filter_counts_are_capped() { + let mut decoder = Unpack29::new(); + decoder + .programs + .resize_with(super::MAX_VM_PROGRAMS, || VmProgram { + kind: VmProgramKind::Standard(StandardFilter::E8), + block_size: 1, + exec_count: 0, + globals: Vec::new().into(), + }) + .unwrap(); + + let mut new_program = BitWriter::default(); + new_program.write_encoded_u32((super::MAX_VM_PROGRAMS + 1) as u32); + new_program.write_encoded_u32(1); + new_program.write_encoded_u32(1); + new_program.write_bits(0, 8); + assert_eq!( + decoder.parse_vm_code(0x80, new_program.finish()), + Err(Error::InvalidData("RAR 2.9 VM program limit exceeded")) + ); + + decoder.programs.truncate(1); + decoder.last_filter = 0; + decoder + .filters + .resize_with(super::MAX_VM_FILTERS, || VmFilter { + program: 0, + start: 0, + size: 1, + regs: [0; 7], + global_data: Vec::new().into(), + }) + .unwrap(); + let mut reused_program = BitWriter::default(); + reused_program.write_encoded_u32(0); + assert_eq!( + decoder.parse_vm_code(0, reused_program.finish()), + Err(Error::InvalidData("RAR 2.9 VM filter limit exceeded")) + ); +} + +#[test] +fn itanium_filter_round_trips_with_high_file_offset() { + let mut data = vec![0u8; 64]; + for (index, byte) in data.iter_mut().enumerate() { + *byte = index as u8; + } + data[0] = 0; + data[7] = 5 << 3; + let original = data.clone(); + + itanium_encode(&mut data, u32::MAX); + itanium_decode(&mut data, u32::MAX); + + assert_eq!(data, original); +} + +fn expected_text() -> Vec { + "Hello, RAR 3.x fixture world.\n".repeat(80).into_bytes() +} diff --git a/arca-rar/vendor/rars/src/codec/rar50.rs b/arca-rar/vendor/rars/src/codec/rar50.rs new file mode 100644 index 00000000..a74d1457 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50.rs @@ -0,0 +1,1927 @@ +#[cfg(feature = "write")] +mod encoder; +#[cfg(feature = "write")] +pub use encoder::{ + encode_compressed_block, encode_literal_only, encode_lz_member, encode_lz_member_with_history, + encode_lz_member_with_history_and_options, encode_lz_member_with_options, encode_table_lengths, + encode_table_lengths_with_bit_count, EncodeOptions, Unpack50Encoder, +}; +#[cfg(all(test, feature = "write"))] +pub(crate) use encoder::{ + encode_lz_member_with_options_and_progress, encode_lz_reader_to, filtered_lz_member, +}; +#[cfg(feature = "write")] +pub(crate) use encoder::{ + encode_owned_member, filtered_owned_member, streaming_blocks_with_allowance, BlockSplitter, + LZ_BLOCK_SIZE, MAX_DELTA_CHANNELS, MAX_LZ_BLOCK_SIZE, +}; + +use super::filters::{self, DeltaErrorMessages}; +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; +use std::io::Read; +use std::ops::Range; + +pub const LEVEL_TABLE_SIZE: usize = 20; +pub const MAIN_TABLE_SIZE: usize = 306; +pub const DISTANCE_TABLE_SIZE_50: usize = 64; +pub const DISTANCE_TABLE_SIZE_70: usize = 80; +pub const ALIGN_TABLE_SIZE: usize = 16; +pub const LENGTH_TABLE_SIZE: usize = 44; +const DEFAULT_DICTIONARY_SIZE: usize = 4 * 1024 * 1024; +const MAX_INITIAL_OUTPUT_CAPACITY: usize = 1024 * 1024; +const STREAM_FLUSH_THRESHOLD: usize = 64 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressedBlock { + pub header: CompressedBlockHeader, + pub header_len: usize, + pub payload: Range, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CompressedBlockHeader { + pub flags: u8, + pub is_last: bool, + pub has_tables: bool, + pub final_byte_bits: u8, + pub payload_size: usize, + pub payload_bits: usize, +} + +struct OwnedCompressedBlock { + header: CompressedBlockHeader, + payload: Buffer, +} + +#[derive(Debug)] +#[doc(hidden)] +pub enum StreamDecodeError { + Decode(Error), + FilteredMember, + Sink(E), +} + +impl From for StreamDecodeError { + fn from(error: Error) -> Self { + Self::Decode(error) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[doc(hidden)] +pub enum DecodedChunk<'a> { + Bytes(&'a [u8]), + Repeated { byte: u8, len: usize }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableLengths { + pub main: Vec, + pub distance: Vec, + pub align: Vec, + pub length: Vec, +} + +struct OwnedLengths { + main: Buffer, + distance: Buffer, + align: Buffer, + length: Buffer, +} + +#[derive(Debug, Clone)] +pub struct DecodeTables { + pub main: HuffmanTable, + pub distance: HuffmanTable, + pub align: HuffmanTable, + pub length: HuffmanTable, + pub align_mode: bool, +} + +impl DecodeTables { + pub fn from_lengths(lengths: &TableLengths) -> Result { + let align_mode = lengths + .align + .iter() + .any(|&length| length != 0 && length != 4); + Ok(Self { + main: HuffmanTable::from_lengths(&lengths.main)?, + distance: HuffmanTable::from_lengths(&lengths.distance)?, + align: HuffmanTable::from_lengths(&lengths.align)?, + length: HuffmanTable::from_lengths(&lengths.length)?, + align_mode, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecodeMode { + LiteralOnly, + Lz, + LzNoFilters, +} + +impl DecodeMode { + fn uses_lz(self) -> bool { + matches!(self, Self::Lz | Self::LzNoFilters) + } + + fn applies_filters(self) -> bool { + matches!(self, Self::Lz) + } +} + +pub fn parse_compressed_block(input: &[u8]) -> Result { + if input.len() < 3 { + return Err(Error::NeedMoreInput); + } + + let flags = input[0]; + let checksum = input[1]; + let size_bytes = match (flags >> 3) & 0x03 { + 0 => 1, + 1 => 2, + 2 => 3, + _ => return Err(Error::InvalidData("RAR 5 block size length is invalid")), + }; + let header_len = 2 + size_bytes; + if input.len() < header_len { + return Err(Error::NeedMoreInput); + } + + let size_data = &input[2..header_len]; + let actual = size_data + .iter() + .fold(checksum ^ flags, |acc, &byte| acc ^ byte); + if actual != 0x5a { + return Err(Error::InvalidData("RAR 5 block header checksum mismatch")); + } + + let payload_size = size_data + .iter() + .enumerate() + .fold(0usize, |acc, (index, &byte)| { + acc | (usize::from(byte) << (index * 8)) + }); + // The size field is at most three bytes and the header at most five, so + // this sum fits even on a 32-bit target. + let payload_end = header_len + payload_size; + if input.len() < payload_end { + return Err(Error::NeedMoreInput); + } + + let final_byte_bits = ((flags & 0x07) + 1).min(8); + let payload_bits = if payload_size == 0 { + 0 + } else { + (payload_size - 1) * 8 + usize::from(final_byte_bits) + }; + + Ok(CompressedBlock { + header: CompressedBlockHeader { + flags, + is_last: flags & 0x40 != 0, + has_tables: flags & 0x80 != 0, + final_byte_bits, + payload_size, + payload_bits, + }, + header_len, + payload: header_len..payload_end, + }) +} + +pub fn read_level_lengths(input: &[u8]) -> Result<([u8; LEVEL_TABLE_SIZE], usize)> { + let mut bits = BitReader::new(input); + let mut lengths = [0; LEVEL_TABLE_SIZE]; + let mut pos = 0; + while pos < LEVEL_TABLE_SIZE { + let length = bits.read_bits(4)? as u8; + if length == 15 { + let zero_count = bits.read_bits(4)? as usize; + if zero_count == 0 { + lengths[pos] = 15; + pos += 1; + } else { + let count = zero_count + 2; + for _ in 0..count { + if pos >= LEVEL_TABLE_SIZE { + break; + } + lengths[pos] = 0; + pos += 1; + } + } + } else { + lengths[pos] = length; + pos += 1; + } + } + Ok((lengths, bits.bit_pos)) +} + +#[derive(Debug)] +struct ReaderTables { + main: HuffmanState, + distance: HuffmanState, + align: HuffmanState, + length: HuffmanState, + align_mode: bool, +} +impl ReaderTables { + fn from_lengths(lengths: &OwnedLengths, allowance: &B) -> Result { + Ok(Self { + main: HuffmanState::from_lengths(&lengths.main, allowance)?, + distance: HuffmanState::from_lengths(&lengths.distance, allowance)?, + align: HuffmanState::from_lengths(&lengths.align, allowance)?, + length: HuffmanState::from_lengths(&lengths.length, allowance)?, + align_mode: lengths + .align + .iter() + .any(|&length| length != 0 && length != 4), + }) + } + fn try_clone(&self) -> Result { + Ok(Self { + main: self.main.try_clone()?, + distance: self.distance.try_clone()?, + align: self.align.try_clone()?, + length: self.length.try_clone()?, + align_mode: self.align_mode, + }) + } +} + +pub fn table_length_count(algorithm_version: u8) -> Result { + match algorithm_version { + 0 => Ok(MAIN_TABLE_SIZE + DISTANCE_TABLE_SIZE_50 + ALIGN_TABLE_SIZE + LENGTH_TABLE_SIZE), + 1 => Ok(MAIN_TABLE_SIZE + DISTANCE_TABLE_SIZE_70 + ALIGN_TABLE_SIZE + LENGTH_TABLE_SIZE), + _ => Err(Error::InvalidData( + "RAR 5 unknown compression algorithm version", + )), + } +} + +pub fn read_table_lengths(input: &[u8], algorithm_version: u8) -> Result<(TableLengths, usize)> { + read_table_lengths_with_allowance(input, algorithm_version, &Allowance::default()).map( + |(lengths, bits)| { + ( + TableLengths { + main: lengths.main.into_vec(), + distance: lengths.distance.into_vec(), + align: lengths.align.into_vec(), + length: lengths.length.into_vec(), + }, + bits, + ) + }, + ) +} + +fn read_table_lengths_with_allowance( + input: &[u8], + algorithm_version: u8, + allowance: &B, +) -> Result<(OwnedLengths, usize)> { + let table_size = table_length_count(algorithm_version)?; + let (level_lengths, level_bits) = read_level_lengths(input)?; + let level_decoder = HuffmanState::from_lengths(&level_lengths, allowance)?; + let mut bits = BitReader::new(input); + bits.bit_pos = level_bits; + + let mut lengths = Buffer::with_capacity(table_size, allowance)?; + while lengths.len() < table_size { + let number = level_decoder.decode(&mut bits)?; + match number { + 0..=15 => lengths.push_admitted(number as u8), + 16 | 17 => { + if lengths.is_empty() { + return Err(Error::InvalidData( + "RAR 5 table repeats missing previous length", + )); + } + let count = if number == 16 { + 3 + bits.read_bits(3)? as usize + } else { + 11 + bits.read_bits(7)? as usize + }; + let previous = *lengths.last().unwrap(); + for _ in 0..count { + if lengths.len() >= table_size { + break; + } + lengths.push_admitted(previous); + } + } + _ => { + // The level table has exactly 20 symbols, so these are 18/19. + let count = if number == 18 { + 3 + bits.read_bits(3)? as usize + } else { + 11 + bits.read_bits(7)? as usize + }; + for _ in 0..count { + if lengths.len() >= table_size { + break; + } + lengths.push_admitted(0); + } + } + } + } + + // table_length_count above rejects every version other than 0 and 1. + let distance_size = if algorithm_version == 0 { + DISTANCE_TABLE_SIZE_50 + } else { + DISTANCE_TABLE_SIZE_70 + }; + let distance_start = MAIN_TABLE_SIZE; + let align_start = distance_start + distance_size; + let length_start = align_start + ALIGN_TABLE_SIZE; + + Ok(( + OwnedLengths { + main: Buffer::copied(&lengths[..distance_start], allowance)?, + distance: Buffer::copied(&lengths[distance_start..align_start], allowance)?, + align: Buffer::copied(&lengths[align_start..length_start], allowance)?, + length: Buffer::copied(&lengths[length_start..], allowance)?, + }, + bits.bit_pos, + )) +} + +pub fn decode_literal_only( + input: &[u8], + algorithm_version: u8, + output_size: usize, +) -> Result> { + let mut decoder = Unpack50Decoder::new(); + decoder.decode_member( + input, + algorithm_version, + output_size, + false, + DecodeMode::LiteralOnly, + ) +} + +pub fn decode_lz(input: &[u8], algorithm_version: u8, output_size: usize) -> Result> { + let mut decoder = Unpack50Decoder::new(); + decoder.decode_member(input, algorithm_version, output_size, false, DecodeMode::Lz) +} + +/// The filters RAR 5 has a builtin type for. +/// +/// Narrower than [`crate::FilterKind`], which names every filter any format +/// can apply. The conversion below is where a filter RAR 5 cannot encode is +/// turned away. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Rar50Filter { + Delta { channels: usize }, + E8, + E8E9, + Arm, +} + +impl TryFrom for Rar50Filter { + type Error = crate::UnsupportedFilterKind; + + fn try_from(kind: crate::FilterKind) -> std::result::Result { + use crate::FilterKind as Kind; + match kind { + Kind::Delta { channels } => Ok(Self::Delta { channels }), + Kind::E8 => Ok(Self::E8), + Kind::E8E9 => Ok(Self::E8E9), + Kind::Arm => Ok(Self::Arm), + // No wildcard arm: an eighth filter has to be decided about here. + kind @ (Kind::Itanium | Kind::Rgb { .. } | Kind::Audio { .. }) => { + Err(crate::UnsupportedFilterKind(kind)) + } + } + } +} + +#[derive(Debug)] +pub struct Unpack50Decoder { + pub(crate) read_control: crate::read_control::ReadControl, + state: ReaderState, +} +impl Clone for Unpack50Decoder { + fn clone(&self) -> Self { + Self { + read_control: self.read_control.clone(), + state: self.state.try_clone().expect("unlimited decoder copy"), + } + } +} +impl Unpack50Decoder { + pub fn new() -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + state: ReaderState::new(&Allowance::default()), + } + } + #[cfg(all(test, feature = "write"))] + fn copy_match( + &self, + output: &mut Vec, + distance: usize, + length: usize, + output_limit: usize, + dictionary_size: usize, + ) -> Result<()> { + let mut owned = Buffer::from_vec(std::mem::take(output)); + let result = + self.state + .copy_match(&mut owned, distance, length, output_limit, dictionary_size); + *output = owned.into_vec(); + result + } + pub fn decode_member( + &mut self, + input: &[u8], + algorithm_version: u8, + output_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member(input, algorithm_version, output_size, solid, mode) + .map(Buffer::into_vec) + } + pub fn decode_member_with_dictionary( + &mut self, + input: &[u8], + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_with_dictionary( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + mode, + ) + .map(Buffer::into_vec) + } + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_from_reader(input, algorithm_version, output_size, solid, mode) + .map(Buffer::into_vec) + } + pub fn decode_member_from_reader_with_dictionary( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_from_reader_with_dictionary( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + mode, + ) + .map(Buffer::into_vec) + } + pub fn decode_member_from_reader_with_dictionary_to_sink( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + sink: impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + self.state.read_control = self.read_control.clone(); + self.state + .decode_member_from_reader_with_dictionary_to_sink( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + sink, + ) + } + #[cfg(all(test, feature = "write"))] + #[allow(clippy::too_many_arguments)] + pub(crate) fn decode_to_sink_with_filters( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + sink: impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + filters: Option<&mut dyn FnMut(PendingFilter) -> std::result::Result<(), E>>, + ) -> std::result::Result<(), StreamDecodeError> { + self.state.read_control = self.read_control.clone(); + self.state.decode_to_sink_with_filters( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + sink, + filters, + ) + } +} + +/// Buffered workers may discard state they will never use. Public and stateful +/// decoding retains history; solid and limited decoding always overrides this. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum HistoryRetention { + Retain, + DiscardIndependent, +} + +#[derive(Debug)] +pub(crate) struct ReaderState { + pub(crate) read_control: crate::read_control::ReadControl, + tables: Option>, + reps: [usize; 4], + last_length: usize, + history: Buffer, +} + +impl ReaderState { + pub(crate) fn allowance(&self) -> B { + self.history.allowance() + } + pub(crate) fn new(allowance: &B) -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + tables: None, + reps: [0; 4], + last_length: 0, + history: Buffer::new(allowance), + } + } + + pub(crate) fn try_clone(&self) -> Result { + self.try_clone_with_spare_capacity(false) + } + + pub(crate) fn try_clone_for_stream(&self) -> Result { + self.try_clone_with_spare_capacity(!B::LIMITED) + } + + fn try_clone_with_spare_capacity(&self, preserve_capacity: bool) -> Result { + Ok(Self { + read_control: self.read_control.clone(), + tables: self + .tables + .as_ref() + .map(ReaderTables::try_clone) + .transpose()?, + reps: self.reps, + last_length: self.last_length, + history: if preserve_capacity { + // Solid streaming checkpoints otherwise discard geometric spare + // capacity and grow again after each member. Keep the capacity + // so a run of small members reuses a few allocation sizes. + let mut history = + Buffer::with_capacity(self.history.capacity(), &self.history.allowance())?; + history + .extend_from_slice(&self.history) + .map_err(Into::into)?; + history + } else { + // Public clones and bounded streaming checkpoints retain their + // existing allocation charge and refusal boundary. + Buffer::copied(&self.history, &self.history.allowance())? + }, + }) + } + pub fn decode_member( + &mut self, + input: &[u8], + algorithm_version: u8, + output_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.read_control.check_codec()?; + self.decode_member_with_dictionary( + input, + algorithm_version, + output_size, + DEFAULT_DICTIONARY_SIZE, + solid, + mode, + ) + } + + pub fn decode_member_with_dictionary( + &mut self, + input: &[u8], + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.decode_member_with_dictionary_and_history( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + mode, + HistoryRetention::Retain, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn decode_member_with_dictionary_and_history( + &mut self, + input: &[u8], + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, + history: HistoryRetention, + ) -> Result> { + self.read_control.check_codec()?; + let mut input = std::io::Cursor::new(input); + self.decode_member_from_reader_with_dictionary_and_history( + &mut input, + algorithm_version, + output_size, + dictionary_size, + solid, + mode, + history, + ) + } + + pub fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + self.decode_member_from_reader_with_dictionary( + input, + algorithm_version, + output_size, + DEFAULT_DICTIONARY_SIZE, + solid, + mode, + ) + } + + pub fn decode_member_from_reader_with_dictionary( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, + ) -> Result> { + self.decode_member_from_reader_with_dictionary_and_history( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + mode, + HistoryRetention::Retain, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn decode_member_from_reader_with_dictionary_and_history( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, + history: HistoryRetention, + ) -> Result> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + if dictionary_size == 0 { + return Err(Error::InvalidData("RAR 5 dictionary size is zero")); + } + if !solid { + self.reset(); + } + + let allowance = self.history.allowance(); + let mut output = + Buffer::with_capacity(output_size.min(MAX_INITIAL_OUTPUT_CAPACITY), &allowance)?; + let mut filters = Buffer::new(&allowance); + + loop { + let block = read_compressed_block_with_allowance(input, &self.history.allowance())?; + let payload = &*block.payload; + let mut payload_bit_pos = 0; + if block.header.has_tables { + let (lengths, table_bits) = read_table_lengths_with_allowance( + payload, + algorithm_version, + &block.payload.allowance(), + )?; + self.tables = Some(ReaderTables::from_lengths( + &lengths, + &block.payload.allowance(), + )?); + payload_bit_pos = table_bits; + } + let tables = self + .tables + .take() + .ok_or(Error::InvalidData("RAR 5 block reuses missing tables"))?; + let mut bits = BitReader::new(payload); + bits.bit_pos = payload_bit_pos; + + let mut poller = self.read_control.poller(); + while bits.bit_pos < block.header.payload_bits && output.len() < output_size { + poller.check_codec(output.len())?; + let symbol = tables.main.decode(&mut bits)?; + match symbol { + 0..=255 => output.try_push(symbol as u8)?, + 256 if mode.uses_lz() => { + filters.try_push(read_filter(&mut bits, output.len())?)?; + } + 257 if mode.uses_lz() => { + if self.last_length != 0 { + self.copy_match( + &mut output, + self.reps[0], + self.last_length, + output_size, + dictionary_size, + )?; + } + } + 258..=261 if mode.uses_lz() => { + let rep_index = symbol - 258; + let distance = self.reps[rep_index]; + if distance == 0 { + return Err(Error::InvalidData( + "RAR 5 repeat distance is not initialized", + )); + } + let length_slot = tables.length.decode(&mut bits)?; + let length_extra = bits.read_bits(length_slot_extra_bits(length_slot))?; + // The length table has 44 symbols and read_bits limits + // the extra value to this slot's declared width. + let length = length_from_slot_parts(length_slot, length_extra); + self.reps[..=rep_index].rotate_right(1); + self.reps[0] = distance; + self.last_length = length; + self.copy_match( + &mut output, + distance, + length, + output_size, + dictionary_size, + )?; + } + 262.. if mode.uses_lz() => { + let length_slot = symbol - 262; + let length_extra = bits.read_bits(length_slot_extra_bits(length_slot))?; + // Main symbols end at slot 43 and read_bits limits the + // extra value to this slot's declared width. + let mut length = length_from_slot_parts(length_slot, length_extra); + let distance_slot = tables.distance.decode(&mut bits)?; + let distance_bit_count = distance_slot_bit_count(distance_slot)?; + let distance_extra = if distance_bit_count >= 4 && tables.align_mode { + let high = bits.read_bits((distance_bit_count - 4) as u8)?; + let low = tables.align.decode(&mut bits)? as u32; + (high << 4) | low + } else { + bits.read_bits(distance_bit_count as u8)? + }; + // distance_slot_bit_count rejected slots above 65 and + // the extra value was read at exactly that width. + let distance = distance_from_slot_parts( + distance_slot, + distance_bit_count, + distance_extra, + ); + length += length_bonus(distance); + self.reps.rotate_right(1); + self.reps[0] = distance; + self.last_length = length; + self.copy_match( + &mut output, + distance, + length, + output_size, + dictionary_size, + )?; + } + _ => { + return Err(Error::InvalidData( + "RAR 5 literal-only decoder encountered non-literal symbol", + )); + } + } + } + + self.tables = Some(tables); + if block.header.is_last || output.len() >= output_size { + break; + } + } + + if output.len() == output_size { + let retain_history = B::LIMITED || solid || history == HistoryRetention::Retain; + let history_output = if retain_history && mode.applies_filters() && !filters.is_empty() + { + Some(Buffer::copied( + &output[output.len().saturating_sub(dictionary_size)..], + &allowance, + )?) + } else { + None + }; + if mode.applies_filters() { + self.read_control.check_codec()?; + apply_filters_with_allowance( + &mut output, + &filters, + &self.read_control, + &allowance, + )?; + } + if retain_history { + self.remember_history( + history_output.as_deref().unwrap_or(&output), + dictionary_size, + )?; + } + Ok(output) + } else { + Err(Error::NeedMoreInput) + } + } + + pub fn decode_member_from_reader_with_dictionary_to_sink( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + sink: impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + self.decode_to_sink_with_filters( + input, + algorithm_version, + output_size, + dictionary_size, + solid, + sink, + None, + ) + } + + // Mirrors the public streaming entry point, adding a filter-record destination. + #[allow(clippy::too_many_arguments)] + pub(crate) fn decode_to_sink_with_filters( + &mut self, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mut sink: impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + mut filters: Option<&mut dyn FnMut(PendingFilter) -> std::result::Result<(), E>>, + ) -> std::result::Result<(), StreamDecodeError> { + self.read_control.check_codec()?; + let control = self.read_control.clone(); + let input = &mut control.reader(input); + if dictionary_size == 0 { + return Err(Error::InvalidData("RAR 5 dictionary size is zero").into()); + } + if !solid { + self.reset(); + } + + // VecDeque grows as decoded bytes arrive, so using the declared + // dictionary here does not allocate a potentially huge RAR 7 window + // up front. It does, however, retain every byte that a legal match may + // reference instead of silently truncating the window at 64 MiB. + let history_limit = dictionary_size; + if self.history.len() > history_limit { + let discard = self.history.len() - history_limit; + self.history.discard_prefix(discard); + } + let mut output = StreamingOutput::new( + { + let allowance = self.history.allowance(); + std::mem::replace(&mut self.history, Buffer::new(&allowance)) + }, + output_size, + dictionary_size, + history_limit, + )?; + + loop { + let block = read_compressed_block_with_allowance(input, &output.history.allowance())?; + let payload = &*block.payload; + let mut payload_bit_pos = 0; + if block.header.has_tables { + let (lengths, table_bits) = read_table_lengths_with_allowance( + payload, + algorithm_version, + &block.payload.allowance(), + )?; + self.tables = Some(ReaderTables::from_lengths( + &lengths, + &block.payload.allowance(), + )?); + payload_bit_pos = table_bits; + } + let tables = self + .tables + .take() + .ok_or(Error::InvalidData("RAR 5 block reuses missing tables"))?; + let mut bits = BitReader::new(payload); + bits.bit_pos = payload_bit_pos; + + let mut poller = self.read_control.poller(); + while bits.bit_pos < block.header.payload_bits && output.written() < output_size { + poller.check_codec(output.written())?; + let symbol = tables.main.decode(&mut bits)?; + match symbol { + 0..=255 => output.push(symbol as u8, &mut sink)?, + 256 => { + let Some(filters) = filters.as_mut() else { + return Err(StreamDecodeError::FilteredMember); + }; + let filter = read_filter(&mut bits, output.written())?; + if filter + .start + .checked_add(filter.length) + .is_none_or(|end| end > output_size) + { + return Err( + Error::InvalidData("RAR 5 filter range exceeds output").into() + ); + } + filters(filter).map_err(StreamDecodeError::Sink)?; + } + 257 => { + if self.last_length != 0 { + output.copy_match(self.reps[0], self.last_length, &mut sink)?; + } + } + 258..=261 => { + let rep_index = symbol - 258; + let distance = self.reps[rep_index]; + if distance == 0 { + return Err(Error::InvalidData( + "RAR 5 repeat distance is not initialized", + ) + .into()); + } + let length_slot = tables.length.decode(&mut bits)?; + let length_extra = bits.read_bits(length_slot_extra_bits(length_slot))?; + // The length table has 44 symbols and read_bits limits + // the extra value to this slot's declared width. + let length = length_from_slot_parts(length_slot, length_extra); + self.reps[..=rep_index].rotate_right(1); + self.reps[0] = distance; + self.last_length = length; + output.copy_match(distance, length, &mut sink)?; + } + 262.. => { + let length_slot = symbol - 262; + let length_extra = bits.read_bits(length_slot_extra_bits(length_slot))?; + // Main symbols end at slot 43 and read_bits limits the + // extra value to this slot's declared width. + let mut length = length_from_slot_parts(length_slot, length_extra); + let distance_slot = tables.distance.decode(&mut bits)?; + let distance_bit_count = distance_slot_bit_count(distance_slot)?; + let distance_extra = if distance_bit_count >= 4 && tables.align_mode { + let high = bits.read_bits((distance_bit_count - 4) as u8)?; + let low = tables.align.decode(&mut bits)? as u32; + (high << 4) | low + } else { + bits.read_bits(distance_bit_count as u8)? + }; + // distance_slot_bit_count rejected slots above 65 and + // the extra value was read at exactly that width. + let distance = distance_from_slot_parts( + distance_slot, + distance_bit_count, + distance_extra, + ); + length += length_bonus(distance); + self.reps.rotate_right(1); + self.reps[0] = distance; + self.last_length = length; + output.copy_match(distance, length, &mut sink)?; + } + } + } + + self.tables = Some(tables); + if block.header.is_last || output.written() >= output_size { + break; + } + } + + if output.written() == output_size { + output.finish(&mut sink)?; + self.history = output.into_history(); + Ok(()) + } else { + Err(Error::NeedMoreInput.into()) + } + } + + fn remember_history(&mut self, output: &[u8], dictionary_size: usize) -> Result<()> { + let incoming = &output[output.len().saturating_sub(dictionary_size)..]; + let keep = self.history.len().min(dictionary_size - incoming.len()); + let required = keep + incoming.len(); + if self.history.capacity() > dictionary_size || self.history.capacity() < required { + // Allocate only the retained tail, never the whole member. Replace + // oversized storage when the active dictionary shrinks as well. + let capacity = + reader_history_capacity(self.history.capacity(), required, dictionary_size); + let mut history = Buffer::with_capacity(capacity, &self.history.allowance())?; + history + .extend_from_slice(&self.history[self.history.len() - keep..]) + .map_err(Into::into)?; + history.extend_from_slice(incoming).map_err(Into::into)?; + self.history = history; + } else { + self.history.discard_prefix(self.history.len() - keep); + self.history + .extend_from_slice(incoming) + .map_err(Into::into)?; + } + Ok(()) + } + + fn reset(&mut self) { + self.tables = None; + self.reps = [0; 4]; + self.last_length = 0; + self.history.clear(); + } + + fn copy_match( + &self, + output: &mut Buffer, + distance: usize, + length: usize, + output_limit: usize, + dictionary_size: usize, + ) -> Result<()> { + if output + .len() + .checked_add(length) + .is_none_or(|end| end > output_limit) + { + return Err(Error::InvalidData("RAR 5 match exceeds output limit")); + } + // A match reaching past the start of the window writes zeroes rather + // than failing. WinRAR never clears its window and guards the copy + // with a first-wrap flag instead, so those bytes read as zero there, + // and an archive that leans on it stays readable here. Nothing is + // swallowed: a stream that is damaged rather than merely odd still + // fails its file hash. + if distance == 0 + || distance > dictionary_size + || distance > self.history.len() + output.len() + { + output.resize(output.len() + length, 0)?; + return Ok(()); + } + let mut remaining = length; + while remaining > 0 { + if distance <= output.len() { + // The match lies entirely in already-decoded output: copy in + // runs rather than one byte at a time. + if distance == 1 { + // A one-byte repeat is a fill, not a copy. + let b = output[output.len() - 1]; + output.resize(output.len() + remaining, b)?; + remaining = 0; + } else { + let start = output.len() - distance; + if remaining <= distance { + return output.extend_from_within(start..start + remaining); + } + // Newly appended bytes are initialized repetitions of + // the same period. Reuse that growing run so short + // overlapping matches take logarithmically many copies. + while remaining > 0 { + let take = remaining.min(output.len() - start); + output.extend_from_within(start..start + take)?; + remaining -= take; + } + return Ok(()); + } + } else { + let history_distance = distance - output.len(); + let index = self.history.len() - history_distance; + let take = remaining.min(history_distance); + output + .extend_from_slice(&self.history[index..index + take]) + .map_err(Into::into)?; + remaining -= take; + } + } + Ok(()) + } +} + +// Preserve amortized growth without letting a retained dictionary allocation +// grow to the member size or keep a previous, larger dictionary alive. +fn reader_history_capacity(current: usize, required: usize, limit: usize) -> usize { + if current > limit { + required + } else { + required.max(current.saturating_mul(2)).max(8).min(limit) + } +} + +struct StreamingOutput { + history: super::workspace::Deque, + pending: Buffer, + written: usize, + output_limit: usize, + dictionary_size: usize, + history_limit: usize, + all_zero: bool, +} + +impl StreamingOutput { + fn new( + mut history: Buffer, + output_limit: usize, + dictionary_size: usize, + history_limit: usize, + ) -> Result { + if history.capacity() > history_limit { + history = Buffer::copied(&history, &history.allowance())?; + } + let allowance = history.allowance(); + Ok(Self { + all_zero: history.iter().all(|&byte| byte == 0), + history: super::workspace::Deque::from_buffer(history), + pending: Buffer::with_capacity(STREAM_FLUSH_THRESHOLD, &allowance)?, + written: 0, + output_limit, + dictionary_size, + history_limit, + }) + } + + fn written(&self) -> usize { + self.written + } + + fn push( + &mut self, + byte: u8, + sink: &mut impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + if self.written >= self.output_limit { + return Err(Error::InvalidData("RAR 5 match exceeds output limit").into()); + } + if byte != 0 { + self.all_zero = false; + } + self.pending.try_push(byte)?; + self.written += 1; + if self.pending.len() >= STREAM_FLUSH_THRESHOLD { + self.flush(sink)?; + } + Ok(()) + } + + fn push_repeated( + &mut self, + byte: u8, + mut count: usize, + sink: &mut impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + if self + .written + .checked_add(count) + .is_none_or(|end| end > self.output_limit) + { + return Err(Error::InvalidData("RAR 5 match exceeds output limit").into()); + } + if byte != 0 { + self.all_zero = false; + } + while count > 0 { + let available = STREAM_FLUSH_THRESHOLD - self.pending.len(); + let take = count.min(available.max(1)); + let old_len = self.pending.len(); + self.pending.resize(old_len + take, byte)?; + self.written += take; + count -= take; + if self.pending.len() >= STREAM_FLUSH_THRESHOLD { + self.flush(sink)?; + } + } + Ok(()) + } + + fn push_zeroes( + &mut self, + count: usize, + sink: &mut impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + if self + .written + .checked_add(count) + .is_none_or(|end| end > self.output_limit) + { + return Err(Error::InvalidData("RAR 5 match exceeds output limit").into()); + } + self.flush(sink)?; + // copy_match only reaches this path with an initialized positive + // distance. Either retained history exists, or flushing the pending + // zero literals retains at least one byte (the dictionary is nonzero). + debug_assert!(!self.history.is_empty()); + sink(DecodedChunk::Repeated { + byte: 0, + len: count, + }) + .map_err(StreamDecodeError::Sink)?; + self.written += count; + Ok(()) + } + + fn copy_match( + &mut self, + distance: usize, + length: usize, + sink: &mut impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + if self.all_zero && distance <= self.written + self.history.len() { + return self.push_zeroes(length, sink); + } + // Zero-fill out-of-window matches, as the buffered decoder does. + if distance == 0 + || distance > self.dictionary_size + || distance > self.history.len() + self.pending.len() + { + return self.push_repeated(0, length, sink); + } + if self + .written + .checked_add(length) + .is_none_or(|end| end > self.output_limit) + { + return Err(Error::InvalidData("RAR 5 match exceeds output limit").into()); + } + if distance == 1 { + let byte = self.byte_at_distance(1); + return self.push_repeated(byte, length, sink); + } + let mut remaining = length; + while remaining != 0 { + let available = STREAM_FLUSH_THRESHOLD - self.pending.len(); + if distance <= self.pending.len() { + // Seed the repeated run from the preceding distance bytes. + // Each append makes a larger initialized run available, so + // overlapping short-distance matches double instead of + // copying one period (or one byte) at a time. + let start = self.pending.len() - distance; + let mut run = remaining.min(available); + while run != 0 { + let take = run.min(self.pending.len() - start); + self.pending.extend_from_within(start..start + take)?; + self.written += take; + remaining -= take; + run -= take; + } + } else { + // VecDeque history can wrap into two slices. Copy only the + // current contiguous part; once the source reaches pending, + // the initialized-run path above handles the overlap. + let history_distance = distance - self.pending.len(); + let start = self.history.len() - history_distance; + let (first, second) = self.history.as_slices(); + let source = if start < first.len() { + &first[start..] + } else { + &second[start - first.len()..] + }; + let take = remaining.min(available).min(source.len()); + self.pending + .extend_from_slice(&source[..take]) + .map_err(|error| StreamDecodeError::Decode(error.into()))?; + self.written += take; + remaining -= take; + } + if self.pending.len() == STREAM_FLUSH_THRESHOLD { + self.flush(sink)?; + } + } + Ok(()) + } + + fn byte_at_distance(&self, distance: usize) -> u8 { + // copy_match admits only distances inside the current window. Every + // copied byte extends that window; flush retains at least distance + // bytes because distance is bounded by dictionary_size. + if distance <= self.pending.len() { + self.pending[self.pending.len() - distance] + } else { + let history_distance = distance - self.pending.len(); + self.history[self.history.len() - history_distance] + } + } + + fn flush( + &mut self, + sink: &mut impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + if self.pending.is_empty() { + return Ok(()); + } + sink(DecodedChunk::Bytes(&self.pending)).map_err(StreamDecodeError::Sink)?; + let incoming = &self.pending[self.pending.len().saturating_sub(self.history_limit)..]; + let keep = self.history.len().min(self.history_limit - incoming.len()); + let required = keep + incoming.len(); + if self.history.capacity() > self.history_limit || self.history.capacity() < required { + let capacity = + reader_history_capacity(self.history.capacity(), required, self.history_limit); + let mut history = + super::workspace::Deque::with_capacity(capacity, &self.history.allowance())?; + history.extend_admitted(self.history.iter().skip(self.history.len() - keep).copied()); + history.extend_admitted(incoming.iter().copied()); + self.history = history; + } else { + self.history.discard_prefix(self.history.len() - keep); + self.history.extend_admitted(incoming.iter().copied()); + } + self.pending.clear(); + Ok(()) + } + + fn finish( + &mut self, + sink: &mut impl FnMut(DecodedChunk<'_>) -> std::result::Result<(), E>, + ) -> std::result::Result<(), StreamDecodeError> { + self.flush(sink) + } + + fn into_history(self) -> Buffer { + self.history.into_buffer() + } +} + +#[cfg(all(test, feature = "write"))] +fn read_compressed_block(input: &mut impl Read) -> Result { + read_compressed_block_with_allowance(input, &Allowance::default()) +} + +fn read_compressed_block_with_allowance( + input: &mut impl Read, + allowance: &B, +) -> Result> { + let mut fixed = [0u8; 2]; + input + .read_exact(&mut fixed) + .map_err(Error::from_read_error)?; + let flags = fixed[0]; + let checksum = fixed[1]; + let size_bytes_len = match (flags >> 3) & 0x03 { + 0 => 1, + 1 => 2, + 2 => 3, + _ => return Err(Error::InvalidData("RAR 5 block size length is invalid")), + }; + let mut size_bytes = [0u8; 3]; + input + .read_exact(&mut size_bytes[..size_bytes_len]) + .map_err(Error::from_read_error)?; + + let actual = size_bytes[..size_bytes_len] + .iter() + .fold(checksum ^ flags, |acc, &byte| acc ^ byte); + if actual != 0x5a { + return Err(Error::InvalidData("RAR 5 block header checksum mismatch")); + } + + let payload_size = size_bytes[..size_bytes_len] + .iter() + .enumerate() + .fold(0usize, |acc, (index, &byte)| { + acc | (usize::from(byte) << (index * 8)) + }); + let mut payload = Buffer::filled(payload_size, 0, allowance)?; + input + .read_exact(&mut payload) + .map_err(Error::from_read_error)?; + let final_byte_bits = ((flags & 0x07) + 1).min(8); + let payload_bits = if payload_size == 0 { + 0 + } else { + (payload_size - 1) * 8 + usize::from(final_byte_bits) + }; + + Ok(OwnedCompressedBlock { + header: CompressedBlockHeader { + flags, + is_last: flags & 0x40 != 0, + has_tables: flags & 0x80 != 0, + final_byte_bits, + payload_size, + payload_bits, + }, + payload, + }) +} + +impl Default for Unpack50Decoder { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct PendingFilter { + pub(crate) start: usize, + pub(crate) length: usize, + pub(crate) filter_type: FilterType, + pub(crate) channels: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FilterType { + Delta, + E8, + E8E9, + Arm, +} + +fn read_filter(bits: &mut BitReader<'_>, current_pos: usize) -> Result { + let offset = read_filter_data(bits)? as usize; + let length = read_filter_data(bits)? as usize; + let filter_type = match bits.read_bits(3)? { + 0 => FilterType::Delta, + 1 => FilterType::E8, + 2 => FilterType::E8E9, + 3 => FilterType::Arm, + _ => return Err(Error::InvalidData("RAR 5 filter type is unsupported")), + }; + let channels = if filter_type == FilterType::Delta { + bits.read_bits(5)? as usize + 1 + } else { + 0 + }; + Ok(PendingFilter { + start: current_pos + .checked_add(offset) + .ok_or(Error::InvalidData("RAR 5 filter start overflows"))?, + length, + filter_type, + channels, + }) +} + +fn read_filter_data(bits: &mut BitReader<'_>) -> Result { + let byte_count = bits.read_bits(2)? as usize + 1; + let mut data = 0; + for index in 0..byte_count { + data |= bits.read_bits(8)? << (index * 8); + } + Ok(data) +} + +#[cfg(all(test, feature = "write"))] +fn apply_filters_with_control( + output: &mut [u8], + filters: &[PendingFilter], + control: &crate::read_control::ReadControl, +) -> Result<()> { + apply_filters_with_allowance(output, filters, control, &Allowance::default()) +} +fn apply_filters_with_allowance( + output: &mut [u8], + filters: &[PendingFilter], + control: &crate::read_control::ReadControl, + allowance: &B, +) -> Result<()> { + control.check_codec()?; + for filter in filters { + control.check_codec()?; + let end = filter + .start + .checked_add(filter.length) + .ok_or(Error::InvalidData("RAR 5 filter range overflows"))?; + let data = output + .get_mut(filter.start..end) + .ok_or(Error::InvalidData("RAR 5 filter range exceeds output"))?; + apply_filter_data_with_allowance(data, filter, control, allowance)?; + } + Ok(()) +} + +#[cfg(all(test, feature = "write"))] +pub(crate) fn apply_filter_data( + data: &mut [u8], + filter: &PendingFilter, + control: &crate::read_control::ReadControl, +) -> Result<()> { + apply_filter_data_with_allowance(data, filter, control, &Allowance::default()) +} +pub(crate) fn apply_filter_data_with_allowance( + data: &mut [u8], + filter: &PendingFilter, + control: &crate::read_control::ReadControl, + allowance: &B, +) -> Result<()> { + match filter.filter_type { + FilterType::Delta => { + let decoded = filters::delta_decode_with_allowance( + data, + filter.channels, + rar50_delta_messages(), + control, + allowance, + )?; + data.copy_from_slice(&decoded); + } + FilterType::E8 => e8e9_decode_with_control(data, filter.start as u32, false, control)?, + FilterType::E8E9 => e8e9_decode_with_control(data, filter.start as u32, true, control)?, + FilterType::Arm => arm_decode_with_control(data, filter.start as u32, control)?, + } + Ok(()) +} + +fn rar50_delta_messages() -> DeltaErrorMessages { + DeltaErrorMessages { + invalid_channels: "RAR 5 DELTA filter channel count is invalid", + zero_channels: "RAR 5 DELTA filter has zero channels", + truncated_source: "RAR 5 DELTA filter source is truncated", + } +} + +#[cfg(all(test, feature = "write"))] +fn e8e9_decode(data: &mut [u8], file_offset: u32, include_e9: bool) { + e8e9_decode_with_control( + data, + file_offset, + include_e9, + &crate::read_control::ReadControl::default(), + ) + .expect("uncancelled filter"); +} + +fn e8e9_decode_with_control( + data: &mut [u8], + file_offset: u32, + include_e9: bool, + control: &crate::read_control::ReadControl, +) -> Result<()> { + control.check_codec()?; + let mut poller = control.poller(); + if data.len() <= 4 { + return Ok(()); + } + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let opcode_limit = data.len() - 4; + let mut opcode_pos = 0usize; + while opcode_pos < opcode_limit { + poller.check_codec(opcode_pos)?; + let scan_end = if control.is_enabled() { + opcode_limit.min(opcode_pos.saturating_add(64 * 1024)) + } else { + opcode_limit + }; + let Some(pos) = super::fast::next_x86_opcode(data, opcode_pos, scan_end, cmp_mask) else { + opcode_pos = scan_end; + continue; + }; + let cur_pos = pos + 1; + let offset = file_offset.wrapping_add(cur_pos as u32) % X86_FILTER_FILE_SIZE; + let addr = u32::from_le_bytes([ + data[cur_pos], + data[cur_pos + 1], + data[cur_pos + 2], + data[cur_pos + 3], + ]); + let new_addr = if addr & 0x8000_0000 != 0 { + (addr.wrapping_add(offset) & 0x8000_0000 == 0) + .then(|| addr.wrapping_add(X86_FILTER_FILE_SIZE)) + } else { + (addr.wrapping_sub(X86_FILTER_FILE_SIZE) & 0x8000_0000 != 0) + .then(|| addr.wrapping_sub(offset)) + }; + if let Some(value) = new_addr { + data[cur_pos..cur_pos + 4].copy_from_slice(&value.to_le_bytes()); + } + opcode_pos = pos + 5; + } + + Ok(()) +} + +const X86_FILTER_FILE_SIZE: u32 = 0x0100_0000; + +#[cfg(all(test, feature = "write"))] +fn arm_decode(data: &mut [u8], file_offset: u32) { + arm_decode_with_control( + data, + file_offset, + &crate::read_control::ReadControl::default(), + ) + .expect("uncancelled filter"); +} + +fn arm_decode_with_control( + data: &mut [u8], + file_offset: u32, + control: &crate::read_control::ReadControl, +) -> Result<()> { + control.check_codec()?; + let mut poller = control.poller(); + let mut pos = 0usize; + while pos + 3 < data.len() { + poller.check_codec(pos)?; + if data[pos + 3] == 0xeb { + let mut offset = u32::from(data[pos]) + | (u32::from(data[pos + 1]) << 8) + | (u32::from(data[pos + 2]) << 16); + offset = offset.wrapping_sub(file_offset.wrapping_add(pos as u32) / 4); + data[pos] = offset as u8; + data[pos + 1] = (offset >> 8) as u8; + data[pos + 2] = (offset >> 16) as u8; + } + pos += 4; + } + + Ok(()) +} + +fn length_slot_extra_bits(slot: usize) -> u8 { + if slot < 8 { + 0 + } else { + ((slot >> 2) - 1) as u8 + } +} + +fn length_bonus(distance: usize) -> usize { + usize::from(distance > 0x100) + usize::from(distance > 0x2000) + usize::from(distance > 0x40000) +} + +pub fn slot_to_length(slot: usize, extra_bits: u32) -> Result { + if slot < 8 { + return Ok(slot + 2); + } + let bit_count = (slot >> 2) - 1; + if bit_count > 24 { + return Err(Error::InvalidData("RAR 5 length slot is too large")); + } + let max_extra = (1u32 << bit_count) - 1; + if extra_bits > max_extra { + return Err(Error::InvalidData("RAR 5 length extra bits exceed slot")); + } + Ok(length_from_slot_parts(slot, extra_bits)) +} + +fn length_from_slot_parts(slot: usize, extra_bits: u32) -> usize { + if slot < 8 { + slot + 2 + } else { + let bit_count = (slot >> 2) - 1; + debug_assert!(bit_count <= 24); + debug_assert!(extra_bits < 1u32 << bit_count); + (((4 | (slot & 3)) << bit_count) | extra_bits as usize) + 2 + } +} + +pub fn distance_slot_bit_count(slot: usize) -> Result { + if slot < 4 { + Ok(0) + } else { + let bit_count = (slot - 2) >> 1; + if bit_count > 31 { + Err(Error::InvalidData("RAR 5 distance slot is too large")) + } else { + Ok(bit_count) + } + } +} + +pub fn slot_to_distance(slot: usize, extra_bits: u32) -> Result { + if slot < 4 { + return Ok(slot + 1); + } + let bit_count = distance_slot_bit_count(slot)?; + let max_extra = (1u32 << bit_count) - 1; + if extra_bits > max_extra { + return Err(Error::InvalidData("RAR 5 distance extra bits exceed slot")); + } + Ok(distance_from_slot_parts(slot, bit_count, extra_bits)) +} + +fn distance_from_slot_parts(slot: usize, bit_count: usize, extra_bits: u32) -> usize { + if slot < 4 { + return slot + 1; + } + debug_assert!(bit_count <= 31); + debug_assert!(extra_bits < 1u32 << bit_count); + let distance = (((2u64 | (slot & 1) as u64) << bit_count) | u64::from(extra_bits)) + 1; + // RAR 5 slots can name more than a 32-bit host can address. Preserve the + // decoder's out-of-window zero-fill behavior instead of wrapping to a + // plausible distance (or panicking on arithmetic overflow). + usize::try_from(distance).unwrap_or(usize::MAX) +} + +#[derive(Debug)] +pub struct HuffmanTable { + state: HuffmanState, +} +impl Clone for HuffmanTable { + fn clone(&self) -> Self { + Self { + state: self.state.try_clone().expect("unlimited table copy"), + } + } +} +impl HuffmanTable { + pub fn from_lengths(lengths: &[u8]) -> Result { + Ok(Self { + state: HuffmanState::from_lengths(lengths, &Allowance::default())?, + }) + } + pub fn is_empty(&self) -> bool { + self.state.is_empty() + } + #[cfg(all(test, feature = "write"))] + fn decode(&self, bits: &mut BitReader<'_>) -> Result { + self.state.decode(bits) + } +} +#[derive(Debug)] +struct HuffmanState { + state: super::canonical::Huffman, +} + +impl HuffmanState { + pub fn is_empty(&self) -> bool { + self.state.is_empty() + } + fn from_lengths(lengths: &[u8], allowance: &B) -> Result { + let mut count = [0u16; 16]; + for &length in lengths { + if length > 15 { + return Err(Error::InvalidData("RAR 5 Huffman length is too large")); + } + if length != 0 { + count[length as usize] = count[length as usize] + .checked_add(1) + .ok_or(Error::InvalidData("RAR 5 oversubscribed Huffman table"))?; + } + } + validate_huffman_counts(&count)?; + + Ok(Self { + state: super::canonical::Huffman::from_counts(lengths, count, allowance)?, + }) + } + fn try_clone(&self) -> Result { + Ok(Self { + state: self.state.try_clone()?, + }) + } + fn decode(&self, bits: &mut BitReader<'_>) -> Result { + self.state.decode( + || bits.read_bits(1).map(|bit| bit as u16), + "RAR 5 empty Huffman table", + "RAR 5 invalid Huffman code", + ) + } +} + +struct BitReader<'a> { + input: &'a [u8], + bit_pos: usize, +} + +impl<'a> BitReader<'a> { + fn new(input: &'a [u8]) -> Self { + Self { input, bit_pos: 0 } + } + + #[inline] + fn read_bits(&mut self, count: u8) -> Result { + let count = usize::from(count); + let byte_pos = self.bit_pos / 8; + let bit_offset = self.bit_pos % 8; + // Compare the local byte span instead of multiplying the entire input + // length by eight, which can overflow for a huge slice. + let bytes_needed = (bit_offset + count).div_ceil(8); + if bytes_needed > self.input.len().saturating_sub(byte_pos) { + return Err(Error::NeedMoreInput); + } + + if count == 0 { + return Ok(0); + } + if count <= 32 { + // Read the at-most-five physical bytes once, then discard the + // leading and trailing bits. No speculative padding is read, and + // input refusal leaves the cursor at its original position. + let bytes = &self.input[byte_pos..byte_pos + bytes_needed]; + let word = match bytes { + [a] => u64::from(*a), + [a, b] => u64::from(u16::from_be_bytes([*a, *b])), + [a, b, c] => u64::from(u32::from_be_bytes([0, *a, *b, *c])), + [a, b, c, d] => u64::from(u32::from_be_bytes([*a, *b, *c, *d])), + [a, b, c, d, e] => u64::from_be_bytes([0, 0, 0, *a, *b, *c, *d, *e]), + _ => unreachable!("a nonempty 32-bit read spans at most five bytes"), + }; + let shift = bytes_needed * 8 - bit_offset - count; + let value = ((word >> shift) & ((1u64 << count) - 1)) as u32; + self.bit_pos += count; + return Ok(value); + } + + // Preserve the existing low-32-bit result for larger internal reads. + let mut value = 0u32; + let mut remaining = count; + while remaining != 0 { + let byte = self.input[self.bit_pos / 8]; + let bit_offset = self.bit_pos % 8; + let available = 8 - bit_offset; + let take = available.min(remaining); + let shift = available - take; + let mask = ((1u16 << take) - 1) as u8; + let chunk = (byte >> shift) & mask; + value = (value << take) | u32::from(chunk); + self.bit_pos += take; + remaining -= take; + } + + Ok(value) + } +} + +fn validate_huffman_counts(count: &[u16; 16]) -> Result<()> { + super::canonical::validate_counts(count, "RAR 5 oversubscribed Huffman table") +} + +#[cfg(test)] +#[path = "rar50/streaming_copy_tests.rs"] +mod streaming_copy_tests; + +#[cfg(test)] +#[path = "rar50/huffman_tests.rs"] +mod huffman_tests; + +#[cfg(test)] +#[path = "rar50/bit_reader_tests.rs"] +mod bit_reader_tests; + +#[cfg(test)] +#[path = "rar50/checkpoint_tests.rs"] +mod checkpoint_tests; + +#[cfg(all(test, feature = "write"))] +#[path = "rar50/history_retention_tests.rs"] +mod history_retention_tests; diff --git a/arca-rar/vendor/rars/src/codec/rar50/bit_reader_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/bit_reader_tests.rs new file mode 100644 index 00000000..3cfd622a --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/bit_reader_tests.rs @@ -0,0 +1,78 @@ +use super::{BitReader, Error, Result}; + +fn bitwise_read(input: &[u8], cursor: &mut usize, count: u8) -> Result { + let count = usize::from(count); + let byte = *cursor / 8; + let span = (*cursor % 8 + count).div_ceil(8); + if span > input.len().saturating_sub(byte) { + return Err(Error::NeedMoreInput); + } + let mut value = 0u32; + for _ in 0..count { + value = (value << 1) | u32::from((input[*cursor / 8] >> (7 - *cursor % 8)) & 1); + *cursor += 1; + } + Ok(value) +} + +#[test] +fn word_reads_match_independent_bits_at_every_alignment_and_physical_tail() { + for input in [ + [0; 40], + [0xff; 40], + std::array::from_fn(|index| (index as u8).wrapping_mul(37).wrapping_add(11)), + ] { + for size in 0..=input.len() { + for start in 0..=size * 8 + 16 { + for count in 0..=40 { + let mut expected_cursor = start; + let expected = bitwise_read(&input[..size], &mut expected_cursor, count); + let mut actual = BitReader { + input: &input[..size], + bit_pos: start, + }; + assert_eq!( + actual.read_bits(count), + expected, + "size={size}, start={start}, count={count}" + ); + assert_eq!(actual.bit_pos, expected_cursor); + } + } + } + } +} + +#[test] +fn word_reads_preserve_mixed_sequences_and_wider_low_word_results() { + let input: Vec<_> = (0..4096usize) + .map(|index| (index as u8).wrapping_mul(71).wrapping_add(29)) + .collect(); + let mut actual = BitReader::new(&input); + let mut expected_cursor = 0; + for count in [1, 4, 0, 8, 15, 26, 32, 33, 64, 127, 255] + .into_iter() + .cycle() + .take(500) + { + let expected = bitwise_read(&input, &mut expected_cursor, count); + assert_eq!(actual.read_bits(count), expected); + assert_eq!(actual.bit_pos, expected_cursor); + } +} + +#[test] +fn word_read_refusal_retains_cursor_near_maximum_usize() { + for start in [usize::MAX, usize::MAX - 7, usize::MAX - 15] { + for count in [0, 1, 15, 32, 255] { + let mut expected_cursor = start; + let expected = bitwise_read(&[], &mut expected_cursor, count); + let mut actual = BitReader { + input: &[], + bit_pos: start, + }; + assert_eq!(actual.read_bits(count), expected); + assert_eq!(actual.bit_pos, start); + } + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/checkpoint_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/checkpoint_tests.rs new file mode 100644 index 00000000..9fce61c6 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/checkpoint_tests.rs @@ -0,0 +1,118 @@ +use super::*; + +#[test] +fn unlimited_checkpoint_retains_capacity_and_independent_history() { + let mut original = ReaderState::new(&Allowance::default()); + original.history = Buffer::with_capacity(16, &Allowance::default()).unwrap(); + original.history.extend_from_slice(b"ABCDE").unwrap(); + original.reps = [1, 2, 3, 4]; + original.last_length = 7; + let public_copy = original.try_clone().unwrap(); + assert_eq!(public_copy.history.capacity(), 5); + assert_eq!(&*public_copy.history, b"ABCDE"); + let mut copied = original.try_clone_for_stream().unwrap(); + assert_eq!(copied.history.capacity(), 16); + assert_eq!(&*copied.history, b"ABCDE"); + assert_ne!(copied.history.as_ptr(), original.history.as_ptr()); + assert_eq!(copied.reps, original.reps); + assert_eq!(copied.last_length, original.last_length); + copied.history[0] = b'X'; + assert_eq!(&*original.history, b"ABCDE"); + drop(original); + assert_eq!(&*copied.history, b"XBCDE"); +} + +#[test] +fn limited_checkpoint_keeps_its_existing_length_charge_and_refusal() { + for limit in [20, 21] { + let budget = Allowance::limited(limit); + let mut original = ReaderState::new(&budget); + original.history = Buffer::with_capacity(16, &budget).unwrap(); + original.history.extend_from_slice(b"ABCDE").unwrap(); + assert_eq!(budget.used(), 16); + match original.try_clone_for_stream() { + Ok(copied) => { + assert_eq!(limit, 21); + assert_eq!(copied.history.capacity(), 5); + assert_eq!(&*copied.history, b"ABCDE"); + assert_ne!(copied.history.as_ptr(), original.history.as_ptr()); + assert_eq!(budget.used(), 21); + drop(copied); + } + Err(error) => { + assert_eq!(limit, 20); + assert!(matches!(error, Error::WorkspaceLimitExceeded(_))); + } + } + assert_eq!(budget.used(), 16); + assert_eq!(&*original.history, b"ABCDE"); + assert_eq!(original.history.capacity(), 16); + drop(original); + assert_eq!(budget.used(), 0); + } +} + +#[test] +fn refused_checkpoint_preserves_the_previous_history_owner() { + let budget = super::super::workspace::RefusingBudget::new(1); + let mut original = ReaderState::new(&budget); + original.history = Buffer::with_capacity(16, &budget).unwrap(); + original.history.extend_from_slice(b"ABCDE").unwrap(); + let address = original.history.as_ptr(); + assert!(matches!( + original.try_clone_for_stream(), + Err(Error::Cancelled) + )); + assert_eq!(original.history.as_ptr(), address); + assert_eq!(original.history.capacity(), 16); + assert_eq!(&*original.history, b"ABCDE"); + assert_eq!(budget.used(), 16); + drop(original); + assert_eq!(budget.used(), 0); +} + +#[cfg(feature = "write")] +#[test] +fn solid_streaming_checkpoints_keep_geometric_capacity_between_small_members() { + use std::collections::BTreeSet; + let dictionary = 16 * 1024; + let data = b"solid checkpoint history and repeated match bytes\n".repeat(20); + let mut expected_history = Vec::new(); + let mut capacities = BTreeSet::new(); + let mut decoder = ReaderState::new(&Allowance::default()); + for member in 0..40 { + let packed = encode_lz_member_with_history(&data, &expected_history, 0).unwrap(); + let mut checkpoint = decoder.try_clone_for_stream().unwrap(); + assert_eq!(checkpoint.history.capacity(), decoder.history.capacity()); + let mut decoded = Vec::new(); + checkpoint + .decode_member_from_reader_with_dictionary_to_sink( + &mut packed.as_slice(), + 0, + data.len(), + dictionary, + member != 0, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + assert_eq!(decoded, data); + expected_history.extend_from_slice(&data); + let discard = expected_history.len().saturating_sub(dictionary); + expected_history.drain(..discard); + assert_eq!(&*checkpoint.history, &expected_history); + assert!(checkpoint.history.capacity() <= dictionary); + capacities.insert(checkpoint.history.capacity()); + decoder = checkpoint; + } + // Each checkpoint retains the preceding geometric capacity. Copying only + // the length creates a fresh capacity on almost every small solid member. + assert!(capacities.len() <= 6, "capacities: {capacities:?}"); +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/convergence_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/convergence_tests.rs new file mode 100644 index 00000000..8dd07495 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/convergence_tests.rs @@ -0,0 +1,328 @@ +use super::*; +use std::cell::Cell; + +thread_local! { + static PRICING_PASSES: Cell = const { Cell::new(0) }; +} + +pub(super) fn record_pricing_pass() { + PRICING_PASSES.with(|count| count.set(count.get() + 1)); +} + +fn count_passes(run: impl FnOnce() -> T) -> (T, usize) { + PRICING_PASSES.with(|count| count.set(0)); + let result = run(); + (result, PRICING_PASSES.with(Cell::get)) +} + +#[derive(Debug)] +struct Reprice { + changed: bool, + tables_converged: bool, + bits: usize, + best_bits_before: usize, +} + +struct Reference { + selected: Vec, + passes: usize, + reprices: Vec, + selected_pass: usize, +} + +// Independent original three-pass pipeline, including its strict winner and +// earlier-winner-on-tie behavior. It deliberately does not stop on equal tables. +fn reference_pipeline( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + filters: &[EncodeFilter], +) -> Reference { + let allowance = Allowance::default(); + let mut collector = OptimalCollector::new(combined, block.start, options); + let matches = collector.collect(combined, block.clone(), options).unwrap(); + let mut workspace = OptimalWorkspace::new(&allowance); + let mut tokens = optimal_tokens_in_workspace( + combined, + block.clone(), + options, + distance_size, + None, + &matches, + &mut workspace, + ) + .unwrap(); + let mut passes = 1; + let mut lengths = + table_lengths_with_allowance(&tokens, filters, distance_size, &allowance).unwrap(); + let mut best_bits = token_stream_bits(&tokens, filters, &lengths, distance_size).unwrap(); + let mut best = None; + let mut selected_pass = 0; + let mut reprices = Vec::new(); + for _ in 1..OPTIMAL_PARSE_PASSES { + let prices = TokenPrices { + lengths: lengths.slices(), + }; + let next = optimal_tokens_in_workspace( + combined, + block.clone(), + options, + distance_size, + Some(&prices), + &matches, + &mut workspace, + ) + .unwrap(); + passes += 1; + if next == tokens { + reprices.push(Reprice { + changed: false, + tables_converged: true, + bits: best_bits, + best_bits_before: best_bits, + }); + break; + } + let next_lengths = + table_lengths_with_allowance(&next, filters, distance_size, &allowance).unwrap(); + let tables_converged = lengths.slices() == next_lengths.slices(); + lengths = next_lengths; + let bits = token_stream_bits(&next, filters, &lengths, distance_size).unwrap(); + reprices.push(Reprice { + changed: true, + tables_converged, + bits, + best_bits_before: best_bits, + }); + if bits < best_bits { + best_bits = bits; + best = None; + selected_pass = passes - 1; + } else if best.is_none() { + best = Some(std::mem::replace(&mut tokens, Buffer::new(&allowance))); + } + tokens = next; + } + Reference { + selected: best.unwrap_or(tokens).into_vec(), + passes, + reprices, + selected_pass, + } +} + +fn generated_noise(seed: u32, size: usize, alphabet: u8) -> Vec { + let mut state = seed; + (0..size) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + ((state >> 24) as u8) % alphabet + }) + .collect() +} + +fn compare_pipeline( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + algorithm_version: u8, + filters: &[EncodeFilter], +) -> (Reference, usize) { + assert!(filters + .iter() + .all(|filter| filter.offset + filter.length <= block.len())); + let distance_size = if algorithm_version == 0 { + DISTANCE_TABLE_SIZE_50 + } else { + DISTANCE_TABLE_SIZE_70 + }; + let (reference, reference_passes) = count_passes(|| { + reference_pipeline(combined, block.clone(), options, distance_size, filters) + }); + assert_eq!(reference.passes, reference_passes); + let (actual, actual_passes) = count_passes(|| { + encode_tokens_with_allowance( + combined, + block.clone(), + MemberSearch::Fresh, + options, + distance_size, + filters, + None, + &Allowance::default(), + ) + .unwrap() + }); + assert_eq!( + &*actual, + reference.selected, + "start={}, candidates={}, version={algorithm_version}, filters={}", + block.start, + options.max_match_candidates, + filters.len() + ); + assert!(actual_passes <= reference.passes); + let expected_full: Vec<_> = filters + .iter() + .copied() + .map(EncodeToken::Filter) + .chain(reference.selected.iter().copied()) + .collect(); + let actual_full: Vec<_> = filters + .iter() + .copied() + .map(EncodeToken::Filter) + .chain(actual.iter().copied()) + .collect(); + assert_eq!( + encode_token_block(&actual_full, algorithm_version, distance_size, true).unwrap(), + encode_token_block(&expected_full, algorithm_version, distance_size, true).unwrap(), + ); + (reference, actual_passes) +} + +#[test] +fn equal_tables_with_changed_tokens_skip_a_real_pricing_pass() { + // The first Huffman-priced pass chooses a different token sequence, but + // generates exactly the same four tables. The old third pass recreates + // that sequence and stops; the new pipeline avoids that work entirely. + let input = generated_noise(92, 256, 2); + let options = EncodeOptions::new(16).with_optimal_parse(true); + for version in [0, 1] { + let (reference, actual_passes) = + compare_pipeline(&input, 0..input.len(), options, version, &[]); + assert_eq!(reference.passes, 3); + assert_eq!(actual_passes, 2); + assert!(reference.reprices[0].changed); + assert!(reference.reprices[0].tables_converged); + assert!(!reference.reprices[1].changed); + assert_eq!( + reference.reprices[0].bits + 3, + reference.reprices[0].best_bits_before + ); + if version == 0 { + assert_eq!(reference.reprices[0].best_bits_before, 575); + assert_eq!(reference.reprices[0].bits, 572); + } + assert_eq!(reference.selected_pass, 1); + } +} + +#[test] +fn convergence_preserves_the_earlier_winner_on_ties_and_losses() { + let options = EncodeOptions::new(16).with_optimal_parse(true); + let mut saw_final_tie = false; + let mut saw_final_loss = false; + for version in [0, 1] { + for seed in 0..128 { + let input = generated_noise(seed, 256, 2); + let (reference, _) = compare_pipeline(&input, 0..input.len(), options, version, &[]); + let last = reference.reprices.last().unwrap(); + if last.changed && reference.selected_pass < reference.passes - 1 { + saw_final_tie |= last.bits == last.best_bits_before; + saw_final_loss |= last.bits > last.best_bits_before; + } + // A first-pass tie must not displace the earlier winner, while + // the later strict improvement must still be selected. + if version == 0 && seed == 3 { + assert!(reference.reprices[0].changed); + assert_eq!( + reference.reprices[0].bits, + reference.reprices[0].best_bits_before + ); + assert_eq!(reference.selected_pass, 2); + } + // A losing reprice followed by an improvement must not leave the + // earlier retained stream in place after that improvement. + if version == 0 && seed == 1 { + assert!(reference.reprices[0].bits > reference.reprices[0].best_bits_before); + assert!(reference.reprices[1].bits < reference.reprices[1].best_bits_before); + assert_eq!(reference.selected_pass, 2); + } + } + } + assert!( + saw_final_tie, + "corpus must retain an earlier winner on a final tie" + ); + assert!( + saw_final_loss, + "corpus must retain an earlier winner on a final loss" + ); +} + +#[test] +fn convergence_matches_the_original_pipeline_with_filters_and_history() { + let filters = [ + EncodeFilter { + offset: 0, + length: 64, + filter_type: FilterType::Delta, + channels: 4, + }, + EncodeFilter { + offset: 64, + length: 64, + filter_type: FilterType::Arm, + channels: 0, + }, + ]; + let mut text = Vec::new(); + for i in 0..80 { + text.extend_from_slice(b"repeated text with changing tails exposes shorter continuations "); + text.push(i); + } + let corpus = [ + vec![0; 4096], + b"abc".repeat(1366), + text, + generated_noise(92, 256, 2), + generated_noise(23, 4096, 2), + generated_noise(17, 4096, 255), + (0..1024u32) + .flat_map(|word| (word * 71).to_le_bytes()) + .collect(), + ]; + for data in corpus { + for start in [0, 128] { + for candidates in [0, 16, 128] { + let options = EncodeOptions::new(candidates).with_optimal_parse(true); + for version in [0, 1] { + for records in [&[][..], &filters[..]] { + compare_pipeline(&data, start..data.len(), options, version, records); + } + } + } + } + } +} + +#[test] +fn converged_pricing_keeps_the_final_progress_and_cancellation_boundary() { + let input = generated_noise(92, 256, 2); + let options = EncodeOptions::new(16).with_optimal_parse(true); + for continue_work in [true, false] { + let mut reports = Vec::new(); + let mut report = |amount| { + reports.push(amount); + continue_work + }; + let result = encode_tokens_with_allowance( + &input, + 0..input.len(), + MemberSearch::Fresh, + options, + DISTANCE_TABLE_SIZE_50, + &[], + Some(&mut report), + &Allowance::default(), + ); + assert_eq!(reports, [input.len()]); + if continue_work { + assert!(result.is_ok()); + } else { + assert!(matches!(result, Err(Error::Cancelled))); + } + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/encoder.rs b/arca-rar/vendor/rars/src/codec/rar50/encoder.rs new file mode 100644 index 00000000..cf296202 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/encoder.rs @@ -0,0 +1,3473 @@ +//! RAR5/7 LZ encoding, block selection, token pricing and table emission. +use super::super::{huffman, match_finder}; +use super::*; +#[cfg(test)] +use std::io::Write; + +/// Matches shorter than 4 bytes are never emitted, so candidate positions are +/// chained by a hash of their first 4 bytes. +type Rar50MatchFinder = match_finder::MatchFinder<4, B>; + +const MAX_ENCODER_MATCH_OFFSET: usize = DEFAULT_DICTIONARY_SIZE; + +const MAX_ENCODER_MATCH_LENGTH: usize = 4096; + +/// The largest block the format allows a writer to emit. +const MAX_COMPRESSED_BLOCK_OUTPUT: usize = 4 * 1024 * 1024; + +/// How much input goes into one compressed block. +/// +/// Every block carries its own Huffman tables, so smaller blocks pay for the +/// extra tables and win back more by fitting each stretch of the data. Matches +/// still reach back across boundaries into the history, so shortening a block +/// costs no match range. Measured over the corpus, 64 KiB packs 6.4% smaller +/// than a mebibyte and within 0.2% of the best size tried at any point between +/// 16 KiB and 256 KiB. The streaming writer reads in the same units, so both +/// paths produce the same blocks for the same input. +pub(crate) const LZ_BLOCK_SIZE: usize = 64 * 1024; + +const _: () = assert!(LZ_BLOCK_SIZE <= MAX_COMPRESSED_BLOCK_OUTPUT); + +/// The most input one block may cover once blocks are being extended. +/// +/// A block only grows over data whose byte distribution is not moving, so the +/// bytes it covers compress to very little and the output stays far inside +/// [`MAX_COMPRESSED_BLOCK_OUTPUT`]. The cap is what the writer charges its +/// workspace for, so it is a memory decision as much as a size one: the +/// optimal parse prices every position in a block and its arrays scale with +/// the block, so a mebibyte is the point where the extra table sets saved stop +/// being worth the pages. +pub(crate) const MAX_LZ_BLOCK_SIZE: usize = 1024 * 1024; + +const _: () = assert!(MAX_LZ_BLOCK_SIZE <= MAX_COMPRESSED_BLOCK_OUTPUT); + +/// How far a chunk's byte distribution may sit from the open block's before +/// the block is closed, as a fraction of the chunk. +/// +/// The statistic is how many of the chunk's bytes the open block's model puts +/// in the wrong place, so the limit reads as "extend while under one in a +/// hundred and twenty-eight of the next chunk's bytes are distributed +/// differently". Measured over the bench corpus, per 64 KiB chunk: +/// +/// ```text +/// class min median max +/// large-compressible 0 0 0 +/// large-incompressible 3,243 3,751 5,133 +/// large-source-tree 6,274 16,972 54,003 +/// large-text 2,319 30,598 90,929 +/// large-bin-unstripped 18,298 38,875 110,933 +/// large-bin-stripped 13,982 60,996 113,086 +/// ``` +/// +/// Only data that does not move at all falls under 512, and the nearest class +/// that does move sits four and a half times above it. That gap is the whole +/// design: a block grows over data a fresh table set could not describe any +/// better, and over nothing else. +const BLOCK_DRIFT_DIVISOR: u64 = 128; + +/// Decides where one block ends, from the raw bytes alone. +/// +/// Both writers have to cut a member the same way or the same input packs to +/// different archives, and they see it differently: the buffered path holds +/// the whole member while the streaming path reads it a chunk at a time and +/// compresses a wave of blocks in parallel. So the rule reads only the bytes +/// already folded into the open block plus the chunk being considered, which +/// both of them have at the moment they have to decide. +/// +/// Every block still carries its own tables. Extending a block is what WinRAR +/// does on data like this; the format's table-reuse flag would say the same +/// thing more directly, but no archive WinRAR writes sets it, so no third +/// party decoder is known to have been tested against one that does. +#[derive(Debug, Clone)] +pub(crate) struct BlockSplitter { + counts: [u32; 256], + total: u64, +} + +impl Default for BlockSplitter { + fn default() -> Self { + Self::new() + } +} + +impl BlockSplitter { + pub(crate) const fn new() -> Self { + Self { + counts: [0; 256], + total: 0, + } + } + + /// Folds a chunk into the open block. + pub(crate) fn accept(&mut self, chunk: &[u8]) { + for &byte in chunk { + self.counts[usize::from(byte)] += 1; + } + self.total += chunk.len() as u64; + } + + /// Starts a new block. + pub(crate) fn reset(&mut self) { + self.counts = [0; 256]; + self.total = 0; + } + + /// Whether the open block should swallow `chunk` rather than end before it. + /// + /// Integer arithmetic throughout, because a block boundary decided by + /// floating point would let the same input pack to different archives on + /// two platforms whose `log2` disagree in the last bit. + /// An accepted chunk is folded into the block; refusal leaves it unchanged. + pub(crate) fn try_extend(&mut self, chunk: &[u8]) -> bool { + let Some(counts) = self.extension_counts(chunk) else { + return false; + }; + // The decision already counted every byte. Fold those counts instead + // of scanning an accepted chunk a second time through accept(). + for (ours, theirs) in self.counts.iter_mut().zip(counts) { + *ours += theirs; + } + self.total += chunk.len() as u64; + true + } + + #[cfg(test)] + pub(crate) fn extends(&self, chunk: &[u8]) -> bool { + self.extension_counts(chunk).is_some() + } + + fn extension_counts(&self, chunk: &[u8]) -> Option<[u32; 256]> { + let open = self.total; + if open == 0 || chunk.is_empty() { + return None; + } + if open + chunk.len() as u64 > MAX_LZ_BLOCK_SIZE as u64 { + return None; + } + let mut counts = [0u32; 256]; + for &byte in chunk { + counts[usize::from(byte)] += 1; + } + let chunk_len = chunk.len() as u64; + // How many of the chunk's bytes the open block's distribution places + // wrongly. Both sides are scaled by `open` so neither divides early. + let mut misplaced = 0u64; + for (theirs, ours) in counts.iter().zip(&self.counts) { + misplaced += (u64::from(*theirs) * open).abs_diff(u64::from(*ours) * chunk_len); + } + (misplaced / open <= chunk_len / BLOCK_DRIFT_DIVISOR).then_some(counts) + } +} + +#[cfg(test)] +mod splitter_tests { + use super::{BlockSplitter, LZ_BLOCK_SIZE, MAX_LZ_BLOCK_SIZE}; + + #[test] + fn drift_threshold_accepts_256_changed_bytes_and_refuses_257() { + let initial = vec![0; LZ_BLOCK_SIZE]; + for (changed, accepted) in [(256, true), (257, false)] { + let mut splitter = BlockSplitter::new(); + splitter.accept(&initial); + let mut next = initial.clone(); + next[..changed].fill(1); + assert_eq!(splitter.try_extend(&next), accepted); + if accepted { + assert_eq!(splitter.total, (2 * LZ_BLOCK_SIZE) as u64); + assert_eq!(splitter.counts[0], (2 * LZ_BLOCK_SIZE - changed) as u32); + assert_eq!(splitter.counts[1], changed as u32); + } else { + assert_eq!(splitter.total, LZ_BLOCK_SIZE as u64); + assert_eq!(splitter.counts[0], LZ_BLOCK_SIZE as u32); + assert_eq!(splitter.counts[1], 0); + } + assert!(splitter.counts[2..].iter().all(|&count| count == 0)); + } + } + + #[test] + fn one_pass_extension_preserves_decisions_counts_and_refused_state() { + let initial = vec![0; LZ_BLOCK_SIZE]; + for changed in [0, 1, 128, 255, 256, 257, 512, 1024] { + let mut next = initial.clone(); + next[..changed].fill(1); + let mut reference = BlockSplitter::new(); + let mut actual = BlockSplitter::new(); + assert!(!actual.try_extend(&next)); + assert_eq!(actual.counts, reference.counts); + assert_eq!(actual.total, reference.total); + reference.accept(&initial); + actual.accept(&initial); + for chunk in [&next[..], &initial[..], &[][..], &next[..], &initial[..31]] { + let expected = reference.extends(chunk); + assert_eq!(actual.try_extend(chunk), expected, "changed={changed}"); + if expected { + reference.accept(chunk); + } + assert_eq!(actual.counts, reference.counts); + assert_eq!(actual.total, reference.total); + } + } + let mut reference = BlockSplitter::new(); + let mut actual = BlockSplitter::new(); + reference.accept(&vec![7; MAX_LZ_BLOCK_SIZE - LZ_BLOCK_SIZE]); + actual.accept(&vec![7; MAX_LZ_BLOCK_SIZE - LZ_BLOCK_SIZE]); + for length in [LZ_BLOCK_SIZE, 1, 0] { + let chunk = vec![7; length]; + let expected = reference.extends(&chunk); + assert_eq!(actual.try_extend(&chunk), expected); + if expected { + reference.accept(&chunk); + } + assert_eq!(actual.counts, reference.counts); + assert_eq!(actual.total, reference.total); + } + } +} + +const MAX_FILTER_BLOCK_LENGTH: usize = 0x3ffff; + +/// The most channels a RAR 5 delta filter record can name. The count is written +/// as five bits biased by one, so this is what the format can say, not a policy. +pub(crate) const MAX_DELTA_CHANNELS: usize = 32; + +/// How much input goes into one compressed block once a filter is carried. +/// +/// A filter record cannot describe more than [`MAX_FILTER_BLOCK_LENGTH`] bytes, +/// so this is the smaller of that ceiling and the plain block size. Splitting a +/// filtered range across blocks costs one more record per block and converts +/// the same bytes either way: the transform reads an absolute file offset, and +/// an instruction straddling a boundary was already left alone at the old +/// 256 KiB one. +const FILTERED_LZ_BLOCK_SIZE: usize = if LZ_BLOCK_SIZE < MAX_FILTER_BLOCK_LENGTH { + LZ_BLOCK_SIZE +} else { + MAX_FILTER_BLOCK_LENGTH +}; + +/// Where the encoder stops looking for anything better. +/// +/// A search that has reached this far ends, whether it is a chain walk or a +/// tree descent, and the optimal parse takes the +/// match and steps over the bytes it covers instead of pricing each of them. The +/// second half matters more than it sounds. The parse prices every position in a +/// block, because a cheaper path can arrive at any of them, so without it a +/// 4 KiB match gets confirmed at all 4096 of its positions to learn what the +/// first one already said. On a mebibyte that repeats, that cost level 5 +/// fifty-three seconds to save four bytes over level 3. +/// +/// 512 is where committing is free. On a mebibyte of source at level 5 it packs +/// two bytes smaller than pricing every position and finishes in 7.08s rather +/// than 7.31s, and the repeating mebibyte drops to 1.19s. Committing sooner does +/// buy time, and it is not worth it: at 128 the source packs 0.16% larger for +/// 1.2x, at 64 it packs 0.47% larger for 1.5x, and neither closes the distance +/// to WinRAR. +const NICE_MATCH_LENGTH: usize = 512; + +const MAX_MATCH_CANDIDATES: usize = 256; + +#[derive(Clone, Copy, PartialEq, Eq)] +struct LengthSlices<'a> { + main: &'a [u8], + distance: &'a [u8], + align: &'a [u8], + length: &'a [u8], +} + +impl OwnedLengths { + fn slices(&self) -> LengthSlices<'_> { + LengthSlices { + main: &self.main, + distance: &self.distance, + align: &self.align, + length: &self.length, + } + } +} + +pub fn encode_table_lengths(lengths: &TableLengths, algorithm_version: u8) -> Result> { + encode_table_lengths_with_bit_count(lengths, algorithm_version).map(|(data, _)| data) +} + +pub fn encode_table_lengths_with_bit_count( + lengths: &TableLengths, + algorithm_version: u8, +) -> Result<(Vec, usize)> { + encode_table_slices( + LengthSlices { + main: &lengths.main, + distance: &lengths.distance, + align: &lengths.align, + length: &lengths.length, + }, + algorithm_version, + &Allowance::default(), + ) + .map(|(bytes, bits)| (bytes.into_vec(), bits)) +} + +fn encode_table_slices( + lengths: LengthSlices<'_>, + algorithm_version: u8, + allowance: &B, +) -> Result<(Buffer, usize)> { + let distance_size = match algorithm_version { + 0 => DISTANCE_TABLE_SIZE_50, + 1 => DISTANCE_TABLE_SIZE_70, + _ => { + return Err(Error::InvalidData( + "RAR 5 unknown compression algorithm version", + )) + } + }; + if lengths.main.len() != MAIN_TABLE_SIZE + || lengths.distance.len() != distance_size + || lengths.align.len() != ALIGN_TABLE_SIZE + || lengths.length.len() != LENGTH_TABLE_SIZE + { + return Err(Error::InvalidData("RAR 5 table length count mismatch")); + } + + // The version and all four slice lengths were checked above. They form + // one exact-sized table, so assembly cannot request further growth. + let flattened = Buffer::from_slices( + &[ + lengths.main, + lengths.distance, + lengths.align, + lengths.length, + ], + allowance, + )?; + for &length in flattened.iter() { + if length > 15 { + return Err(Error::InvalidData("RAR 5 Huffman length is too large")); + } + } + + let level_tokens = encode_table_level_tokens_with_allowance(&flattened, allowance)?; + let level_lengths = level_code_lengths_with_allowance(&level_tokens, allowance)?; + let level_table = EncoderCodeTable::from_lengths(&level_lengths, allowance)?; + let mut writer = BitWriter::with_allowance(allowance); + try_write_level_lengths(&mut writer, &level_lengths)?; + for token in level_tokens.iter() { + let (code, len) = level_table.code_for_present_symbol(token.symbol); + writer + .try_write_bits(usize::from(code), usize::from(len)) + .map_err(Into::into)?; + if token.extra_bits != 0 { + writer + .try_write_bits( + usize::from(token.extra_value), + usize::from(token.extra_bits), + ) + .map_err(Into::into)?; + } + } + let bit_count = writer.bit_pos; + Ok((writer.bytes, bit_count)) +} + +pub fn encode_compressed_block( + payload: &[u8], + payload_bits: usize, + has_tables: bool, + is_last: bool, +) -> Result> { + encode_compressed_block_with_allowance( + payload, + payload_bits, + has_tables, + is_last, + &Allowance::default(), + ) + .map(Buffer::into_vec) +} + +fn encode_compressed_block_with_allowance( + payload: &[u8], + payload_bits: usize, + has_tables: bool, + is_last: bool, + allowance: &B, +) -> Result> { + if payload_bits > payload.len() * 8 { + return Err(Error::InvalidData("RAR 5 block bit count exceeds payload")); + } + if !payload.is_empty() && payload_bits <= (payload.len() - 1) * 8 { + return Err(Error::InvalidData("RAR 5 block has unused payload bytes")); + } + if payload.len() > 0x00ff_ffff { + return Err(Error::InvalidData("RAR 5 block payload is too large")); + } + + let size_len = if payload.len() <= 0xff { + 1 + } else if payload.len() <= 0xffff { + 2 + } else { + 3 + }; + let final_byte_bits = if payload.is_empty() { + 1 + } else { + ((payload_bits - 1) % 8) + 1 + }; + let mut flags = (final_byte_bits as u8) - 1; + flags |= ((size_len - 1) as u8) << 3; + if is_last { + flags |= 0x40; + } + if has_tables { + flags |= 0x80; + } + + let mut size_bytes = [0u8; 3]; + let mut size = payload.len(); + for byte in &mut size_bytes[..size_len] { + *byte = size as u8; + size >>= 8; + } + let checksum = size_bytes[..size_len] + .iter() + .fold(0x5a ^ flags, |acc, &byte| acc ^ byte); + // The header and bounded payload have an exact final size. + Buffer::from_slices( + &[&[flags, checksum], &size_bytes[..size_len], payload], + allowance, + ) +} + +pub fn encode_literal_only(data: &[u8], algorithm_version: u8) -> Result> { + let distance_size = match algorithm_version { + 0 => DISTANCE_TABLE_SIZE_50, + 1 => DISTANCE_TABLE_SIZE_70, + _ => { + return Err(Error::InvalidData( + "RAR 5 unknown compression algorithm version", + )) + } + }; + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; distance_size], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + let present = literal_presence(data); + let literal_count = present.iter().filter(|&&used| used).count(); + let literal_length = huffman::bits_for_symbol_count(literal_count); + let mut literal_codes = [0u16; 256]; + let mut next_code = 0u16; + for (symbol, used) in present.into_iter().enumerate() { + if used { + lengths.main[symbol] = literal_length; + // Every literal has the same length, so canonical order is simply + // the order of the present byte values. + literal_codes[symbol] = next_code; + next_code += 1; + } + } + + // The version, table sizes and generated lengths are all fixed above; + // malformed-table errors remain available on the public table encoder. + let (table_data, table_bits) = encode_table_lengths_with_bit_count(&lengths, algorithm_version) + .expect("generated literal table is valid"); + let mut writer = BitWriter { + bytes: Buffer::from_vec(table_data), + bit_pos: table_bits, + }; + for &byte in data { + writer.write_bits( + usize::from(literal_codes[byte as usize]), + usize::from(literal_length), + ); + } + let payload_bits = writer.bit_pos; + encode_compressed_block(&writer.finish(), payload_bits, true, true) +} + +pub fn encode_lz_member(data: &[u8], algorithm_version: u8) -> Result> { + encode_lz_member_with_history(data, &[], algorithm_version) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct EncodeOptions { + pub max_match_candidates: usize, + pub lazy_matching: bool, + pub lazy_lookahead: usize, + pub max_match_distance: usize, + pub optimal_parse: bool, + /// Arca: the length at which a match is taken without looking further, + /// [`NICE_MATCH_LENGTH`] unless a level asks for less. + pub nice_length: usize, + /// Arca: how many times the optimal parse prices a block, at most + /// [`OPTIMAL_PARSE_PASSES`]. + pub parse_passes: usize, +} + +impl EncodeOptions { + pub const fn new(max_match_candidates: usize) -> Self { + Self { + max_match_candidates, + lazy_matching: false, + lazy_lookahead: 1, + max_match_distance: MAX_ENCODER_MATCH_OFFSET, + optimal_parse: false, + nice_length: NICE_MATCH_LENGTH, + parse_passes: OPTIMAL_PARSE_PASSES, + } + } + + /// Arca: a shorter nice length searches and prices less per position. + pub const fn with_nice_length(mut self, length: usize) -> Self { + self.nice_length = if length < 8 { 8 } else { length }; + self + } + + /// Arca: fewer passes reprice each block fewer times. + pub const fn with_parse_passes(mut self, passes: usize) -> Self { + self.parse_passes = if passes == 0 { 1 } else { passes }; + self + } + + pub const fn with_optimal_parse(mut self, enabled: bool) -> Self { + self.optimal_parse = enabled; + self + } + + pub const fn with_lazy_matching(mut self, enabled: bool) -> Self { + self.lazy_matching = enabled; + self + } + + pub const fn with_lazy_lookahead(mut self, bytes: usize) -> Self { + self.lazy_lookahead = bytes; + self + } + + pub const fn with_max_match_distance(mut self, distance: usize) -> Self { + self.max_match_distance = distance; + self + } +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self::new(MAX_MATCH_CANDIDATES) + } +} + +pub fn encode_lz_member_with_history( + data: &[u8], + history: &[u8], + algorithm_version: u8, +) -> Result> { + encode_lz_member_inner( + data, + history, + algorithm_version, + EncodeOptions::default(), + None, + ) +} + +pub fn encode_lz_member_with_options( + data: &[u8], + algorithm_version: u8, + options: EncodeOptions, +) -> Result> { + encode_lz_member_with_history_and_options(data, &[], algorithm_version, options) +} + +#[cfg(test)] +pub(crate) fn encode_lz_member_with_options_and_progress( + data: &[u8], + algorithm_version: u8, + options: EncodeOptions, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + encode_lz_member_inner(data, &[], algorithm_version, options, Some(progress)) +} + +#[cfg(test)] +pub(crate) fn encode_lz_reader_to( + reader: &mut dyn Read, + input_size: u64, + output: &mut dyn Write, + algorithm_version: u8, + options: EncodeOptions, + block_size: usize, + progress: Option<&mut dyn FnMut(u64) -> bool>, +) -> crate::Result<()> { + reader_to_with_allowance( + reader, + input_size, + output, + algorithm_version, + options, + block_size, + progress, + &Allowance::default(), + ) +} + +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +fn reader_to_with_allowance( + reader: &mut dyn Read, + input_size: u64, + output: &mut dyn Write, + algorithm_version: u8, + options: EncodeOptions, + block_size: usize, + mut progress: Option<&mut dyn FnMut(u64) -> bool>, + allowance: &B, +) -> crate::Result<()> { + if block_size == 0 { + return Err(crate::Error::InvalidHeader( + "RAR 5 streaming block size is zero", + )); + } + let block_size = block_size.min(MAX_COMPRESSED_BLOCK_OUTPUT); + let mut history = Buffer::new(allowance); + let mut chunk = Buffer::filled(input_size.min(block_size as u64) as usize, 0u8, allowance)?; + let mut block = Buffer::new(allowance); + let mut remaining = input_size; + let mut completed = 0u64; + let mut held: Option> = None; + while remaining != 0 || held.is_some() { + // One chunk, then further chunks while the data is not moving, which is + // the cut [`BlockSplitter`] makes for the other two writers. Deciding + // needs the chunk in hand, so the one that ends a block is held over. + let mut splitter = BlockSplitter::new(); + block.clear(); + match held.take() { + Some(first) => block + .extend_from_slice(&first) + .map_err(Into::::into)?, + None => { + let wanted = usize::try_from(remaining.min(block_size as u64)) + .map_err(|_| crate::Error::InvalidHeader("RAR 5 block size overflows usize"))?; + reader.read_exact(&mut chunk[..wanted])?; + remaining -= wanted as u64; + block + .extend_from_slice(&chunk[..wanted]) + .map_err(Into::::into)?; + } + } + splitter.accept(&block); + while remaining != 0 { + let wanted = usize::try_from(remaining.min(block_size as u64)) + .map_err(|_| crate::Error::InvalidHeader("RAR 5 block size overflows usize"))?; + reader.read_exact(&mut chunk[..wanted])?; + remaining -= wanted as u64; + if !splitter.try_extend(&chunk[..wanted]) { + held = Some(Buffer::copied(&chunk[..wanted], allowance)?); + break; + } + block + .extend_from_slice(&chunk[..wanted]) + .map_err(Into::::into)?; + } + let (window, start) = member_window_with_allowance(&block, &history, options, allowance)?; + let packed = encode_lz_block_with_allowance( + &window, + start..window.len(), + MemberSearch::Fresh, + algorithm_version, + &[], + options, + remaining == 0 && held.is_none(), + None, + allowance, + )?; + drop(window); + output.write_all(&packed)?; + history.remember(&block, options.max_match_distance)?; + completed += block.len() as u64; + if progress + .as_deref_mut() + .is_some_and(|report| !report(completed)) + { + return Err(crate::Error::Cancelled); + } + } + let mut trailing = [0u8; 1]; + if reader.read(&mut trailing)? != 0 { + return Err(crate::Error::InvalidHeader( + "entry source size changed while compressing", + )); + } + Ok(()) +} + +#[cfg(test)] +pub(crate) fn encode_lz_streaming_block( + data: &[u8], + history: &[u8], + algorithm_version: u8, + options: EncodeOptions, + is_last: bool, +) -> Result> { + encode_lz_block( + data, + history, + algorithm_version, + &[], + options, + is_last, + None, + ) +} + +/// Encode adjacent streaming blocks with one seeded chain finder. The first +/// block without history keeps its existing tree parse; subsequent blocks use +/// chains just as separately seeded streaming blocks do. +#[cfg(test)] +pub(crate) fn encode_lz_streaming_blocks( + data: &[u8], + history: &[u8], + blocks: &[(usize, bool)], + algorithm_version: u8, + options: EncodeOptions, + block_done: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result>> { + streaming_blocks_with_allowance( + data, + history, + blocks, + algorithm_version, + options, + block_done, + &Allowance::default(), + ) + .map(|outputs| outputs.into_iter().map(Buffer::into_vec).collect()) +} + +pub(crate) fn streaming_blocks_with_allowance( + data: &[u8], + history: &[u8], + blocks: &[(usize, bool)], + algorithm_version: u8, + options: EncodeOptions, + mut block_done: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result, B>> { + let mut previous = 0; + for &(end, _) in blocks { + if end <= previous || end > data.len() { + return Err(Error::InvalidData("RAR 5 streaming block range is invalid")); + } + previous = end; + } + if previous != data.len() { + return Err(Error::InvalidData( + "RAR 5 streaming blocks do not cover input", + )); + } + let (combined, start) = member_window_with_allowance(data, history, options, allowance)?; + + let mut at = start; + let mut output = Buffer::with_capacity(blocks.len(), allowance)?; + // Block ends are offsets into `data`, so the bytes a block covers are the + // step from the end before it. Reporting them as they land is what keeps a + // progress bar moving: a whole member can be one run, and a run that + // reports only when it finishes reports nothing until it is done. + let mut reported = 0usize; + let mut report = |end: usize, block_done: &mut Option<&mut dyn FnMut(usize) -> bool>| { + let delta = end - reported; + reported = end; + match block_done { + Some(report) => report(delta), + None => true, + } + }; + // Arca: a member with no history keeps the tree finder for every block, + // not only the first. The tree is what makes searching at every position + // affordable, and the blocks arrive in order, which is all it asks. + if blocks.is_empty() { + return Ok(output); + } + let mut search = if start == 0 && options.optimal_parse { + SharedMemberSearch::Optimal(OptimalCollector::with_allowance( + &combined, 0, options, allowance, + )?) + } else { + let finder = member_finder_with_allowance(&combined, at, options, allowance)?; + if options.optimal_parse { + SharedMemberSearch::Optimal(OptimalCollector { + finder: CollectorFinder::Chains(finder), + }) + } else { + SharedMemberSearch::Lazy(finder) + } + }; + for &(end, is_last) in blocks { + let end = start + end; + output.push_admitted(encode_lz_block_with_allowance( + &combined, + at..end, + search.borrow(), + algorithm_version, + &[], + options, + is_last, + None, + allowance, + )?); + at = end; + if !report(end - start, &mut block_done) { + return Err(Error::Cancelled); + } + } + Ok(output) +} + +pub fn encode_lz_member_with_history_and_options( + data: &[u8], + history: &[u8], + algorithm_version: u8, + options: EncodeOptions, +) -> Result> { + encode_lz_member_inner(data, history, algorithm_version, options, None) +} + +/// The writer rejects these before compressing anything, so reaching this is +/// either a direct `codec` caller or a bug. Either way the codec stays total. +fn rar50_filter(kind: crate::FilterKind) -> Result { + Rar50Filter::try_from(kind) + .map_err(|_| Error::InvalidData("RAR 5 has no builtin type for this filter")) +} + +/// Applies `filters` to a copy of `data`, returning the transformed bytes and +/// the records that describe them. +#[cfg(test)] +pub(crate) fn filtered_lz_member( + data: &[u8], + filters: &[crate::FilterSpec], +) -> Result<(Vec, Vec)> { + filtered_member_with_allowance(data, filters, &Allowance::default()) + .map(|(data, records)| (data.into_vec(), records.into_vec())) +} + +fn filtered_member_with_allowance( + data: &[u8], + filters: &[crate::FilterSpec], + allowance: &B, +) -> Result<(Buffer, Buffer)> { + let mut filtered = Buffer::copied(data, allowance)?; + let mut records = Buffer::with_capacity(filters.len(), allowance)?; + for filter in filters { + let range = filter.range.clone().unwrap_or(0..data.len()); + if range.start >= range.end || range.end > data.len() { + return Err(Error::InvalidData("RAR 5 filter range is invalid")); + } + let filter_data = &mut filtered[range.clone()]; + let (filter_type, channels) = encode_filter_data( + rar50_filter(filter.kind)?, + filter_data, + range.start, + allowance, + )?; + records.push_admitted(EncodeFilter { + offset: range.start, + length: range.len(), + filter_type, + channels, + }); + } + Ok((filtered, records)) +} + +fn encode_filter_data( + kind: Rar50Filter, + data: &mut [u8], + file_offset: usize, + allowance: &B, +) -> Result<(FilterType, usize)> { + if file_offset > u32::MAX as usize { + return Err(Error::InvalidData("RAR 5 filter offset is too large")); + } + match kind { + Rar50Filter::Delta { channels } => { + let transformed = filters::delta_encode_with_allowance( + data, + channels, + rar50_delta_messages(), + allowance, + )?; + data.copy_from_slice(&transformed); + Ok((FilterType::Delta, channels)) + } + Rar50Filter::E8 => { + e8e9_encode(data, file_offset as u32, false); + Ok((FilterType::E8, 0)) + } + Rar50Filter::E8E9 => { + e8e9_encode(data, file_offset as u32, true); + Ok((FilterType::E8E9, 0)) + } + Rar50Filter::Arm => { + arm_encode(data, file_offset as u32); + Ok((FilterType::Arm, 0)) + } + } +} + +/// Transforms the member and cuts it into the blocks a filter record can +/// describe, then compresses those blocks against one search state. +/// +/// The transform runs first and over the whole member, because the search has +/// to read final bytes: a block reaches back into the ones before it, and a +/// match must point at what the decoder will really have. Each block still +/// gets its own records covering only its own bytes, so a filtered range +/// spanning several blocks converts exactly as it did when each block was +/// transformed alone: the transform reads an absolute file offset, which the +/// cut does not change. +/// +/// Blocks used to be compressed one at a time, each against a fresh copy of +/// the history behind it and a finder rebuilt from that copy. That is the +/// cost [`encode_lz_member_inner`] took off the unfiltered path and left +/// here: at 64 KiB a block, a four-megabyte member re-copied and re-inserted +/// 126 MiB of history, thirty-one times what it holds. +fn filtered_lz_blocks( + data: &[u8], + filters: &[crate::FilterSpec], + history: &[u8], + algorithm_version: u8, + options: EncodeOptions, + mut progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let filters = normalized_filter_specs(data.len(), filters, allowance)?; + let history = &history[history.len().saturating_sub(options.max_match_distance)..]; + let start = history.len(); + let mut combined = Buffer::from_slices(&[history, data], allowance)?; + + let mut blocks = Buffer::new(allowance); + let mut chunk_start = 0usize; + while chunk_start < data.len() { + let chunk_end = (chunk_start + FILTERED_LZ_BLOCK_SIZE).min(data.len()); + let mut records = Buffer::new(allowance); + for filter in filters.iter() { + let filter_start = filter.range.start.max(chunk_start); + let filter_end = filter.range.end.min(chunk_end); + if filter_start >= filter_end { + continue; + } + let (filter_type, channels) = encode_filter_data( + filter.kind, + &mut combined[start + filter_start..start + filter_end], + filter_start, + allowance, + )?; + records + .push(EncodeFilter { + offset: filter_start - chunk_start, + length: filter_end - filter_start, + filter_type, + channels, + }) + .map_err(Into::into)?; + } + blocks + .push((chunk_start..chunk_end, records)) + .map_err(Into::into)?; + chunk_start = chunk_end; + } + + // One search state for the whole member, as the unfiltered path has. + let mut search = if options.optimal_parse { + SharedMemberSearch::Optimal(OptimalCollector::with_allowance( + &combined, start, options, allowance, + )?) + } else { + SharedMemberSearch::Lazy(member_finder_with_allowance( + &combined, start, options, allowance, + )?) + }; + + let mut out = Buffer::new(allowance); + for (block, records) in blocks { + let mut chunk_progress = |position: usize| { + progress + .as_deref_mut() + .is_none_or(|report| report(block.start.saturating_add(position))) + }; + let packed = encode_lz_block_with_allowance( + &combined, + start + block.start..start + block.end, + search.borrow(), + algorithm_version, + &records, + options, + block.end == data.len(), + Some(&mut chunk_progress), + allowance, + )?; + out.extend_from_slice(&packed).map_err(Into::into)?; + } + Ok(out) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct NormalizedFilterSpec { + kind: Rar50Filter, + range: Range, +} + +fn normalized_filter_specs( + data_len: usize, + filters: &[crate::FilterSpec], + allowance: &B, +) -> Result> { + let mut normalized = Buffer::with_capacity(filters.len(), allowance)?; + for filter in filters { + let range = filter.range.clone().unwrap_or(0..data_len); + if range.start >= range.end || range.end > data_len { + return Err(Error::InvalidData("RAR 5 filter range is invalid")); + } + normalized.push_admitted(NormalizedFilterSpec { + kind: rar50_filter(filter.kind)?, + range, + }); + } + Ok(normalized) +} + +fn encode_lz_member_inner( + data: &[u8], + history: &[u8], + algorithm_version: u8, + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + encode_member_with_allowance( + data, + history, + algorithm_version, + options, + progress, + &Allowance::default(), + ) + .map(Buffer::into_vec) +} + +pub(crate) fn encode_owned_member( + data: &[u8], + version: u8, + options: EncodeOptions, + filters: Option<&[crate::FilterSpec]>, + progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + match filters { + None => encode_member_with_allowance(data, &[], version, options, progress, allowance), + Some(filters) => { + EncoderState::new(options, allowance).encode(data, version, Some(filters), progress) + } + } +} + +pub(crate) fn filtered_owned_member( + data: &[u8], + filters: &[crate::FilterSpec], + allowance: &B, +) -> Result> { + filtered_member_with_allowance(data, filters, allowance).map(|(bytes, _)| bytes) +} + +fn encode_member_with_allowance( + data: &[u8], + history: &[u8], + algorithm_version: u8, + options: EncodeOptions, + mut progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let (window, start) = member_window_with_allowance(data, history, options, allowance)?; + let combined = &*window; + if data.len() > LZ_BLOCK_SIZE { + // One search state for the whole member. It used to be built per + // block, which meant rehashing a window of history every 64 KiB: on a + // 16 MiB member that was half the encode. The optimal parse used to be + // worse still, rebuilding per pass; its collector searches each block + // once and lets the passes replay the answers. + let mut search = if options.optimal_parse { + SharedMemberSearch::Optimal(OptimalCollector::with_allowance( + combined, start, options, allowance, + )?) + } else { + SharedMemberSearch::Lazy(member_finder_with_allowance( + combined, start, options, allowance, + )?) + }; + + let mut out = Buffer::new(allowance); + let mut completed = 0usize; + let mut block_start = start; + let mut splitter = BlockSplitter::new(); + while block_start < combined.len() { + // Take one chunk, then keep taking them while the data is not + // moving. See [`BlockSplitter`]. + splitter.reset(); + let mut block_end = (block_start + LZ_BLOCK_SIZE).min(combined.len()); + splitter.accept(&combined[block_start..block_end]); + while block_end < combined.len() { + let next_end = (block_end + LZ_BLOCK_SIZE).min(combined.len()); + let next = &combined[block_end..next_end]; + if !splitter.try_extend(next) { + break; + } + block_end = next_end; + } + let is_last = block_end == combined.len(); + let mut chunk_progress = |position: usize| { + progress + .as_deref_mut() + .is_none_or(|report| report(completed.saturating_add(position))) + }; + let packed = encode_lz_block_with_allowance( + combined, + block_start..block_end, + search.borrow(), + algorithm_version, + &[], + options, + is_last, + Some(&mut chunk_progress), + allowance, + )?; + out.extend_from_slice(&packed).map_err(Into::into)?; + completed = completed.saturating_add(block_end - block_start); + block_start = block_end; + } + return Ok(out); + } + encode_lz_block_with_allowance( + combined, + start..combined.len(), + MemberSearch::Fresh, + algorithm_version, + &[], + options, + true, + progress, + allowance, + ) +} + +#[allow(clippy::too_many_arguments)] +fn encode_filtered_member_with_allowance( + data: &[u8], + history: &[u8], + algorithm_version: u8, + filters: &[EncodeFilter], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let (window, start) = member_window_with_allowance(data, history, options, allowance)?; + let end = window.len(); + encode_lz_block_with_allowance( + &window, + start..end, + MemberSearch::Fresh, + algorithm_version, + filters, + options, + true, + progress, + allowance, + ) +} + +/// How far back a parse over `reach` bytes has to remember. +/// +/// Nothing further than the maximum distance is ever accepted as a match, so a +/// link to anything older can be dropped. Neither can a match reach back past +/// the start of what is being parsed, so a member shorter than the dictionary +/// sets the window instead: asking for `--dict-size 32m` on a one-megabyte +/// member should not reserve thirty-two megabytes of links that can never name +/// a position. +fn finder_window(options: EncodeOptions, reach: usize) -> usize { + options.max_match_distance.min(reach).max(LZ_BLOCK_SIZE) +} + +/// A finder for the whole member, seeded with the history it carries in. It +/// keeps growing as the blocks are parsed, so it is sized to the widest window +/// the member could ever want rather than to any one block. +fn member_finder_with_allowance( + combined: &[u8], + start: usize, + options: EncodeOptions, + allowance: &B, +) -> Result> { + let mut finder = + Rar50MatchFinder::with_allowance(finder_window(options, combined.len()), allowance)?; + for pos in 0..start { + finder.insert(combined, pos); + } + Ok(finder) +} + +/// A finder holding everything a parse of `block` may reach back to, and +/// nothing older. +fn seeded_finder( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + allowance: &B, +) -> Result> { + // Sized to what this block can actually reach, not to the maximum distance, + // so the first blocks of a member do not clear a window the data is not yet + // long enough to fill. + let behind = block.start.min(options.max_match_distance); + let mut finder = + Rar50MatchFinder::with_allowance(behind + (block.end - block.start), allowance)?; + for pos in block.start - behind..block.start { + finder.insert(combined, pos); + } + Ok(finder) +} + +/// The search state a member shares across its blocks, when it has any. +enum MemberSearch<'a, B: Budget = Allowance> { + /// Nothing shared: the block builds what it needs and drops it. + Fresh, + /// The member's chain finder, which the lazy path feeds block by block. + Lazy(&'a mut Rar50MatchFinder), + /// The member's match collector, which the optimal parse feeds block by + /// block. + Optimal(&'a mut OptimalCollector), +} + +enum SharedMemberSearch { + Lazy(Rar50MatchFinder), + Optimal(OptimalCollector), +} + +impl SharedMemberSearch { + fn borrow(&mut self) -> MemberSearch<'_, B> { + match self { + Self::Lazy(finder) => MemberSearch::Lazy(finder), + Self::Optimal(collector) => MemberSearch::Optimal(collector), + } + } +} + +/// The matches at every position of one block, found once and priced by every +/// pass of the optimal parse. The runs at one position carry strictly +/// increasing lengths and distances, so each is the nearest distance found +/// that reaches its length. +/// +/// One position holds at most one run per length it can reach, so the whole +/// block is bounded by the block size times [`NICE_MATCH_LENGTH`]. Nothing +/// approaches that: the worst measured is about six runs per position, on a +/// mebibyte of two-symbol noise, where every position has many candidates whose +/// lengths creep up one byte at a time. That block cost three megabytes. +struct BlockMatches { + /// Every position's runs, one position after another. + runs: Buffer<(u32, u32), B>, + /// Where each position's runs start in `runs`, with one extra entry to + /// close the last position. + starts: Buffer, +} + +/// One match finder for a member's whole optimal parse, and the walk that asks +/// it about each block once. +/// +/// The parse prices each block [`OPTIMAL_PARSE_PASSES`] times, but nothing the +/// finder answers depends on the prices, so it used to be asked the same +/// questions once per pass, through a finder rebuilt and reseeded once per +/// pass. Collecting the answers first and replaying them lets every pass after +/// the first skip the finder entirely. +/// +/// Searching once is also what makes the tree finder affordable, and the tree +/// is where the speed is: pricing every position means searching at every +/// position, which is the load a chain walk carries worst and a tree carries +/// best. A member that starts with no history gets the tree. One that carries +/// history keeps the chains, because the only way into a tree is a descent per +/// position, and paying that across a dictionary of history would cost more +/// than the chains ever did. +struct OptimalCollector { + finder: CollectorFinder, +} + +enum CollectorFinder { + Tree(match_finder::TreeMatchFinder), + Chains(Rar50MatchFinder), +} + +impl OptimalCollector { + #[cfg(test)] + fn new(combined: &[u8], start: usize, options: EncodeOptions) -> Self { + Self::with_allowance(combined, start, options, &Allowance::default()) + .expect("unlimited collector allocation") + } +} + +impl OptimalCollector { + fn with_allowance( + combined: &[u8], + start: usize, + options: EncodeOptions, + allowance: &B, + ) -> Result { + let finder = if start == 0 { + CollectorFinder::Tree(match_finder::TreeMatchFinder::with_allowance( + finder_window(options, combined.len()), + allowance, + )?) + } else { + CollectorFinder::Chains(member_finder_with_allowance( + combined, start, options, allowance, + )?) + }; + Ok(Self { finder }) + } + + /// Finds the matches the parse will price at each position of `block`, + /// taking the positions into the finder as it goes. Blocks must arrive in + /// order, each exactly once, the same discipline the member's shared chain + /// finder already asks of the lazy path. + /// + /// Searching stops where the parse stops pricing. A match that reaches + /// [`NICE_MATCH_LENGTH`] is one the parse commits to and steps over, so the + /// positions it covers are not searched from either. Skipping the pricing + /// alone would have left the search doing all the work it used to: on a + /// mebibyte that repeats, one search per 4 KiB became a million. + fn collect( + &mut self, + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + ) -> Result> { + let allowance = match &self.finder { + CollectorFinder::Tree(finder) => finder.allowance(), + CollectorFinder::Chains(finder) => finder.allowance(), + }; + let span = block.end - block.start; + let mut matches = BlockMatches { + // One run per position to start with, which is where data that + // matches at all lands, so the common case grows this once. + runs: Buffer::with_capacity(span, &allowance)?, + starts: Buffer::with_capacity(span + 1, &allowance)?, + }; + // The first position past a match the parse will commit to. The parse + // reaches the same decision from the same lengths, so the two agree on + // which positions matter without having to be told. + let mut committed_through = block.start; + for pos in block.clone() { + matches.starts.push_admitted(matches.runs.len() as u32); + let searching = pos >= committed_through && options.max_match_candidates != 0; + let max_distance = pos.min(options.max_match_distance); + let before = matches.runs.len(); + match &mut self.finder { + CollectorFinder::Tree(tree) => { + // Inserting into a tree is the same descent as searching + // it, so a position the parse steps over is stepped over + // here too rather than inserted for nothing. Its bytes are + // a copy of what the match already points at, so the tree + // loses little by not holding them. + let avail = combined.len() - pos; + if !searching || avail < 4 { + continue; + } + // Compares stop where the parse stops caring about better + // alternatives. A match that reaches that far is measured + // out to its real end, which is the length the parse + // commits to and steps over. + let len_limit = avail.min(options.nice_length); + tree.matches( + combined, + pos, + len_limit, + max_distance, + options.max_match_candidates, + &mut matches.runs, + ) + .map_err(Into::into)?; + if let Some(last) = matches.runs[before..].last_mut() { + let limit = avail.min(MAX_ENCODER_MATCH_LENGTH); + if last.0 as usize == len_limit && len_limit < limit { + last.0 = match_length(combined, pos, last.1 as usize, limit) as u32; + } + } + } + CollectorFinder::Chains(finder) => { + // Inserting into a chain is one store, so every position + // goes in whether or not it is searched from. That keeps a + // solid member's candidates exactly what they were. + finder.insert(combined, pos); + let max_length = (block.end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if !searching || max_distance == 0 || max_length < 4 { + continue; + } + // The chain walks nearest first, so the first distance to + // reach a length is the cheapest one that can. + let mut longest = 0usize; + let mut checked = 0usize; + let mut candidate = finder.first(combined, pos); + while candidate != match_finder::NO_POSITION + && longest < max_length + && longest < options.nice_length + { + if candidate >= pos { + candidate = finder.previous(candidate); + continue; + } + let distance = pos - candidate; + if distance > max_distance { + break; + } + checked += 1; + if combined[candidate + longest] == combined[pos + longest] { + let length = match_length(combined, pos, distance, max_length); + if length > longest { + matches + .runs + .push((length as u32, distance as u32)) + .map_err(Into::into)?; + longest = length; + } + } + if checked >= options.max_match_candidates { + break; + } + candidate = finder.previous(candidate); + } + } + } + // The parse can only take a match the block still has room for, so + // the reach it will commit to is measured the way it measures it. + if let Some(&(length, _)) = matches.runs[before..].last() { + let reach = (length as usize) + .min(block.end - pos) + .min(MAX_ENCODER_MATCH_LENGTH); + if reach >= options.nice_length { + committed_through = pos + reach; + } + } + } + matches.starts.push_admitted(matches.runs.len() as u32); + Ok(matches) + } +} + +/// The bytes one member's parse reaches across, and where its own data starts. +/// +/// A member with no history to carry borrows its own data rather than copying +/// it, which is every member of a non-solid archive. +enum MemberWindow<'a, B: Budget> { + Borrowed(&'a [u8]), + Owned(Buffer), +} + +impl std::ops::Deref for MemberWindow<'_, B> { + type Target = [u8]; + fn deref(&self) -> &[u8] { + match self { + Self::Borrowed(data) => data, + Self::Owned(data) => data, + } + } +} + +fn member_window_with_allowance<'a, B: Budget>( + data: &'a [u8], + history: &[u8], + options: EncodeOptions, + allowance: &B, +) -> Result<(MemberWindow<'a, B>, usize)> { + let history = &history[history.len().saturating_sub(options.max_match_distance)..]; + if history.is_empty() { + return Ok((MemberWindow::Borrowed(data), 0)); + } + let combined = Buffer::from_slices(&[history, data], allowance)?; + Ok((MemberWindow::Owned(combined), history.len())) +} + +/// One block, with its own history and its own finder. The member path shares +/// a finder across blocks instead; this is for the callers that encode a block +/// on its own, which are the filtered path and the tests. +#[cfg(test)] +fn encode_lz_block( + data: &[u8], + history: &[u8], + algorithm_version: u8, + initial_filters: &[EncodeFilter], + options: EncodeOptions, + is_last: bool, + progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let (combined, start) = + member_window_with_allowance(data, history, options, &Allowance::default())?; + encode_lz_block_in_window( + &combined, + start..combined.len(), + MemberSearch::Fresh, + algorithm_version, + initial_filters, + options, + is_last, + progress, + ) +} + +#[allow(clippy::too_many_arguments)] +#[cfg(test)] +fn encode_lz_block_in_window( + combined: &[u8], + block: std::ops::Range, + search: MemberSearch<'_>, + algorithm_version: u8, + initial_filters: &[EncodeFilter], + options: EncodeOptions, + is_last: bool, + progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let allowance = match &search { + MemberSearch::Fresh => Allowance::default(), + MemberSearch::Lazy(finder) => finder.allowance(), + MemberSearch::Optimal(collector) => match &collector.finder { + CollectorFinder::Tree(finder) => finder.allowance(), + CollectorFinder::Chains(finder) => finder.allowance(), + }, + }; + encode_lz_block_with_allowance( + combined, + block, + search, + algorithm_version, + initial_filters, + options, + is_last, + progress, + &allowance, + ) + .map(Buffer::into_vec) +} + +#[allow(clippy::too_many_arguments)] +fn encode_lz_block_with_allowance( + combined: &[u8], + block: std::ops::Range, + search: MemberSearch<'_, B>, + algorithm_version: u8, + initial_filters: &[EncodeFilter], + options: EncodeOptions, + is_last: bool, + progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let distance_size = match algorithm_version { + 0 => DISTANCE_TABLE_SIZE_50, + 1 => DISTANCE_TABLE_SIZE_70, + _ => { + return Err(Error::InvalidData( + "RAR 5 unknown compression algorithm version", + )) + } + }; + let mut tokens = encode_tokens_with_allowance( + combined, + block, + search, + options, + distance_size, + initial_filters, + progress, + allowance, + )?; + if !initial_filters.is_empty() { + tokens.prepend(initial_filters.iter().copied().map(EncodeToken::Filter))?; + } + encode_token_block_with_allowance( + &tokens, + algorithm_version, + distance_size, + is_last, + allowance, + ) +} + +#[cfg(test)] +fn encode_token_block( + tokens: &[EncodeToken], + algorithm_version: u8, + distance_size: usize, + is_last: bool, +) -> Result> { + encode_token_block_with_allowance( + tokens, + algorithm_version, + distance_size, + is_last, + &Allowance::default(), + ) + .map(Buffer::into_vec) +} + +fn encode_token_block_with_allowance( + tokens: &[EncodeToken], + algorithm_version: u8, + distance_size: usize, + is_last: bool, + allowance: &B, +) -> Result> { + let lengths = table_lengths_with_allowance(tokens, &[], distance_size, allowance)?; + + let main_table = EncoderCodeTable::from_lengths(&lengths.main, allowance)?; + let distance_table = EncoderCodeTable::from_lengths(&lengths.distance, allowance)?; + let align_table = EncoderCodeTable::from_lengths(&lengths.align, allowance)?; + let length_table = EncoderCodeTable::from_lengths(&lengths.length, allowance)?; + let (table_data, table_bits) = + encode_table_slices(lengths.slices(), algorithm_version, allowance)?; + let payload_bits = + token_stream_bits_after_tables(tokens, &[], &lengths, distance_size, table_bits)?; + let mut writer = BitWriter { + bytes: table_data, + bit_pos: table_bits, + }; + writer + .bytes + .reserve_total_capacity(payload_bits.div_ceil(8))?; + let mut state = EncoderMatchState::default(); + for &token in tokens { + match token { + EncodeToken::Filter(filter) => { + let (code, len) = main_table.code_for_present_symbol(256); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + // token_stream_bits_after_tables validated this same record + // before its exact payload allocation was admitted. + write_valid_filter(&mut writer, filter); + } + EncodeToken::Literal(byte) => { + let (code, len) = main_table.code_for_present_symbol(byte as usize); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + } + EncodeToken::Match { length, distance } => { + match state.encode_valid_match(length, distance, distance_size) { + EncodedMatch::LastLengthRepeat => { + let (code, len) = main_table.code_for_present_symbol(257); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + } + EncodedMatch::RepeatDistance { + index, + length_slot, + length_extra, + } => { + let (code, len) = main_table.code_for_present_symbol(258 + index); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + let (code, len) = length_table.code_for_present_symbol(length_slot); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + let length_extra_bits = length_slot_extra_bits(length_slot); + if length_extra_bits != 0 { + writer + .write_admitted_bits(length_extra, usize::from(length_extra_bits)); + } + } + EncodedMatch::New { + length_slot, + length_extra, + distance_slot, + distance_extra, + distance_bit_count, + } => { + let (code, len) = main_table.code_for_present_symbol(262 + length_slot); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + let length_extra_bits = length_slot_extra_bits(length_slot); + if length_extra_bits != 0 { + writer + .write_admitted_bits(length_extra, usize::from(length_extra_bits)); + } + let (code, len) = distance_table.code_for_present_symbol(distance_slot); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + if distance_bit_count >= 4 { + if distance_bit_count > 4 { + writer.write_admitted_bits( + distance_extra >> 4, + distance_bit_count - 4, + ); + } + let (code, len) = + align_table.code_for_present_symbol(distance_extra & 0x0f); + writer.write_admitted_bits(usize::from(code), usize::from(len)); + } else if distance_bit_count != 0 { + writer.write_admitted_bits(distance_extra, distance_bit_count); + } + } + } + state.remember(length, distance); + } + } + } + + debug_assert_eq!(writer.bit_pos, payload_bits); + encode_compressed_block_with_allowance(&writer.bytes, payload_bits, true, is_last, allowance) +} + +#[derive(Debug)] +struct EncoderState { + history: Buffer, + options: EncodeOptions, +} + +impl EncoderState { + fn new(options: EncodeOptions, allowance: &B) -> Self { + Self { + history: Buffer::new(allowance), + options, + } + } + fn encode( + &mut self, + input: &[u8], + version: u8, + filters: Option<&[crate::FilterSpec]>, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + let allowance = self.history.allowance(); + let packed = match filters { + None => encode_member_with_allowance( + input, + &self.history, + version, + self.options, + progress, + &allowance, + )?, + Some(filters) if input.len() > FILTERED_LZ_BLOCK_SIZE => filtered_lz_blocks( + input, + filters, + &self.history, + version, + self.options, + progress, + &allowance, + )?, + Some(filters) => { + let (filtered, records) = + filtered_member_with_allowance(input, filters, &allowance)?; + encode_filtered_member_with_allowance( + &filtered, + &self.history, + version, + &records, + self.options, + progress, + &allowance, + )? + } + }; + // Commit history only after the encode and its callback succeed. The + // packed result remains charged while any history growth is admitted. + self.history + .remember(input, self.options.max_match_distance)?; + Ok(packed) + } +} + +#[derive(Debug)] +pub struct Unpack50Encoder { + state: EncoderState, +} + +impl Clone for Unpack50Encoder { + fn clone(&self) -> Self { + Self { + state: EncoderState { + history: Buffer::from_vec(self.state.history.to_vec()), + options: self.state.options, + }, + } + } +} + +impl Default for Unpack50Encoder { + fn default() -> Self { + Self::with_options(EncodeOptions::default()) + } +} + +impl Unpack50Encoder { + pub fn new() -> Self { + Self::default() + } + pub fn with_options(options: EncodeOptions) -> Self { + Self { + state: EncoderState::new(options, &Allowance::default()), + } + } + pub fn encode_member(&mut self, input: &[u8], algorithm_version: u8) -> Result> { + self.state + .encode(input, algorithm_version, None, None) + .map(Buffer::into_vec) + } + pub fn encode_member_with_filter( + &mut self, + input: &[u8], + algorithm_version: u8, + filter: crate::FilterSpec, + ) -> Result> { + self.encode_member_with_filters(input, algorithm_version, &[filter]) + } + pub fn encode_member_with_filters( + &mut self, + input: &[u8], + algorithm_version: u8, + filters: &[crate::FilterSpec], + ) -> Result> { + self.state + .encode(input, algorithm_version, Some(filters), None) + .map(Buffer::into_vec) + } + #[cfg(test)] + pub(crate) fn encode_member_with_filters_and_progress( + &mut self, + input: &[u8], + algorithm_version: u8, + filters: &[crate::FilterSpec], + progress: &mut dyn FnMut(usize) -> bool, + ) -> Result> { + self.state + .encode(input, algorithm_version, Some(filters), Some(progress)) + .map(Buffer::into_vec) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EncodeToken { + Filter(EncodeFilter), + Literal(u8), + Match { length: usize, distance: usize }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct EncodeFilter { + offset: usize, + length: usize, + filter_type: FilterType, + channels: usize, +} + +#[derive(Debug, Clone, Copy, Default)] +struct EncoderMatchState { + reps: [usize; 4], + last_length: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EncodedMatch { + LastLengthRepeat, + RepeatDistance { + index: usize, + length_slot: usize, + length_extra: usize, + }, + New { + length_slot: usize, + length_extra: usize, + distance_slot: usize, + distance_extra: usize, + distance_bit_count: usize, + }, +} + +impl EncoderMatchState { + fn encode_match( + &self, + length: usize, + distance: usize, + distance_size: usize, + ) -> Result { + if distance == self.reps[0] && length == self.last_length { + return Ok(EncodedMatch::LastLengthRepeat); + } + if let Some(index) = self + .reps + .iter() + .position(|&repeat_distance| repeat_distance == distance && repeat_distance != 0) + { + let (length_slot, length_extra) = length_slot_for_match(length)?; + return Ok(EncodedMatch::RepeatDistance { + index, + length_slot, + length_extra, + }); + } + + let (distance_slot, distance_extra) = distance_slot_for_match(distance, distance_size)?; + let encoded_length = length + .checked_sub(length_bonus(distance)) + .ok_or(Error::InvalidData("RAR 5 adjusted match length underflows"))?; + let distance_bit_count = distance_slot_bit_count(distance_slot)?; + let (length_slot, length_extra) = length_slot_for_match(encoded_length)?; + Ok(EncodedMatch::New { + length_slot, + length_extra, + distance_slot, + distance_extra, + distance_bit_count, + }) + } + + /// Encode a match after [`table_lengths_with_allowance`] has validated the + /// same immutable token sequence from the same initial state. + fn encode_valid_match( + &self, + length: usize, + distance: usize, + distance_size: usize, + ) -> EncodedMatch { + if distance == self.reps[0] && length == self.last_length { + return EncodedMatch::LastLengthRepeat; + } + if let Some(index) = self + .reps + .iter() + .position(|&repeat_distance| repeat_distance == distance && repeat_distance != 0) + { + let (length_slot, length_extra) = length_slot_for_valid_match(length); + return EncodedMatch::RepeatDistance { + index, + length_slot, + length_extra, + }; + } + + let (distance_slot, distance_extra) = distance_slot_for_valid_match(distance); + debug_assert!(distance_slot < distance_size); + let bonus = length_bonus(distance); + debug_assert!(length >= bonus + 2); + let (length_slot, length_extra) = length_slot_for_valid_match(length - bonus); + EncodedMatch::New { + length_slot, + length_extra, + distance_slot, + distance_extra, + distance_bit_count: distance_slot_bit_count_valid(distance_slot), + } + } + + fn remember(&mut self, length: usize, distance: usize) { + if distance == self.reps[0] && length == self.last_length { + return; + } + if let Some(index) = self + .reps + .iter() + .position(|&repeat_distance| repeat_distance == distance) + { + self.reps[..=index].rotate_right(1); + } else { + self.reps.rotate_right(1); + } + self.reps[0] = distance; + self.last_length = length; + } +} + +/// The Huffman code lengths a block of tokens produces. The block writer needs +/// these to emit the tables; the optimal parse needs them to know what each +/// token it is considering will actually cost. +#[cfg(test)] +fn table_lengths_for_tokens(tokens: &[EncodeToken], distance_size: usize) -> Result { + table_lengths_with_filters(tokens, &[], distance_size) +} + +#[cfg(test)] +fn table_lengths_with_filters( + tokens: &[EncodeToken], + filters: &[EncodeFilter], + distance_size: usize, +) -> Result { + table_lengths_with_allowance(tokens, filters, distance_size, &Allowance::default()) +} + +fn table_lengths_with_allowance( + tokens: &[EncodeToken], + filters: &[EncodeFilter], + distance_size: usize, + allowance: &B, +) -> Result> { + let mut main_frequencies = Buffer::filled(MAIN_TABLE_SIZE, 0usize, allowance)?; + main_frequencies[256] = filters.len(); + let mut distance_frequencies = Buffer::filled(distance_size, 0usize, allowance)?; + let mut align_frequencies = Buffer::filled(ALIGN_TABLE_SIZE, 0usize, allowance)?; + let mut length_frequencies = Buffer::filled(LENGTH_TABLE_SIZE, 0usize, allowance)?; + let mut state = EncoderMatchState::default(); + for token in tokens { + match *token { + EncodeToken::Filter(_) => main_frequencies[256] += 1, + EncodeToken::Literal(byte) => main_frequencies[byte as usize] += 1, + EncodeToken::Match { length, distance } => { + match state.encode_match(length, distance, distance_size)? { + EncodedMatch::LastLengthRepeat => main_frequencies[257] += 1, + EncodedMatch::RepeatDistance { + index, length_slot, .. + } => { + main_frequencies[258 + index] += 1; + length_frequencies[length_slot] += 1; + } + EncodedMatch::New { + length_slot, + distance_slot, + distance_extra, + distance_bit_count, + .. + } => { + main_frequencies[262 + length_slot] += 1; + distance_frequencies[distance_slot] += 1; + if distance_bit_count >= 4 { + align_frequencies[distance_extra & 0x0f] += 1; + } + } + } + state.remember(length, distance); + } + } + } + + Ok(OwnedLengths { + main: huffman::complete_lengths_with_allowance(&main_frequencies, 15, allowance)?, + distance: huffman::complete_lengths_with_allowance(&distance_frequencies, 15, allowance)?, + align: huffman::complete_lengths_with_allowance(&align_frequencies, 15, allowance)?, + length: huffman::complete_lengths_with_allowance(&length_frequencies, 15, allowance)?, + }) +} + +/// Actual payload size, including the transmitted tables and filter records. +/// Padding and block-header size are monotonic in this bit count. +fn token_stream_bits( + tokens: &[EncodeToken], + filters: &[EncodeFilter], + lengths: &OwnedLengths, + distance_size: usize, +) -> Result { + let version = if distance_size == DISTANCE_TABLE_SIZE_70 { + 1 + } else { + 0 + }; + let allowance = lengths.main.allowance(); + let (_, bits) = encode_table_slices(lengths.slices(), version, &allowance)?; + token_stream_bits_after_tables(tokens, filters, lengths, distance_size, bits) +} + +fn token_stream_bits_after_tables( + tokens: &[EncodeToken], + filters: &[EncodeFilter], + lengths: &OwnedLengths, + distance_size: usize, + mut bits: usize, +) -> Result { + let allowance = lengths.main.allowance(); + let prices = TokenPrices { + lengths: lengths.slices(), + }; + let mut state = EncoderMatchState::default(); + for token in filters + .iter() + .copied() + .map(EncodeToken::Filter) + .chain(tokens.iter().copied()) + { + match token { + EncodeToken::Literal(byte) => bits += prices.literal(byte), + EncodeToken::Match { length, distance } => { + bits += prices.match_cost(&state, length, distance, distance_size)?; + state.remember(length, distance); + } + EncodeToken::Filter(filter) => { + let mut writer = BitWriter::with_allowance(&allowance); + try_write_filter(&mut writer, filter)?; + bits += usize::from(lengths.main[256]) + writer.bit_pos; + } + } + } + Ok(bits) +} + +struct OptimalWorkspace { + price: Buffer, + arrive_length: Buffer, + arrive_distance: Buffer, + arrive_reps: Buffer<[u32; 4], B>, + arrive_last_length: Buffer, +} + +impl OptimalWorkspace { + fn new(allowance: &B) -> Self { + Self { + price: Buffer::new(allowance), + arrive_length: Buffer::new(allowance), + arrive_distance: Buffer::new(allowance), + arrive_reps: Buffer::new(allowance), + arrive_last_length: Buffer::new(allowance), + } + } +} + +/// What a literal is assumed to cost before any block has been coded, in the +/// same bit units [`estimated_match_cost`] reports. A literal is one main-table +/// symbol out of 256 plus the odds that the table is skewed, so eight is the +/// floor and nine is what real blocks measure. +const ESTIMATED_LITERAL_COST: u32 = 9; + +/// How many times the optimal parse runs over a block. The first pass guesses +/// prices; the rest reprice against the tables the pass before produced. +const OPTIMAL_PARSE_PASSES: usize = 3; + +/// What a symbol the first pass never used is assumed to cost. Reaching for +/// one is not forbidden, only expensive: the tables are rebuilt from whatever +/// the last pass chose, so a symbol that earns its place gets a real code. +const UNUSED_SYMBOL_COST: usize = 15; + +/// Prices a token against the code lengths a previous pass produced, which is +/// what the block will really spend, rather than against the flat guess in +/// [`estimated_match_cost`]. +struct TokenPrices<'a> { + lengths: LengthSlices<'a>, +} + +impl TokenPrices<'_> { + fn code(bits: u8) -> usize { + if bits == 0 { + UNUSED_SYMBOL_COST + } else { + usize::from(bits) + } + } + + fn literal(&self, byte: u8) -> usize { + Self::code(self.lengths.main[byte as usize]) + } + + // This is the inner optimal-parse loop's pricing operation. Keep it in + // that loop rather than returning a codec Result through a stack slot for + // every candidate length, including across generic codegen units. + #[inline(always)] + fn match_cost( + &self, + state: &EncoderMatchState, + length: usize, + distance: usize, + distance_size: usize, + ) -> Result { + Ok(match state.encode_match(length, distance, distance_size)? { + EncodedMatch::LastLengthRepeat => Self::code(self.lengths.main[257]), + EncodedMatch::RepeatDistance { + index, length_slot, .. + } => { + Self::code(self.lengths.main[258 + index]) + + Self::code(self.lengths.length[length_slot]) + + usize::from(length_slot_extra_bits(length_slot)) + } + EncodedMatch::New { + length_slot, + distance_slot, + distance_extra, + distance_bit_count, + .. + } => { + let align = if distance_bit_count >= 4 { + distance_bit_count - 4 + Self::code(self.lengths.align[distance_extra & 0x0f]) + } else { + distance_bit_count + }; + Self::code(self.lengths.main[262 + length_slot]) + + usize::from(length_slot_extra_bits(length_slot)) + + Self::code(self.lengths.distance[distance_slot]) + + align + } + }) + } +} + +/// Length-slot prices are invariant throughout one pass. A run also keeps the +/// same distance and arriving state while it prices every possible endpoint. +struct OptimalMatchPrices { + new_lengths: [u32; LENGTH_TABLE_SIZE], + repeat_lengths: [u32; LENGTH_TABLE_SIZE], + repeats: [u32; 4], + last_repeat: u32, +} + +struct MatchRunPrice<'a> { + lengths: &'a [u32; LENGTH_TABLE_SIZE], + distance: u32, + bonus: usize, + last_length: Option, + last_repeat: u32, +} + +impl OptimalMatchPrices { + fn new(prices: Option<&TokenPrices<'_>>) -> Self { + Self { + new_lengths: std::array::from_fn(|slot| { + let code = prices.map_or(0, |prices| { + TokenPrices::code(prices.lengths.main[262 + slot]) + }); + (code + usize::from(length_slot_extra_bits(slot))) as u32 + }), + repeat_lengths: std::array::from_fn(|slot| { + let code = + prices.map_or(0, |prices| TokenPrices::code(prices.lengths.length[slot])); + (code + usize::from(length_slot_extra_bits(slot))) as u32 + }), + repeats: std::array::from_fn(|index| { + prices.map_or(5, |prices| { + TokenPrices::code(prices.lengths.main[258 + index]) as u32 + }) + }), + last_repeat: prices.map_or(2, |prices| { + TokenPrices::code(prices.lengths.main[257]) as u32 + }), + } + } + + fn run( + &self, + state: &EncoderMatchState, + distance: usize, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + ) -> Result> { + if let Some(index) = state + .reps + .iter() + .position(|&repeat| repeat == distance && repeat != 0) + { + return Ok(MatchRunPrice { + lengths: &self.repeat_lengths, + distance: self.repeats[index], + bonus: 0, + last_length: (index == 0).then_some(state.last_length), + last_repeat: self.last_repeat, + }); + } + let (slot, extra) = distance_slot_for_match(distance, distance_size)?; + let bits = distance_slot_bit_count(slot)?; + let cost = prices.map_or(10 + bits, |prices| { + TokenPrices::code(prices.lengths.distance[slot]) + + if bits >= 4 { + bits - 4 + TokenPrices::code(prices.lengths.align[extra & 0x0f]) + } else { + bits + } + }); + Ok(MatchRunPrice { + lengths: &self.new_lengths, + distance: cost as u32, + bonus: length_bonus(distance), + last_length: None, + last_repeat: self.last_repeat, + }) + } +} + +impl MatchRunPrice<'_> { + #[inline] + fn cost(&self, length: usize) -> Option { + if self.last_length == Some(length) { + return Some(self.last_repeat); + } + let encoded = length.checked_sub(self.bonus)?; + if encoded < 2 { + return None; + } + debug_assert!(length <= MAX_ENCODER_MATCH_LENGTH); + let (slot, _) = length_slot_for_valid_match(encoded); + Some(self.distance + self.lengths[slot]) + } +} + +struct OptimalSlices<'a> { + price: &'a mut [u32], + arrive_length: &'a mut [u32], + arrive_distance: &'a mut [u32], + arrive_reps: &'a mut [[u32; 4]], + arrive_last_length: &'a mut [u32], +} + +// No growth or ownership changes occur here. Keep one pricing implementation +// for bounded and unlimited execution instead of specializing this hot loop on +// their differently sized allocation owners and fallible push operations. +#[allow(clippy::too_many_arguments)] +fn price_optimal_paths( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + runs: &[(u32, u32)], + starts: &[u32], + workspace: OptimalSlices<'_>, + reaches: &mut [(usize, usize, usize)], +) { + let start = block.start; + let end = block.end; + let span = end - start; + let OptimalSlices { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + } = workspace; + // The first position past a match the parse committed to. Nothing is + // priced from the positions before it. See [`NICE_MATCH_LENGTH`]. + let mut committed_through = 0usize; + let match_prices = OptimalMatchPrices::new(prices); + + for index in 0..span { + let pos = start + index; + if index < committed_through { + continue; + } + // Every priced position extends a literal path to its successor; + // a committed match skips only to an already priced endpoint. Blocks + // are at most 1 MiB, so even all 15-bit literals stay below u32::MAX. + let here = price[index]; + let literal_cost = prices.map_or(ESTIMATED_LITERAL_COST, |prices| { + prices.literal(combined[pos]) as u32 + }); + let literal = here.saturating_add(literal_cost); + if literal < price[index + 1] { + price[index + 1] = literal; + arrive_length[index + 1] = 0; + arrive_distance[index + 1] = 0; + // A literal emits no distance, so it leaves the remembered ones + // exactly as it found them. + arrive_reps[index + 1] = arrive_reps[index]; + arrive_last_length[index + 1] = arrive_last_length[index]; + } + + let max_distance = pos.min(options.max_match_distance); + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if options.max_match_candidates == 0 || max_distance == 0 || max_length < 4 { + continue; + } + + let state = EncoderMatchState { + reps: arrive_reps[index].map(|distance| distance as usize), + last_length: arrive_last_length[index] as usize, + }; + + let mut reaches_len = 0; + let mut longest = 0usize; + + // A match at a remembered distance is priced out of the main table + // alone, a handful of bits against twenty for a fresh distance, so it + // earns its place even when it is shorter than anything the collector + // found. The collector only reports a candidate that beats the longest + // found so far, so these have to be asked for separately. + for repeat in state.reps { + if repeat == 0 || repeat > max_distance { + continue; + } + let length = match_length(combined, pos, repeat, max_length); + if length >= 4 { + reaches[reaches_len] = (4, length, repeat); + reaches_len += 1; + } + } + + // The collector reports nearest first, so the first distance to reach + // a length is the cheapest one that can. Each report that improves on + // the longest so far owns one run of lengths. The tree measures + // against the whole member where the chains stopped at the block, so + // a length is capped here to what this block can still hold. + for &(length, distance) in &runs[starts[index] as usize..starts[index + 1] as usize] { + let length = (length as usize).min(max_length); + if length > longest { + reaches[reaches_len] = (longest + 1, length, distance as usize); + reaches_len += 1; + longest = length; + } + } + + let reaches = &reaches[..reaches_len]; + + // Equal token prices do not make shorter matches redundant: their + // endpoints can expose a better continuation. Price every endpoint + // unless the explicit long-match heuristic commits past all of them. + let committed_reach = reaches + .iter() + .map(|&(_, end, _)| end) + .max() + .filter(|&length| length >= options.nice_length); + for &(run_start, run_end, distance) in reaches.iter() { + let mut length = run_start.max(4); + if let Some(committed) = committed_reach { + if run_end < committed { + continue; + } + length = committed; + } + let Ok(run_price) = match_prices.run(&state, distance, distance_size, prices) else { + continue; + }; + while length <= run_end { + let reach = length; + if let Some(cost) = run_price.cost(reach) { + let reached = here.saturating_add(cost); + let target = index + reach; + if reached < price[target] { + price[target] = reached; + arrive_length[target] = reach as u32; + arrive_distance[target] = distance as u32; + let mut next = state; + next.remember(reach, distance); + arrive_reps[target] = next.reps.map(|distance| distance as u32); + arrive_last_length[target] = next.last_length as u32; + } + } + length = reach + 1; + } + } + + // A committed match is at least 512 bytes long. Even after the + // distance bonus, that length and every collected u32 distance fit + // their encoder tables, so the endpoint has already been priced. + let longest_reach = reaches.iter().map(|&(_, length, _)| length).max(); + if let Some(reach) = longest_reach { + if reach >= options.nice_length { + committed_through = index + reach; + } + } + } +} + +/// Prices every path through the block and keeps the cheapest, instead of +/// taking the longest match at each position and checking one or two bytes +/// ahead. Prices come from [`estimated_match_cost`], so this is only as good +/// as that estimate, but it sees the whole block where lazy matching sees two +/// bytes. +/// +/// The repeated-distance discount depends on the path taken, which a forward +/// pass does not know. Each node carries the whole four-slot distance memory +/// the cheapest path to it leaves behind, so the next hop is priced against +/// what that path would really have remembered. Two paths reaching one node +/// with different memories still collapse into whichever was cheaper, so this +/// stays an approximation, just a far closer one than carrying the arriving +/// match alone. It is also not quite every path: once a match reaches +/// [`NICE_MATCH_LENGTH`] the parse takes it and steps over the bytes it covers +/// rather than pricing each of them. +/// +/// Does no searching of its own: `matches` holds what an [`OptimalCollector`] +/// found at each position of this block, and prices never change what a +/// search would find, so every pass prices the same collection. +fn optimal_tokens_in_workspace( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + matches: &BlockMatches, + workspace: &mut OptimalWorkspace, +) -> Result> { + #[cfg(test)] + convergence_tests::record_pricing_pass(); + let start = block.start; + let end = block.end; + let span = end - start; + + let allowance = workspace.price.allowance().clone(); + if !B::LIMITED && matches.runs.is_empty() { + // Every parse starts with empty distance memory. Without a collected + // match, literals cannot introduce a remembered distance, so they are + // the only path through this block under any set of table prices. + // Limited parses retain their allocation and refusal order below. + return Buffer::collect( + combined[block].iter().copied().map(EncodeToken::Literal), + &allowance, + ); + } + let OptimalWorkspace { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + } = workspace; + price.resize(span + 1, u32::MAX)?; + price.fill(u32::MAX); + arrive_length.resize(span + 1, 0)?; + arrive_length.fill(0); + arrive_distance.resize(span + 1, 0)?; + arrive_distance.fill(0); + arrive_reps.resize(span + 1, [0; 4])?; + arrive_reps.fill([0; 4]); + arrive_last_length.resize(span + 1, 0)?; + arrive_last_length.fill(0); + price[0] = 0; + + // Admit the largest candidate list once. Pricing only borrows already + // charged arrays, so its inner loop is identical for both budget policies. + // Each position contributes at most four remembered-distance candidates. + let longest_run = matches + .starts + .windows(2) + .map(|pair| (pair[1] - pair[0]) as usize) + .max() + .unwrap_or(0); + // A position contributes at most one run for each encodable match length. + // Four remembered distances are added while pricing, so this bound cannot + // approach usize::MAX for a valid BlockMatches collection. + debug_assert!(longest_run <= MAX_ENCODER_MATCH_LENGTH); + let reach_capacity = longest_run + 4; + let mut reaches = Buffer::filled(reach_capacity, (0usize, 0usize, 0usize), &allowance)?; + price_optimal_paths( + combined, + block.clone(), + options, + distance_size, + prices, + &matches.runs, + &matches.starts, + OptimalSlices { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + }, + &mut reaches, + ); + + let mut reversed = Buffer::new(&allowance); + let mut index = span; + while index > 0 { + let length = arrive_length[index] as usize; + if length == 0 { + reversed + .push(EncodeToken::Literal(combined[start + index - 1])) + .map_err(Into::into)?; + index -= 1; + } else { + reversed + .push(EncodeToken::Match { + length, + distance: arrive_distance[index] as usize, + }) + .map_err(Into::into)?; + index -= length; + } + } + reversed.reverse(); + Ok(reversed) +} + +#[cfg(test)] +fn encode_tokens_with_progress( + combined: &[u8], + block: std::ops::Range, + search: MemberSearch<'_>, + options: EncodeOptions, + distance_size: usize, + initial_filters: &[EncodeFilter], + progress: Option<&mut dyn FnMut(usize) -> bool>, +) -> Result> { + let allowance = match &search { + MemberSearch::Fresh => Allowance::default(), + MemberSearch::Lazy(finder) => finder.allowance().clone(), + MemberSearch::Optimal(collector) => match &collector.finder { + CollectorFinder::Tree(finder) => finder.allowance().clone(), + CollectorFinder::Chains(finder) => finder.allowance().clone(), + }, + }; + encode_tokens_with_allowance( + combined, + block, + search, + options, + distance_size, + initial_filters, + progress, + &allowance, + ) +} + +#[allow(clippy::too_many_arguments)] +fn encode_tokens_with_allowance( + combined: &[u8], + block: std::ops::Range, + search: MemberSearch<'_, B>, + options: EncodeOptions, + distance_size: usize, + initial_filters: &[EncodeFilter], + mut progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let start = block.start; + let end = block.end; + if options.optimal_parse { + let mut own; + let collector = match search { + MemberSearch::Optimal(collector) => collector, + _ => { + own = OptimalCollector::with_allowance(combined, start, options, allowance)?; + &mut own + } + }; + let matches = collector.collect(combined, block.clone(), options)?; + // The prices come from the Huffman tables, and the tables come from + // the parse, so the first pass has to guess. Each pass after it prices + // against what the pass before actually produced. + let mut workspace = OptimalWorkspace::new(allowance); + let mut tokens = optimal_tokens_in_workspace( + combined, + block.clone(), + options, + distance_size, + None, + &matches, + &mut workspace, + )?; + let mut lengths = + table_lengths_with_allowance(&tokens, initial_filters, distance_size, allowance)?; + let mut best_bits = token_stream_bits(&tokens, initial_filters, &lengths, distance_size)?; + // Without a collected match, every pass starts with no remembered + // distance and can only produce the same literals. Keep the first + // tables and stream validation before skipping the equal reparse. + // Limited callers retain every original admission and refusal point. + let passes = if !B::LIMITED && matches.runs.is_empty() { + 1 + } else { + options.parse_passes + }; + let mut best = None; + for _ in 1..passes { + let prices = TokenPrices { + lengths: lengths.slices(), + }; + let next = optimal_tokens_in_workspace( + combined, + block.clone(), + options, + distance_size, + Some(&prices), + &matches, + &mut workspace, + )?; + if next == tokens { + break; + } + let next_lengths = + table_lengths_with_allowance(&next, initial_filters, distance_size, allowance)?; + let tables_converged = lengths.slices() == next_lengths.slices(); + lengths = next_lengths; + let bits = token_stream_bits(&next, initial_filters, &lengths, distance_size)?; + if bits < best_bits { + best_bits = bits; + best = None; + } else if best.is_none() { + // Keep an earlier winner only when repricing actually loses. + best = Some(std::mem::replace(&mut tokens, Buffer::new(allowance))); + } + tokens = next; + if tables_converged { + // With the same matches, initial state and table prices, the + // next pass would reproduce these tokens and stop above. Keep + // the normal winner and tie handling before skipping it. + break; + } + } + let tokens = best.unwrap_or(tokens); + if progress.is_some_and(|report| !report(end - start)) { + return Err(Error::Cancelled); + } + return Ok(tokens); + } + + let mut own; + let finder = match search { + MemberSearch::Lazy(finder) => finder, + _ => { + own = seeded_finder(combined, start..end, options, allowance)?; + &mut own + } + }; + let mut tokens = Buffer::new(allowance); + let mut pos = start; + let mut state = EncoderMatchState::default(); + let mut next_report = 0usize; + let mut pending_match: Option = None; + while pos < end { + let candidate = pending_match + .take() + .or_else(|| best_match(combined, pos, end, finder, options, &state, distance_size)); + if let Some(candidate) = candidate { + let (emit_literal, cached_next) = lazy_match_decision( + combined, + pos, + end, + finder, + options, + &state, + distance_size, + candidate, + ); + if emit_literal { + tokens + .push(EncodeToken::Literal(combined[pos])) + .map_err(Into::into)?; + finder.insert(combined, pos); + pos += 1; + pending_match = cached_next; + continue; + } + let MatchCandidate { + length, distance, .. + } = candidate; + tokens + .push(EncodeToken::Match { length, distance }) + .map_err(Into::into)?; + state.remember(length, distance); + for history_pos in pos..pos + length { + finder.insert(combined, history_pos); + } + pos += length; + } else { + tokens + .push(EncodeToken::Literal(combined[pos])) + .map_err(Into::into)?; + finder.insert(combined, pos); + pos += 1; + } + let consumed = pos - start; + if consumed >= next_report { + if progress + .as_deref_mut() + .is_some_and(|report| !report(consumed)) + { + return Err(Error::Cancelled); + } + next_report = consumed.saturating_add(1024 * 1024); + } + } + if progress.is_some_and(|report| !report(end - start)) { + return Err(Error::Cancelled); + } + Ok(tokens) +} + +#[cfg(test)] +#[path = "convergence_tests.rs"] +mod convergence_tests; + +/// Decides whether a literal should be emitted instead of `current` because a +/// better match starts within the lazy lookahead window. Also returns the +/// match found one byte ahead (when computed) so the caller can reuse it for +/// the next position instead of searching again. +#[allow(clippy::too_many_arguments)] +fn lazy_match_decision( + input: &[u8], + pos: usize, + end: usize, + finder: &Rar50MatchFinder, + options: EncodeOptions, + state: &EncoderMatchState, + distance_size: usize, + current: MatchCandidate, +) -> (bool, Option) { + if !options.lazy_matching { + return (false, None); + } + let lookahead = options.lazy_lookahead.max(1); + let mut cached_next = None; + for offset in 1..=lookahead { + if pos + offset >= end { + break; + } + let next = best_match( + input, + pos + offset, + end, + finder, + options, + state, + distance_size, + ); + if offset == 1 { + cached_next = next; + } + let skipped_literal_score = offset as isize * 8; + if next.is_some_and(|next| next.score > current.score + skipped_literal_score) { + return (true, cached_next); + } + } + (false, None) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct MatchCandidate { + length: usize, + distance: usize, + score: isize, +} + +fn best_match( + input: &[u8], + pos: usize, + end: usize, + finder: &Rar50MatchFinder, + options: EncodeOptions, + state: &EncoderMatchState, + distance_size: usize, +) -> Option { + let max_distance = pos.min(options.max_match_distance); + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if options.max_match_candidates == 0 || max_distance == 0 || max_length < 4 { + return None; + } + let mut best = None; + let mut checked = 0usize; + for distance in state.reps { + // Remembered matches were admitted under this same distance limit at + // an earlier position, so only unfilled repeat slots need skipping. + if distance == 0 { + continue; + } + let length = match_length(input, pos, distance, max_length); + consider_match_candidate(&mut best, state, distance_size, length, distance); + } + if let Some(best) = best { + if best.length == max_length || best.length >= options.nice_length { + return Some(best); + } + } + let mut candidate = finder.first(input, pos); + while candidate != match_finder::NO_POSITION { + let distance = pos - candidate; + if distance > max_distance { + break; + } + checked += 1; + // A candidate can only improve on the current best when it matches at + // least one byte past the best length, so probe that byte first. + let best_length = best.map_or(0, |best: MatchCandidate| best.length); + if best_length == 0 || input[candidate + best_length] == input[pos + best_length] { + let length = match_length(input, pos, distance, max_length); + consider_match_candidate(&mut best, state, distance_size, length, distance); + } + if let Some(best) = best { + if best.length == max_length || best.length >= options.nice_length { + break; + } + } + if checked >= options.max_match_candidates { + break; + } + candidate = finder.previous(candidate); + } + best +} + +fn match_length(input: &[u8], pos: usize, distance: usize, max_length: usize) -> usize { + super::super::fast::match_length(input, pos, distance, max_length) +} + +fn consider_match_candidate( + best: &mut Option, + state: &EncoderMatchState, + distance_size: usize, + length: usize, + distance: usize, +) { + if length < 4 { + return; + } + let Ok(cost) = estimated_match_cost(state, length, distance, distance_size) else { + return; + }; + let candidate = MatchCandidate { + length, + distance, + score: (length as isize * 16) - cost as isize, + }; + if best.is_none_or(|best| { + candidate.score > best.score + || (candidate.score == best.score + && (candidate.length > best.length + || (candidate.length == best.length && candidate.distance < best.distance))) + }) { + *best = Some(candidate); + } +} + +// Generic parsers can be instantiated in a different codegen unit. Keep the +// estimate available for inlining into their per-length pricing loop. +#[inline] +fn estimated_match_cost( + state: &EncoderMatchState, + length: usize, + distance: usize, + distance_size: usize, +) -> Result { + if distance == state.reps[0] && length == state.last_length { + return Ok(2); + } + if state + .reps + .iter() + .any(|&repeat_distance| repeat_distance == distance && repeat_distance != 0) + { + let (length_slot, _) = length_slot_for_match(length)?; + return Ok(5 + usize::from(length_slot_extra_bits(length_slot))); + } + + let (distance_slot, _) = distance_slot_for_match(distance, distance_size)?; + let encoded_length = length + .checked_sub(length_bonus(distance)) + .ok_or(Error::InvalidData("RAR 5 adjusted match length underflows"))?; + let (length_slot, _) = length_slot_for_match(encoded_length)?; + Ok(10 + + usize::from(length_slot_extra_bits(length_slot)) + + distance_slot_bit_count(distance_slot)?) +} + +fn length_slot_for_match(length: usize) -> Result<(usize, usize)> { + if length < 2 { + return Err(Error::InvalidData("RAR 5 match length is too short")); + } + Ok(length_slot_for_valid_match(length)) +} + +fn length_slot_for_valid_match(length: usize) -> (usize, usize) { + debug_assert!(length >= 2); + let value = length - 2; + if value < 8 { + return (value, 0); + } + let bit_count = value.ilog2() as usize - 2; + let slot = ((bit_count + 1) << 2) | ((value >> bit_count) & 3); + // The encoder caps matches at 4096 bytes, which fits slots 0..44. + (slot, value & ((1 << bit_count) - 1)) +} + +fn distance_slot_for_match(distance: usize, distance_size: usize) -> Result<(usize, usize)> { + // Every emitted match comes from an earlier input position, and the two + // production distance tables both have at least four entries. + if distance == 0 { + return Err(Error::InvalidData("RAR 5 match distance is zero")); + } + let result = distance_slot_for_valid_match(distance); + if result.0 >= distance_size { + return Err(Error::InvalidData("RAR 5 match distance is too large")); + } + Ok(result) +} + +fn distance_slot_for_valid_match(distance: usize) -> (usize, usize) { + debug_assert!(distance != 0); + let value = distance - 1; + if value < 4 { + return (value, 0); + } + let bit_count = value.ilog2() as usize - 1; + let slot = (bit_count << 1) + 2 + ((value >> bit_count) & 1); + (slot, value & ((1 << bit_count) - 1)) +} + +fn distance_slot_bit_count_valid(slot: usize) -> usize { + if slot < 4 { + 0 + } else { + (slot - 2) >> 1 + } +} + +fn literal_presence(data: &[u8]) -> [bool; 256] { + let mut present = [false; 256]; + for &byte in data { + present[byte as usize] = true; + } + present +} + +fn try_write_filter(writer: &mut BitWriter, filter: EncodeFilter) -> Result<()> { + if filter.offset > u32::MAX as usize { + return Err(Error::InvalidData("RAR 5 filter offset is too large")); + } + if filter.length > u32::MAX as usize { + return Err(Error::InvalidData("RAR 5 filter length is too large")); + } + try_write_filter_data(writer, filter.offset as u32)?; + try_write_filter_data(writer, filter.length as u32)?; + match filter.filter_type { + FilterType::Delta => { + if filter.channels == 0 || filter.channels > MAX_DELTA_CHANNELS { + return Err(Error::InvalidData( + "RAR 5 DELTA filter channel count is invalid", + )); + } + writer.try_write_bits(0, 3).map_err(Into::into)?; + writer + .try_write_bits(filter.channels - 1, 5) + .map_err(Into::into)?; + } + FilterType::E8 => writer.try_write_bits(1, 3).map_err(Into::into)?, + FilterType::E8E9 => writer.try_write_bits(2, 3).map_err(Into::into)?, + FilterType::Arm => writer.try_write_bits(3, 3).map_err(Into::into)?, + } + Ok(()) +} + +fn write_valid_filter(writer: &mut BitWriter, filter: EncodeFilter) { + debug_assert!(u32::try_from(filter.offset).is_ok()); + debug_assert!(u32::try_from(filter.length).is_ok()); + write_filter_data_admitted(writer, filter.offset as u32); + write_filter_data_admitted(writer, filter.length as u32); + match filter.filter_type { + FilterType::Delta => { + debug_assert!((1..=MAX_DELTA_CHANNELS).contains(&filter.channels)); + writer.write_admitted_bits(0, 3); + writer.write_admitted_bits(filter.channels - 1, 5); + } + FilterType::E8 => writer.write_admitted_bits(1, 3), + FilterType::E8E9 => writer.write_admitted_bits(2, 3), + FilterType::Arm => writer.write_admitted_bits(3, 3), + } +} + +fn try_write_filter_data(writer: &mut BitWriter, value: u32) -> Result<()> { + let byte_count = filter_data_byte_count(value); + writer + .try_write_bits(byte_count - 1, 2) + .map_err(Into::into)?; + for index in 0..byte_count { + writer + .try_write_bits(((value >> (index * 8)) & 0xff) as usize, 8) + .map_err(Into::into)?; + } + Ok(()) +} + +fn write_filter_data_admitted(writer: &mut BitWriter, value: u32) { + let byte_count = filter_data_byte_count(value); + writer.write_admitted_bits(byte_count - 1, 2); + for index in 0..byte_count { + writer.write_admitted_bits(((value >> (index * 8)) & 0xff) as usize, 8); + } +} + +fn filter_data_byte_count(value: u32) -> usize { + ((u32::BITS - value.leading_zeros()).div_ceil(8) as usize).max(1) +} + +fn e8e9_encode(data: &mut [u8], file_offset: u32, include_e9: bool) { + if data.len() <= 4 { + return; + } + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let opcode_limit = data.len() - 4; + let mut opcode_pos = 0usize; + while let Some(pos) = + super::super::fast::next_x86_opcode(data, opcode_pos, opcode_limit, cmp_mask) + { + let cur_pos = pos + 1; + let offset = file_offset.wrapping_add(cur_pos as u32) % X86_FILTER_FILE_SIZE; + let addr = u32::from_le_bytes([ + data[cur_pos], + data[cur_pos + 1], + data[cur_pos + 2], + data[cur_pos + 3], + ]); + let candidate = addr.wrapping_add(offset); + let new_addr = if candidate < X86_FILTER_FILE_SIZE { + Some(candidate) + } else { + let candidate = addr.wrapping_sub(X86_FILTER_FILE_SIZE); + (candidate & 0x8000_0000 != 0 && candidate.wrapping_add(offset) & 0x8000_0000 == 0) + .then_some(candidate) + }; + if let Some(value) = new_addr { + data[cur_pos..cur_pos + 4].copy_from_slice(&value.to_le_bytes()); + } + opcode_pos = pos + 5; + } +} + +fn arm_encode(data: &mut [u8], file_offset: u32) { + let mut pos = 0usize; + while pos + 3 < data.len() { + if data[pos + 3] == 0xeb { + let mut offset = u32::from(data[pos]) + | (u32::from(data[pos + 1]) << 8) + | (u32::from(data[pos + 2]) << 16); + offset = offset.wrapping_add(file_offset.wrapping_add(pos as u32) / 4); + data[pos] = offset as u8; + data[pos + 1] = (offset >> 8) as u8; + data[pos + 2] = (offset >> 16) as u8; + } + pos += 4; + } +} + +struct EncoderCodeTable { + symbols: Buffer<(u16, u8), B>, +} + +impl EncoderCodeTable { + fn from_lengths(lengths: &[u8], allowance: &B) -> Result { + let mut counts = [0u16; 16]; + // Both callers use the encoder's length generators, capped at 15 bits. + for &length in lengths { + if length != 0 { + counts[length as usize] += 1; + } + } + validate_huffman_counts(&counts)?; + let mut next = [0u16; 16]; + let mut code = 0; + for length in 1..16 { + code = (code + counts[length - 1]) << 1; + next[length] = code; + } + let mut symbols = Buffer::filled(lengths.len(), (0u16, 0u8), allowance)?; + for (symbol, &length) in lengths.iter().enumerate() { + if length != 0 { + symbols[symbol] = (next[length as usize], length); + next[length as usize] += 1; + } + } + Ok(Self { symbols }) + } + fn code_for_present_symbol(&self, symbol: usize) -> (u16, u8) { + // Both encoder callers build frequencies from the exact token stream + // they emit immediately afterwards. Every requested symbol therefore + // has a non-zero code in this fixed-size array. + debug_assert!(symbol < self.symbols.len()); + debug_assert_ne!(self.symbols[symbol].1, 0); + self.symbols[symbol] + } +} + +struct BitWriter { + bytes: Buffer, + bit_pos: usize, +} + +impl BitWriter { + #[cfg(test)] + fn new() -> Self { + Self::with_allowance(&Allowance::default()) + } + fn write_bits(&mut self, value: usize, count: usize) { + self.try_write_bits(value, count).unwrap(); + } + fn finish(self) -> Vec { + self.bytes.into_vec() + } +} + +impl BitWriter { + fn with_allowance(allowance: &B) -> Self { + Self { + bytes: Buffer::new(allowance), + bit_pos: 0, + } + } + #[inline] + fn try_write_bits( + &mut self, + value: usize, + count: usize, + ) -> std::result::Result<(), B::Failure> { + self.bytes + .write_msb_bits(&mut self.bit_pos, value as u64, count) + } + fn write_admitted_bits(&mut self, value: usize, count: usize) { + self.bytes + .write_msb_bits_admitted(&mut self.bit_pos, value as u64, count); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct LevelToken { + symbol: usize, + extra_bits: u8, + extra_value: u8, +} + +impl LevelToken { + const fn plain(symbol: usize) -> Self { + Self { + symbol, + extra_bits: 0, + extra_value: 0, + } + } + + const fn repeat_previous_short(count: usize) -> Self { + Self { + symbol: 16, + extra_bits: 3, + extra_value: (count - 3) as u8, + } + } + + const fn repeat_previous_long(count: usize) -> Self { + Self { + symbol: 17, + extra_bits: 7, + extra_value: (count - 11) as u8, + } + } + + const fn zero_run_short(count: usize) -> Self { + Self { + symbol: 18, + extra_bits: 3, + extra_value: (count - 3) as u8, + } + } + + const fn zero_run_long(count: usize) -> Self { + Self { + symbol: 19, + extra_bits: 7, + extra_value: (count - 11) as u8, + } + } +} + +#[cfg(test)] +fn encode_table_level_tokens(lengths: &[u8]) -> Vec { + encode_table_level_tokens_with_allowance(lengths, &Allowance::default()) + .unwrap() + .into_vec() +} + +fn encode_table_level_tokens_with_allowance( + lengths: &[u8], + allowance: &B, +) -> Result> { + let mut tokens = Buffer::new(allowance); + let mut pos = 0usize; + let mut previous = None; + while pos < lengths.len() { + let value = lengths[pos]; + let mut run = 1usize; + while pos + run < lengths.len() && lengths[pos + run] == value { + run += 1; + } + + if value == 0 { + emit_zero_level_run(&mut tokens, run)?; + previous = Some(0); + pos += run; + continue; + } + + if previous == Some(value) && run >= 3 { + emit_repeat_level_run(&mut tokens, run)?; + pos += run; + continue; + } + + tokens + .push(LevelToken::plain(value as usize)) + .map_err(Into::into)?; + previous = Some(value); + pos += 1; + } + Ok(tokens) +} + +fn emit_repeat_level_run( + tokens: &mut Buffer, + mut run: usize, +) -> Result<()> { + while run >= 3 { + if run >= 11 { + let mut chunk = run.min(138); + if matches!(run - chunk, 1 | 2) { + chunk -= 3; + } + tokens + .push(LevelToken::repeat_previous_long(chunk)) + .map_err(Into::into)?; + run -= chunk; + } else { + let chunk = run.min(10); + tokens + .push(LevelToken::repeat_previous_short(chunk)) + .map_err(Into::into)?; + run -= chunk; + } + } + debug_assert_eq!(run, 0); + Ok(()) +} + +fn emit_zero_level_run( + tokens: &mut Buffer, + mut run: usize, +) -> Result<()> { + while run != 0 { + if run >= 11 { + let mut chunk = run.min(138); + if matches!(run - chunk, 1 | 2) { + chunk -= 3; + } + tokens + .push(LevelToken::zero_run_long(chunk)) + .map_err(Into::into)?; + run -= chunk; + } else if run >= 3 { + let chunk = run.min(10); + tokens + .push(LevelToken::zero_run_short(chunk)) + .map_err(Into::into)?; + run -= chunk; + } else { + for _ in 0..run { + tokens.push(LevelToken::plain(0)).map_err(Into::into)?; + } + break; + } + } + Ok(()) +} + +/// Prices the level alphabet by how often each symbol is used, where a flat +/// code charged the same for every symbol in play. +/// +/// A block's table is mostly runs and short lengths, so its tokens are far from +/// evenly spread and a flat code overpays for the common ones. Both codings are +/// costed here and the cheaper is written, because weighting can lose: a code +/// this deep spends eight bits rather than four to declare a length of fifteen, +/// and over twenty symbols that occasionally outweighs what the tokens save. +/// +/// Either way the code must be *complete*. Strict decoders rebuild the +/// pre-table (7-Zip's `k_BuildMode_Full`) and reject an under-full one. Huffman +/// gives Kraft equality by construction once two symbols are in play. The +/// near-uniform assignment also satisfies equality for any used-symbol count, +/// adding a phantom code when only one symbol is used. +#[cfg(test)] +fn level_code_lengths_for_tokens(tokens: &[LevelToken]) -> [u8; LEVEL_TABLE_SIZE] { + level_code_lengths_with_allowance(tokens, &Allowance::default()).unwrap() +} + +fn level_code_lengths_with_allowance( + tokens: &[LevelToken], + allowance: &B, +) -> Result<[u8; LEVEL_TABLE_SIZE]> { + let mut frequencies = [0usize; LEVEL_TABLE_SIZE]; + for token in tokens { + frequencies[token.symbol] += 1; + } + + let mut flat = [0u8; LEVEL_TABLE_SIZE]; + for (symbol, &count) in frequencies.iter().enumerate() { + flat[symbol] = u8::from(count != 0); + } + huffman::assign_flat_complete_code(&mut flat); + // One symbol in play leaves an empty branch beside it, and the flat + // assignment is the only one that pads it into a complete code. + if frequencies.iter().filter(|&&count| count != 0).count() <= 1 { + return Ok(flat); + } + + let owned = huffman::lengths_with_allowance(&frequencies, 15, allowance)?; + let mut weighted = [0; LEVEL_TABLE_SIZE]; + weighted.copy_from_slice(&owned); + Ok( + match level_code_cost(&weighted, &frequencies) < level_code_cost(&flat, &frequencies) { + true => weighted, + false => flat, + }, + ) +} + +/// What a level code costs in bits: the lengths at the head of the table as +/// [`write_level_lengths`] will write them, plus the tokens they code. +/// +/// The tokens' own extra bits are the same under either code and are left out. +fn level_code_cost( + lengths: &[u8; LEVEL_TABLE_SIZE], + frequencies: &[usize; LEVEL_TABLE_SIZE], +) -> usize { + let mut bits = 0usize; + let mut pos = 0usize; + while pos < LEVEL_TABLE_SIZE { + if lengths[pos] != 0 { + bits += if lengths[pos] == 15 { 8 } else { 4 }; + pos += 1; + continue; + } + let mut run = 1usize; + while pos + run < LEVEL_TABLE_SIZE && lengths[pos + run] == 0 { + run += 1; + } + pos += run; + while run >= 3 { + bits += 8; + run -= run.min(17); + } + bits += run * 4; + } + bits + (0..LEVEL_TABLE_SIZE) + .map(|symbol| usize::from(lengths[symbol]) * frequencies[symbol]) + .sum::() +} + +#[cfg(test)] +fn write_level_lengths(writer: &mut BitWriter, lengths: &[u8; LEVEL_TABLE_SIZE]) { + try_write_level_lengths(writer, lengths).unwrap(); +} + +fn try_write_level_lengths( + writer: &mut BitWriter, + lengths: &[u8; LEVEL_TABLE_SIZE], +) -> Result<()> { + let mut pos = 0usize; + while pos < LEVEL_TABLE_SIZE { + let length = lengths[pos]; + if length == 0 { + let mut count = 1usize; + while pos + count < LEVEL_TABLE_SIZE && lengths[pos + count] == 0 { + count += 1; + } + while count >= 3 { + let chunk = count.min(17); + writer.try_write_bits(15, 4).map_err(Into::into)?; + writer.try_write_bits(chunk - 2, 4).map_err(Into::into)?; + pos += chunk; + count -= chunk; + } + for _ in 0..count { + writer.try_write_bits(0, 4).map_err(Into::into)?; + pos += 1; + } + } else { + writer + .try_write_bits(usize::from(length), 4) + .map_err(Into::into)?; + if length == 15 { + writer.try_write_bits(0, 4).map_err(Into::into)?; + } + pos += 1; + } + } + Ok(()) +} + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "pricing_tests.rs"] +mod pricing_tests; + +#[cfg(test)] +#[path = "literal_pricing_tests.rs"] +mod literal_pricing_tests; diff --git a/arca-rar/vendor/rars/src/codec/rar50/history_retention_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/history_retention_tests.rs new file mode 100644 index 00000000..7f5e5286 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/history_retention_tests.rs @@ -0,0 +1,708 @@ +use super::*; +use crate::codec::workspace::RefusingBudget; +use std::io::{Cursor, Read}; + +#[derive(Debug, PartialEq, Eq)] +struct StateSnapshot { + tables: String, + reps: [usize; 4], + last_length: usize, + history: Vec, + history_capacity: usize, +} + +fn snapshot(state: &ReaderState) -> StateSnapshot { + StateSnapshot { + tables: format!("{:?}", state.tables), + reps: state.reps, + last_length: state.last_length, + history: state.history.to_vec(), + history_capacity: state.history.capacity(), + } +} + +struct TracedReader<'a> { + input: Cursor<&'a [u8]>, + calls: Vec<(u64, usize, usize)>, + fail_at: usize, +} +impl<'a> TracedReader<'a> { + fn new(input: &'a [u8], fail_at: usize) -> Self { + Self { + input: Cursor::new(input), + calls: Vec::new(), + fail_at, + } + } +} +impl Read for TracedReader<'_> { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + let at = self.input.position(); + if self.calls.len() == self.fail_at { + self.calls.push((at, output.len(), usize::MAX)); + return Err(std::io::Error::other("injected packed reader failure")); + } + let read = self.input.read(output)?; + self.calls.push((at, output.len(), read)); + Ok(read) + } +} + +fn source() -> Vec { + let mut data: Vec<_> = (0..4096).map(|i| ((i * 73 + i / 17) & 255) as u8).collect(); + for at in (32..data.len() - 8).step_by(64) { + data[at..at + 5].copy_from_slice(&[0xe8, 0x41, 0, 0, 0]); + data[at + 5..at + 8].copy_from_slice(&[0, 0, 0xeb]); + } + data +} +fn filter_cases() -> [Option; 5] { + [ + None, + Some(crate::FilterKind::Delta { channels: 4 }), + Some(crate::FilterKind::E8), + Some(crate::FilterKind::E8E9), + Some(crate::FilterKind::Arm), + ] +} +fn packed(data: &[u8], version: u8, filter: Option) -> Vec { + let specs: Vec<_> = filter.into_iter().map(crate::FilterSpec::whole).collect(); + Unpack50Encoder::new() + .encode_member_with_filters(data, version, &specs) + .unwrap() +} +fn result(result: Result>) -> std::result::Result, String> { + result + .map(|buffer| buffer.to_vec()) + .map_err(|error| format!("{error:?}")) +} + +#[test] +fn independent_history_unlimited_matches_original_bytes_and_live_decode_state() { + let data = source(); + for version in [0, 1] { + for filter in filter_cases() { + for dictionary in [32, 128, 4096] { + let specs: Vec<_> = filter.into_iter().map(crate::FilterSpec::whole).collect(); + let packed = Unpack50Encoder::with_options( + EncodeOptions::new(64).with_max_match_distance(dictionary), + ) + .encode_member_with_filters(&data, version, &specs) + .unwrap(); + for mode in [DecodeMode::Lz, DecodeMode::LzNoFilters] { + let mut old = ReaderState::new(&Allowance::default()); + let mut actual = ReaderState::new(&Allowance::default()); + let mut old_input = TracedReader::new(&packed, usize::MAX); + let mut actual_input = TracedReader::new(&packed, usize::MAX); + let expected = result(reference_decode( + &mut old, + &mut old_input, + version, + data.len(), + dictionary, + false, + mode, + )); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut actual_input, + version, + data.len(), + dictionary, + false, + mode, + HistoryRetention::DiscardIndependent, + ), + ); + assert_eq!(output, expected, "version={version}, filter={filter:?}, dictionary={dictionary}, mode={mode:?}"); + assert_eq!(actual_input.calls, old_input.calls); + let mut old_state = snapshot(&old); + let actual_state = snapshot(&actual); + if let Ok(bytes) = &output { + if mode == DecodeMode::Lz { + assert_eq!(bytes, &data, "version={version}, filter={filter:?}, dictionary={dictionary}, mode={mode:?}"); + } + assert!(!old_state.history.is_empty()); + old_state.history.clear(); + old_state.history_capacity = 0; + assert!(actual.history.is_empty()); + assert_eq!(actual.history.capacity(), 0); + } + assert_eq!(actual_state, old_state); + } + } + } + } +} + +#[test] +fn independent_history_default_and_solid_routes_retain_original_raw_dictionary() { + let data = source(); + for version in [0, 1] { + for filter in filter_cases() { + let packed = packed(&data, version, filter); + let mut old = ReaderState::new(&Allowance::default()); + let mut actual = ReaderState::new(&Allowance::default()); + let expected = result(reference_decode( + &mut old, + &mut Cursor::new(&packed), + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + )); + let output = result(actual.decode_member_with_dictionary( + &packed, + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + )); + assert_eq!(output, expected); + assert_eq!(snapshot(&actual), snapshot(&old)); + let specs: Vec<_> = filter.into_iter().map(crate::FilterSpec::whole).collect(); + let (raw, _) = filtered_lz_member(&data, &specs).unwrap(); + assert_eq!(&actual.history[..], raw); + // The internal discard request must be overridden for solid decoding. + let next = + encode_lz_member_with_history(&raw[raw.len() - 128..], &raw, version).unwrap(); + let expected = result(reference_decode( + &mut old, + &mut Cursor::new(&next), + version, + 128, + 4096, + true, + DecodeMode::Lz, + )); + let output = result(actual.decode_member_with_dictionary_and_history( + &next, + version, + 128, + 4096, + true, + DecodeMode::Lz, + HistoryRetention::DiscardIndependent, + )); + assert_eq!(output, expected); + assert_eq!(snapshot(&actual), snapshot(&old)); + } + } +} + +#[test] +fn independent_history_limited_keeps_every_original_growth_refusal_and_charge() { + let data = source()[..256].to_vec(); + for version in [0, 1] { + for filter in [None, Some(crate::FilterKind::Delta { channels: 4 })] { + let packed = packed(&data, version, filter); + let baseline = RefusingBudget::new(usize::MAX); + { + let mut state = ReaderState::new(&baseline); + reference_decode( + &mut state, + &mut Cursor::new(&packed), + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + ) + .unwrap(); + } + assert_eq!(baseline.used(), 0); + for fail_at in 0..=baseline.attempts() { + for cancel_at in [0, 3, 7, usize::MAX] { + let old_budget = RefusingBudget::new(fail_at); + let actual_budget = RefusingBudget::new(fail_at); + let mut old = ReaderState::new(&old_budget); + let mut actual = ReaderState::new(&actual_budget); + let old_token = crate::ReadCancellation::new(); + let actual_token = crate::ReadCancellation::new(); + old.read_control = crate::read_control::ReadControl::new(Some(&old_token)); + actual.read_control = + crate::read_control::ReadControl::new(Some(&actual_token)); + old.read_control.cancel_after_checks(cancel_at); + actual.read_control.cancel_after_checks(cancel_at); + let mut old_input = TracedReader::new(&packed, usize::MAX); + let mut actual_input = TracedReader::new(&packed, usize::MAX); + let expected = result(reference_decode( + &mut old, + &mut old_input, + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + )); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut actual_input, + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + HistoryRetention::DiscardIndependent, + ), + ); + assert_eq!( + output, expected, + "version={version}, filter={filter:?}, fail_at={fail_at}" + ); + assert_eq!(actual_input.calls, old_input.calls); + assert_eq!(snapshot(&actual), snapshot(&old)); + assert_eq!(actual_budget.attempts(), old_budget.attempts()); + assert_eq!(actual_budget.used(), old_budget.used()); + drop(actual); + drop(old); + assert_eq!(actual_budget.used(), 0); + assert_eq!(old_budget.used(), 0); + assert_eq!(actual_token.is_cancelled(), old_token.is_cancelled()); + } + } + } + } +} + +#[test] +fn independent_history_preserves_reader_corruption_and_cancellation_order() { + let data = source()[..256].to_vec(); + for version in [0, 1] { + for filter in [None, Some(crate::FilterKind::Delta { channels: 4 })] { + let packed = packed(&data, version, filter); + for cut in [0, 1, 3, packed.len() / 2, packed.len() - 1, packed.len()] { + for fail_at in [0, 1, 2, usize::MAX] { + for cancel_at in [0, 1, 3, 7, 15, usize::MAX] { + let old_token = crate::ReadCancellation::new(); + let actual_token = crate::ReadCancellation::new(); + let mut old = ReaderState::new(&Allowance::default()); + let mut actual = ReaderState::new(&Allowance::default()); + old.read_control = crate::read_control::ReadControl::new(Some(&old_token)); + actual.read_control = + crate::read_control::ReadControl::new(Some(&actual_token)); + old.read_control.cancel_after_checks(cancel_at); + actual.read_control.cancel_after_checks(cancel_at); + let mut old_input = TracedReader::new(&packed[..cut], fail_at); + let mut actual_input = TracedReader::new(&packed[..cut], fail_at); + let expected = result(reference_decode( + &mut old, + &mut old_input, + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + )); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut actual_input, + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + HistoryRetention::DiscardIndependent, + ), + ); + assert_eq!(output, expected); + assert_eq!(actual_input.calls, old_input.calls); + assert_eq!(actual_token.is_cancelled(), old_token.is_cancelled()); + let mut expected_state = snapshot(&old); + if output.is_ok() { + expected_state.history.clear(); + expected_state.history_capacity = 0; + } + assert_eq!(snapshot(&actual), expected_state); + } + } + } + } + } +} + +#[test] +fn independent_history_numeric_limits_keep_original_refusal_values_and_state() { + let data = source()[..256].to_vec(); + for filter in [None, Some(crate::FilterKind::Delta { channels: 4 })] { + let packed = packed(&data, 0, filter); + for limit in [0, 32, 128, 512, 2048, 8192, 65536] { + let old_budget = Allowance::limited(limit); + let actual_budget = Allowance::limited(limit); + let mut old = ReaderState::new(&old_budget); + let mut actual = ReaderState::new(&actual_budget); + let expected = result(reference_decode( + &mut old, + &mut Cursor::new(&packed), + 0, + data.len(), + 4096, + false, + DecodeMode::Lz, + )); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut Cursor::new(&packed), + 0, + data.len(), + 4096, + false, + DecodeMode::Lz, + HistoryRetention::DiscardIndependent, + ), + ); + assert_eq!(output, expected, "limit={limit}, filter={filter:?}"); + assert_eq!(snapshot(&actual), snapshot(&old)); + assert_eq!(actual_budget.used(), old_budget.used()); + drop(actual); + drop(old); + assert_eq!(actual_budget.used(), 0); + assert_eq!(old_budget.used(), 0); + } + } +} + +#[derive(Clone, Debug, Default)] +struct AllocationTrace(std::sync::Arc>>); +impl Budget for AllocationTrace { + type Charge = Self; + type Failure = std::convert::Infallible; + const LIMITED: bool = false; + fn grow( + values: &mut Vec, + _: &mut Self, + additional: usize, + ) -> std::result::Result<(), Self::Failure> { + values.reserve(additional); + Ok(()) + } + fn charge(&self) -> Self { + self.clone() + } + fn allowance(charge: &Self) -> Self { + charge.clone() + } + fn resize(charge: &mut Self, bytes: u64) -> Result<()> { + charge.0.lock().unwrap().push(bytes); + Ok(()) + } +} + +#[test] +fn independent_history_omits_only_final_explicit_buffer_admissions() { + let data = source(); + for version in [0, 1] { + for filter in filter_cases() { + let packed = packed(&data, version, filter); + let retained = AllocationTrace::default(); + let discarded = AllocationTrace::default(); + let mut old = ReaderState::new(&retained); + let mut actual = ReaderState::new(&discarded); + let expected = result(reference_decode( + &mut old, + &mut Cursor::new(&packed), + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + )); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut Cursor::new(&packed), + version, + data.len(), + 4096, + false, + DecodeMode::Lz, + HistoryRetention::DiscardIndependent, + ), + ); + assert_eq!(output, expected); + assert_eq!(output.unwrap(), data); + assert_eq!(old.history.len(), 4096); + assert_eq!(old.history.capacity(), 4096); + assert!(actual.history.is_empty()); + assert_eq!(actual.history.capacity(), 0); + let old = retained.0.lock().unwrap(); + let actual = discarded.0.lock().unwrap(); + let omitted = if filter.is_some() { 2 } else { 1 }; + assert_eq!(old.len() - actual.len(), omitted); + assert_eq!( + old.iter().sum::() - actual.iter().sum::(), + omitted as u64 * 4096 + ); + // This observes only explicit Buffer capacity admissions, not every + // allocator call or whole-chain peak memory. + } + } +} + +#[test] +fn independent_history_preserves_undersized_window_zero_fill_compatibility() { + let data = source(); + for version in [0, 1] { + let packed = packed(&data, version, None); + let mut old = ReaderState::new(&Allowance::default()); + let mut actual = ReaderState::new(&Allowance::default()); + let expected = result(reference_decode( + &mut old, + &mut Cursor::new(&packed), + version, + data.len(), + 32, + false, + DecodeMode::Lz, + )) + .unwrap(); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut Cursor::new(&packed), + version, + data.len(), + 32, + false, + DecodeMode::Lz, + HistoryRetention::DiscardIndependent, + ), + ) + .unwrap(); + assert_eq!(output, expected); + // The retained original decoder fills an out-of-window reference with + // zeroes. This deliberately undersized codec call is not a source + // roundtrip; archive integrity checks still reject damaged payloads. + assert_ne!(output, data); + assert_eq!(output[96], 0); + assert_eq!(data[96], 0xe8); + let mut old_state = snapshot(&old); + old_state.history.clear(); + old_state.history_capacity = 0; + assert_eq!(snapshot(&actual), old_state); + } +} + +#[test] +fn independent_history_filtered_unknown_version_keeps_validation_and_late_cancel_order() { + let data = source()[..256].to_vec(); + let packed = packed(&data, 0, Some(crate::FilterKind::Delta { channels: 4 })); + for version in [2, 255] { + for dictionary in [0, 4096] { + for mode in [ + DecodeMode::LiteralOnly, + DecodeMode::Lz, + DecodeMode::LzNoFilters, + ] { + for cancel_at in [0, 1, 3, 7, usize::MAX] { + for fail_at in [0, 1, 3, usize::MAX] { + let old_budget = RefusingBudget::new(fail_at); + let actual_budget = RefusingBudget::new(fail_at); + let old_token = crate::ReadCancellation::new(); + let actual_token = crate::ReadCancellation::new(); + let mut old = ReaderState::new(&old_budget); + let mut actual = ReaderState::new(&actual_budget); + old.read_control = crate::read_control::ReadControl::new(Some(&old_token)); + actual.read_control = + crate::read_control::ReadControl::new(Some(&actual_token)); + old.read_control.cancel_after_checks(cancel_at); + actual.read_control.cancel_after_checks(cancel_at); + let mut old_input = TracedReader::new(&packed, usize::MAX); + let mut actual_input = TracedReader::new(&packed, usize::MAX); + let expected = result(reference_decode( + &mut old, + &mut old_input, + version, + data.len(), + dictionary, + false, + mode, + )); + let output = result( + actual.decode_member_from_reader_with_dictionary_and_history( + &mut actual_input, + version, + data.len(), + dictionary, + false, + mode, + HistoryRetention::DiscardIndependent, + ), + ); + assert!(output.is_err()); + assert_eq!(output, expected); + assert_eq!(actual_input.calls, old_input.calls); + assert_eq!(snapshot(&actual), snapshot(&old)); + assert_eq!(actual_token.is_cancelled(), old_token.is_cancelled()); + assert_eq!(actual_budget.attempts(), old_budget.attempts()); + drop(actual); + drop(old); + assert_eq!(actual_budget.used(), 0); + assert_eq!(old_budget.used(), 0); + } + } + } + } + } +} + +// Full unchanged d2 buffered decoding body, independent of history-policy plumbing. +#[allow(clippy::too_many_arguments)] +fn reference_decode( + state: &mut ReaderState, + input: &mut impl Read, + algorithm_version: u8, + output_size: usize, + dictionary_size: usize, + solid: bool, + mode: DecodeMode, +) -> Result> { + state.read_control.check_codec()?; + let control = state.read_control.clone(); + let input = &mut control.reader(input); + if dictionary_size == 0 { + return Err(Error::InvalidData("RAR 5 dictionary size is zero")); + } + if !solid { + state.reset(); + } + + let allowance = state.history.allowance(); + let mut output = + Buffer::with_capacity(output_size.min(MAX_INITIAL_OUTPUT_CAPACITY), &allowance)?; + let mut filters = Buffer::new(&allowance); + + loop { + let block = read_compressed_block_with_allowance(input, &state.history.allowance())?; + let payload = &*block.payload; + let mut payload_bit_pos = 0; + if block.header.has_tables { + let (lengths, table_bits) = read_table_lengths_with_allowance( + payload, + algorithm_version, + &block.payload.allowance(), + )?; + state.tables = Some(ReaderTables::from_lengths( + &lengths, + &block.payload.allowance(), + )?); + payload_bit_pos = table_bits; + } + let tables = state + .tables + .take() + .ok_or(Error::InvalidData("RAR 5 block reuses missing tables"))?; + let mut bits = BitReader::new(payload); + bits.bit_pos = payload_bit_pos; + + let mut poller = state.read_control.poller(); + while bits.bit_pos < block.header.payload_bits && output.len() < output_size { + poller.check_codec(output.len())?; + let symbol = tables.main.decode(&mut bits)?; + match symbol { + 0..=255 => output.try_push(symbol as u8)?, + 256 if mode.uses_lz() => { + filters.try_push(read_filter(&mut bits, output.len())?)?; + } + 257 if mode.uses_lz() => { + if state.last_length != 0 { + state.copy_match( + &mut output, + state.reps[0], + state.last_length, + output_size, + dictionary_size, + )?; + } + } + 258..=261 if mode.uses_lz() => { + let rep_index = symbol - 258; + let distance = state.reps[rep_index]; + if distance == 0 { + return Err(Error::InvalidData( + "RAR 5 repeat distance is not initialized", + )); + } + let length_slot = tables.length.decode(&mut bits)?; + let length_extra = bits.read_bits(length_slot_extra_bits(length_slot))?; + // The length table has 44 symbols and read_bits limits + // the extra value to this slot's declared width. + let length = length_from_slot_parts(length_slot, length_extra); + state.reps[..=rep_index].rotate_right(1); + state.reps[0] = distance; + state.last_length = length; + state.copy_match( + &mut output, + distance, + length, + output_size, + dictionary_size, + )?; + } + 262.. if mode.uses_lz() => { + let length_slot = symbol - 262; + let length_extra = bits.read_bits(length_slot_extra_bits(length_slot))?; + // Main symbols end at slot 43 and read_bits limits the + // extra value to this slot's declared width. + let mut length = length_from_slot_parts(length_slot, length_extra); + let distance_slot = tables.distance.decode(&mut bits)?; + let distance_bit_count = distance_slot_bit_count(distance_slot)?; + let distance_extra = if distance_bit_count >= 4 && tables.align_mode { + let high = bits.read_bits((distance_bit_count - 4) as u8)?; + let low = tables.align.decode(&mut bits)? as u32; + (high << 4) | low + } else { + bits.read_bits(distance_bit_count as u8)? + }; + // distance_slot_bit_count rejected slots above 65 and + // the extra value was read at exactly that width. + let distance = + distance_from_slot_parts(distance_slot, distance_bit_count, distance_extra); + length += length_bonus(distance); + state.reps.rotate_right(1); + state.reps[0] = distance; + state.last_length = length; + state.copy_match( + &mut output, + distance, + length, + output_size, + dictionary_size, + )?; + } + _ => { + return Err(Error::InvalidData( + "RAR 5 literal-only decoder encountered non-literal symbol", + )); + } + } + } + + state.tables = Some(tables); + if block.header.is_last || output.len() >= output_size { + break; + } + } + + if output.len() == output_size { + let history_output = if mode.applies_filters() && !filters.is_empty() { + Some(Buffer::copied( + &output[output.len().saturating_sub(dictionary_size)..], + &allowance, + )?) + } else { + None + }; + if mode.applies_filters() { + state.read_control.check_codec()?; + apply_filters_with_allowance(&mut output, &filters, &state.read_control, &allowance)?; + } + state.remember_history( + history_output.as_deref().unwrap_or(&output), + dictionary_size, + )?; + Ok(output) + } else { + Err(Error::NeedMoreInput) + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/huffman_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/huffman_tests.rs new file mode 100644 index 00000000..e2f38bd0 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/huffman_tests.rs @@ -0,0 +1,23 @@ +use super::*; + +#[test] +fn public_huffman_constructor_rejects_active_alphabet_count_overflow() { + for length in [1, 15] { + for active in [65_535, 65_536, 65_537] { + assert!(matches!( + HuffmanTable::from_lengths(&vec![length; active]), + Err(Error::InvalidData("RAR 5 oversubscribed Huffman table")) + )); + } + } +} + +#[test] +fn public_huffman_constructor_keeps_wide_sparse_symbol_indices() { + let mut lengths = vec![0; 65_538]; + lengths[65_537] = 1; + let table = HuffmanTable::from_lengths(&lengths).unwrap(); + let mut bits = BitReader::new(&[0]); + assert_eq!(table.state.decode(&mut bits), Ok(65_537)); + assert_eq!(bits.bit_pos, 1); +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/literal_pricing_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/literal_pricing_tests.rs new file mode 100644 index 00000000..a7d21a00 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/literal_pricing_tests.rs @@ -0,0 +1,907 @@ +use super::*; +use crate::codec::workspace::RefusingBudget; + +// Retained upstream initialization, reach admission and backtracking. Pricing +// uses the original scalar implementation below, never the literal shortcut. +#[allow(clippy::too_many_arguments)] +fn reference_optimal_tokens( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + matches: &BlockMatches, + workspace: &mut OptimalWorkspace, +) -> Result> { + let start = block.start; + let end = block.end; + let span = end - start; + + let allowance = workspace.price.allowance().clone(); + let OptimalWorkspace { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + } = workspace; + price.resize(span + 1, u32::MAX)?; + price.fill(u32::MAX); + arrive_length.resize(span + 1, 0)?; + arrive_length.fill(0); + arrive_distance.resize(span + 1, 0)?; + arrive_distance.fill(0); + arrive_reps.resize(span + 1, [0; 4])?; + arrive_reps.fill([0; 4]); + arrive_last_length.resize(span + 1, 0)?; + arrive_last_length.fill(0); + price[0] = 0; + + // Admit the largest candidate list once. Pricing only borrows already + // charged arrays, so its inner loop is identical for both budget policies. + // Each position contributes at most four remembered-distance candidates. + let longest_run = matches + .starts + .windows(2) + .map(|pair| (pair[1] - pair[0]) as usize) + .max() + .unwrap_or(0); + // A position contributes at most one run for each encodable match length. + // Four remembered distances are added while pricing, so this bound cannot + // approach usize::MAX for a valid BlockMatches collection. + debug_assert!(longest_run <= MAX_ENCODER_MATCH_LENGTH); + let reach_capacity = longest_run + 4; + let mut reaches = Buffer::filled(reach_capacity, (0usize, 0usize, 0usize), &allowance)?; + reference_price_optimal_paths( + combined, + block.clone(), + options, + distance_size, + prices, + &matches.runs, + &matches.starts, + OptimalSlices { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + }, + &mut reaches, + ); + + let mut reversed = Buffer::new(&allowance); + let mut index = span; + while index > 0 { + let length = arrive_length[index] as usize; + if length == 0 { + reversed + .push(EncodeToken::Literal(combined[start + index - 1])) + .map_err(Into::into)?; + index -= 1; + } else { + reversed + .push(EncodeToken::Match { + length, + distance: arrive_distance[index] as usize, + }) + .map_err(Into::into)?; + index -= length; + } + } + reversed.reverse(); + Ok(reversed) +} + +#[allow(clippy::too_many_arguments)] +fn reference_encode_tokens( + combined: &[u8], + block: std::ops::Range, + search: MemberSearch<'_, B>, + options: EncodeOptions, + distance_size: usize, + initial_filters: &[EncodeFilter], + progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let start = block.start; + let end = block.end; + if options.optimal_parse { + let mut own; + let collector = match search { + MemberSearch::Optimal(collector) => collector, + _ => { + own = OptimalCollector::with_allowance(combined, start, options, allowance)?; + &mut own + } + }; + let matches = collector.collect(combined, block.clone(), options)?; + // The prices come from the Huffman tables, and the tables come from + // the parse, so the first pass has to guess. Each pass after it prices + // against what the pass before actually produced. + let mut workspace = OptimalWorkspace::new(allowance); + let mut tokens = reference_optimal_tokens( + combined, + block.clone(), + options, + distance_size, + None, + &matches, + &mut workspace, + )?; + let mut lengths = + table_lengths_with_allowance(&tokens, initial_filters, distance_size, allowance)?; + let mut best_bits = token_stream_bits(&tokens, initial_filters, &lengths, distance_size)?; + let mut best = None; + for _ in 1..OPTIMAL_PARSE_PASSES { + let prices = TokenPrices { + lengths: lengths.slices(), + }; + let next = reference_optimal_tokens( + combined, + block.clone(), + options, + distance_size, + Some(&prices), + &matches, + &mut workspace, + )?; + if next == tokens { + break; + } + lengths = + table_lengths_with_allowance(&next, initial_filters, distance_size, allowance)?; + let bits = token_stream_bits(&next, initial_filters, &lengths, distance_size)?; + if bits < best_bits { + best_bits = bits; + best = None; + } else if best.is_none() { + // Keep an earlier winner only when repricing actually loses. + best = Some(std::mem::replace(&mut tokens, Buffer::new(allowance))); + } + tokens = next; + } + let tokens = best.unwrap_or(tokens); + if progress.is_some_and(|report| !report(end - start)) { + return Err(Error::Cancelled); + } + return Ok(tokens); + } + + unreachable!("the reference covers only optimal parsing"); +} + +#[allow(clippy::too_many_arguments)] +fn reference_encode_block( + combined: &[u8], + block: std::ops::Range, + search: MemberSearch<'_, B>, + algorithm_version: u8, + initial_filters: &[EncodeFilter], + options: EncodeOptions, + is_last: bool, + progress: Option<&mut dyn FnMut(usize) -> bool>, + allowance: &B, +) -> Result> { + let distance_size = match algorithm_version { + 0 => DISTANCE_TABLE_SIZE_50, + 1 => DISTANCE_TABLE_SIZE_70, + _ => { + return Err(Error::InvalidData( + "RAR 5 unknown compression algorithm version", + )) + } + }; + let mut tokens = reference_encode_tokens( + combined, + block, + search, + options, + distance_size, + initial_filters, + progress, + allowance, + )?; + if !initial_filters.is_empty() { + tokens.prepend(initial_filters.iter().copied().map(EncodeToken::Filter))?; + } + encode_token_block_with_allowance( + &tokens, + algorithm_version, + distance_size, + is_last, + allowance, + ) +} + +fn noise(size: usize) -> Vec { + let mut state = 0x8bad_f00du32; + (0..size) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect() +} + +fn empty_matches(span: usize, allowance: &B) -> Result> { + Ok(BlockMatches { + runs: Buffer::new(allowance), + starts: Buffer::filled(span + 1, 0, allowance)?, + }) +} + +#[derive(Debug, PartialEq, Eq)] +struct WorkspaceState { + price: Vec, + length: Vec, + distance: Vec, + reps: Vec<[u32; 4]>, + last_length: Vec, +} + +fn workspace_state(workspace: &OptimalWorkspace) -> WorkspaceState { + WorkspaceState { + price: workspace.price.to_vec(), + length: workspace.arrive_length.to_vec(), + distance: workspace.arrive_distance.to_vec(), + reps: workspace.arrive_reps.to_vec(), + last_length: workspace.arrive_last_length.to_vec(), + } +} + +fn poison(workspace: &mut OptimalWorkspace, span: usize) { + workspace.price.resize(span + 1, 0x1234_5678).unwrap(); + workspace.price.fill(0x1234_5678); + workspace.arrive_length.resize(span + 1, 4096).unwrap(); + workspace.arrive_length.fill(4096); + workspace.arrive_distance.resize(span + 1, 257).unwrap(); + workspace.arrive_distance.fill(257); + workspace + .arrive_reps + .resize(span + 1, [3, 3, 7, 3]) + .unwrap(); + workspace.arrive_reps.fill([3, 3, 7, 3]); + workspace.arrive_last_length.resize(span + 1, 511).unwrap(); + workspace.arrive_last_length.fill(511); +} + +#[test] +fn empty_collections_match_the_retained_parser_for_every_price_mode() { + let allowance = Allowance::default(); + for data in [ + Vec::new(), + vec![1], + (0..=255).collect(), + noise(4096), + b"abcd".repeat(1024), + ] { + for start in [0, data.len().min(128)] { + let block = start..data.len(); + for candidates in [0, 128] { + // The candidate-zero case also proves that identical bytes and + // nonzero history cannot invent a remembered distance. + let options = EncodeOptions::new(candidates) + .with_optimal_parse(true) + .with_max_match_distance(if candidates == 0 { usize::MAX } else { 0 }); + let mut collector = OptimalCollector::new(&data, start, options); + let matches = collector.collect(&data, block.clone(), options).unwrap(); + assert!(matches.runs.is_empty()); + for distance_size in [DISTANCE_TABLE_SIZE_50, DISTANCE_TABLE_SIZE_70] { + let mut workspace = OptimalWorkspace::new(&allowance); + let mut old = OptimalWorkspace::new(&allowance); + let literals = reference_optimal_tokens( + &data, + block.clone(), + options, + distance_size, + None, + &matches, + &mut old, + ) + .unwrap(); + let lengths = table_lengths_for_tokens(&literals, distance_size).unwrap(); + let actual = TokenPrices { + lengths: lengths.slices(), + }; + for prices in [None, Some(&actual)] { + poison(&mut workspace, block.len()); + let before = workspace_state(&workspace); + let expected = reference_optimal_tokens( + &data, + block.clone(), + options, + distance_size, + prices, + &matches, + &mut old, + ) + .unwrap(); + let result = optimal_tokens_in_workspace( + &data, + block.clone(), + options, + distance_size, + prices, + &matches, + &mut workspace, + ) + .unwrap(); + assert_eq!(result, expected); + assert_eq!( + workspace_state(&workspace), + before, + "literal shortcut must not initialize or read stale arrays" + ); + assert!(result + .iter() + .all(|token| matches!(token, EncodeToken::Literal(_)))); + } + } + } + } + } +} + +#[test] +fn real_no_match_collections_use_the_shortcut_and_later_matches_reset_poison() { + let allowance = Allowance::default(); + let options = EncodeOptions::new(128).with_optimal_parse(true); + let repeated = b"abcd".repeat(1024); + let literal = noise(repeated.len()); + let mut repeat_collector = OptimalCollector::new(&repeated, 0, options); + let repeat_matches = repeat_collector + .collect(&repeated, 0..repeated.len(), options) + .unwrap(); + assert!(!repeat_matches.runs.is_empty()); + let mut literal_collector = OptimalCollector::new(&literal, 0, options); + let literal_matches = literal_collector + .collect(&literal, 0..literal.len(), options) + .unwrap(); + assert!( + literal_matches.runs.is_empty(), + "fixture must really contain no collected four-byte match" + ); + for distance_size in [DISTANCE_TABLE_SIZE_50, DISTANCE_TABLE_SIZE_70] { + let mut workspace = OptimalWorkspace::new(&allowance); + let mut old = OptimalWorkspace::new(&allowance); + for (data, matches) in [ + (&repeated, &repeat_matches), + (&literal, &literal_matches), + (&repeated, &repeat_matches), + ] { + poison(&mut workspace, repeated.len()); + poison(&mut old, repeated.len()); + let before = workspace_state(&workspace); + let expected = reference_optimal_tokens( + data, + 0..data.len(), + options, + distance_size, + None, + matches, + &mut old, + ) + .unwrap(); + let result = optimal_tokens_in_workspace( + data, + 0..data.len(), + options, + distance_size, + None, + matches, + &mut workspace, + ) + .unwrap(); + assert_eq!(result, expected); + if matches.runs.is_empty() { + assert_eq!(workspace_state(&workspace), before); + } else { + assert_eq!(workspace_state(&workspace), workspace_state(&old)); + } + } + } +} + +#[test] +fn literal_pipeline_keeps_filters_winner_selection_and_packed_bytes() { + let allowance = Allowance::default(); + let input = noise(4096); + let specs = [ + crate::FilterSpec::whole(crate::FilterKind::Delta { channels: 4 }), + crate::FilterSpec::whole(crate::FilterKind::Arm), + ]; + for candidates in [0, 128] { + let options = EncodeOptions::new(candidates).with_optimal_parse(true); + for version in [0, 1] { + for specs in [&[][..], &specs[..1], &specs[1..]] { + let (transformed, records) = + filtered_member_with_allowance(&input, specs, &allowance).unwrap(); + let mut collector = OptimalCollector::new(&transformed, 0, options); + let matches = collector + .collect(&transformed, 0..transformed.len(), options) + .unwrap(); + assert!(matches.runs.is_empty()); + let expected = reference_encode_block( + &transformed, + 0..transformed.len(), + MemberSearch::Fresh, + version, + &records, + options, + true, + None, + &allowance, + ) + .unwrap(); + let actual = Unpack50Encoder::with_options(options) + .encode_member_with_filters(&input, version, specs) + .unwrap(); + assert_eq!(&*expected, actual); + let decoded = Unpack50Decoder::new() + .decode_member(&actual, version, input.len(), false, DecodeMode::Lz) + .unwrap(); + assert_eq!(decoded, input); + } + } + } +} + +#[test] +fn literal_pipeline_retains_validation_before_final_progress_and_cancellation() { + let allowance = Allowance::default(); + let data: Vec<_> = (0..32).collect(); + let options = EncodeOptions::new(128).with_optimal_parse(true); + let valid = EncodeFilter { + offset: 0, + length: data.len(), + filter_type: FilterType::Delta, + channels: 1, + }; + let invalid = EncodeFilter { + channels: 0, + ..valid + }; + for version in [0, 1, 2] { + for records in [&[][..], &[valid][..], &[invalid][..]] { + for continue_work in [true, false] { + let mut expected_reports = Vec::new(); + let mut expected_report = |amount| { + expected_reports.push(amount); + continue_work + }; + let expected = reference_encode_block( + &data, + 0..data.len(), + MemberSearch::Fresh, + version, + records, + options, + true, + Some(&mut expected_report), + &allowance, + ) + .map(|buffer| buffer.to_vec()); + let mut actual_reports = Vec::new(); + let mut actual_report = |amount| { + actual_reports.push(amount); + continue_work + }; + let actual = encode_lz_block_with_allowance( + &data, + 0..data.len(), + MemberSearch::Fresh, + version, + records, + options, + true, + Some(&mut actual_report), + &allowance, + ) + .map(|buffer| buffer.to_vec()); + assert_eq!(actual, expected); + assert_eq!(actual_reports, expected_reports); + if version > 1 || records == [invalid] { + assert!(actual_reports.is_empty()); + assert!(matches!(actual, Err(Error::InvalidData(_)))); + } else { + assert_eq!(actual_reports, [data.len()]); + if !continue_work { + assert!(matches!(actual, Err(Error::Cancelled))); + } + } + } + } + } +} + +fn bounded_parse( + allowance: &B, + old: bool, +) -> (Result>, WorkspaceState) { + let data: Vec<_> = (0..32).collect(); + let options = EncodeOptions::new(128).with_optimal_parse(true); + let mut workspace = OptimalWorkspace::new(allowance); + let result = (|| { + let matches = empty_matches(data.len(), allowance)?; + if old { + reference_optimal_tokens( + &data, + 0..data.len(), + options, + DISTANCE_TABLE_SIZE_50, + None, + &matches, + &mut workspace, + ) + } else { + optimal_tokens_in_workspace( + &data, + 0..data.len(), + options, + DISTANCE_TABLE_SIZE_50, + None, + &matches, + &mut workspace, + ) + } + })() + .map(|buffer| buffer.to_vec()); + (result, workspace_state(&workspace)) +} + +#[test] +fn bounded_literal_parses_preserve_all_quota_failures_and_growth_attempts() { + let mut saw_limit = false; + let mut saw_success = false; + for limit in [0, 32, 128, 132, 256, 512, 1024, 2048, 4096, 65536] { + let expected_budget = Allowance::limited(limit); + let actual_budget = Allowance::limited(limit); + let expected = bounded_parse(&expected_budget, true); + let actual = bounded_parse(&actual_budget, false); + saw_limit |= matches!(actual.0, Err(Error::WorkspaceLimitExceeded(_))); + saw_success |= actual.0.is_ok(); + assert_eq!(actual, expected, "limit={limit}"); + assert_eq!(actual_budget.used(), expected_budget.used()); + assert_eq!( + actual_budget.used(), + 0, + "all owned charges must be released" + ); + } + assert!(saw_limit && saw_success); + let mut saw_refusal = false; + let mut saw_success = false; + for fail_at in 0..24 { + let expected_budget = RefusingBudget::new(fail_at); + let actual_budget = RefusingBudget::new(fail_at); + let expected = bounded_parse(&expected_budget, true); + let actual = bounded_parse(&actual_budget, false); + saw_refusal |= matches!(actual.0, Err(Error::Cancelled)); + saw_success |= actual.0.is_ok(); + assert_eq!(actual, expected, "growth refusal={fail_at}"); + assert_eq!(actual_budget.attempts(), expected_budget.attempts()); + assert_eq!(actual_budget.used(), expected_budget.used()); + assert_eq!(actual_budget.used(), 0); + } + assert!(saw_refusal && saw_success); +} + +#[test] +fn bounded_outer_pipeline_keeps_errors_before_progress() { + let data: Vec<_> = (0..32).collect(); + let options = EncodeOptions::new(128).with_optimal_parse(true); + for limit in [0, 128, 4096, 65536, 8 * 1024 * 1024] { + let mut expected_reports = Vec::new(); + let mut expected_report = |amount| { + expected_reports.push(amount); + false + }; + let expected = reference_encode_block( + &data, + 0..data.len(), + MemberSearch::Fresh, + 0, + &[], + options, + true, + Some(&mut expected_report), + &Allowance::limited(limit), + ) + .map(|buffer| buffer.to_vec()); + let mut actual_reports = Vec::new(); + let mut actual_report = |amount| { + actual_reports.push(amount); + false + }; + let actual = encode_lz_block_with_allowance( + &data, + 0..data.len(), + MemberSearch::Fresh, + 0, + &[], + options, + true, + Some(&mut actual_report), + &Allowance::limited(limit), + ) + .map(|buffer| buffer.to_vec()); + assert_eq!(actual, expected, "limit={limit}"); + assert_eq!(actual_reports, expected_reports); + } +} + +#[test] +fn literal_outer_pipeline_matches_old_parses_for_history_tails_and_real_matches() { + let allowance = Allowance::default(); + let mut rare_repeat = noise(4096); + rare_repeat.copy_within(128..144, 192); + let cases = [ + Vec::new(), + vec![7], + (0..32).collect(), + (0..=255).collect(), + noise(4096), + b"abcd".repeat(8), + rare_repeat, + ]; + let mut saw_empty = false; + let mut saw_matches = false; + for data in cases { + for start in [0, data.len().min(3), data.len().min(128)] { + for candidates in [0, 128] { + let options = EncodeOptions::new(candidates).with_optimal_parse(true); + let mut collector = OptimalCollector::new(&data, start, options); + let matches = collector + .collect(&data, start..data.len(), options) + .unwrap(); + saw_empty |= matches.runs.is_empty(); + saw_matches |= !matches.runs.is_empty(); + for distance_size in [DISTANCE_TABLE_SIZE_50, DISTANCE_TABLE_SIZE_70] { + for continue_work in [true, false] { + let mut expected_reports = Vec::new(); + let mut expected_report = |amount| { + expected_reports.push(amount); + continue_work + }; + let expected = reference_encode_tokens( + &data, + start..data.len(), + MemberSearch::Fresh, + options, + distance_size, + &[], + Some(&mut expected_report), + &allowance, + ); + let mut actual_reports = Vec::new(); + let mut actual_report = |amount| { + actual_reports.push(amount); + continue_work + }; + let actual = encode_tokens_with_allowance( + &data, + start..data.len(), + MemberSearch::Fresh, + options, + distance_size, + &[], + Some(&mut actual_report), + &allowance, + ); + assert_eq!(actual, expected, "start={start}, candidates={candidates}"); + assert_eq!(actual_reports, expected_reports); + assert_eq!(actual_reports, [data.len() - start]); + } + } + } + } + } + assert!(saw_empty && saw_matches); +} + +#[test] +fn bounded_outer_reparses_keep_every_growth_refusal_and_callback() { + let data: Vec<_> = (0..32).collect(); + let options = EncodeOptions::new(128).with_optimal_parse(true); + let mut saw_refusal = false; + let mut saw_success = false; + for fail_at in 0..128 { + for continue_work in [true, false] { + let expected_budget = RefusingBudget::new(fail_at); + let actual_budget = RefusingBudget::new(fail_at); + let mut expected_reports = Vec::new(); + let mut expected_report = |amount| { + expected_reports.push(amount); + continue_work + }; + let expected = reference_encode_block( + &data, + 0..data.len(), + MemberSearch::Fresh, + 0, + &[], + options, + true, + Some(&mut expected_report), + &expected_budget, + ) + .map(|buffer| buffer.to_vec()); + let mut actual_reports = Vec::new(); + let mut actual_report = |amount| { + actual_reports.push(amount); + continue_work + }; + let actual = encode_lz_block_with_allowance( + &data, + 0..data.len(), + MemberSearch::Fresh, + 0, + &[], + options, + true, + Some(&mut actual_report), + &actual_budget, + ) + .map(|buffer| buffer.to_vec()); + saw_refusal |= expected_reports.is_empty() && matches!(actual, Err(Error::Cancelled)); + saw_success |= actual.is_ok(); + assert_eq!( + actual, expected, + "fail_at={fail_at}, continue={continue_work}" + ); + assert_eq!(actual_reports, expected_reports); + assert_eq!(actual_budget.attempts(), expected_budget.attempts()); + assert_eq!(actual_budget.used(), expected_budget.used()); + assert_eq!(actual_budget.used(), 0); + } + } + assert!(saw_refusal && saw_success); +} + +// Retain the original upstream scalar pricing loop as an independent oracle. +#[allow(clippy::too_many_arguments)] +fn reference_price_optimal_paths( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + runs: &[(u32, u32)], + starts: &[u32], + workspace: OptimalSlices<'_>, + reaches: &mut [(usize, usize, usize)], +) { + let start = block.start; + let end = block.end; + let span = end - start; + let OptimalSlices { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + } = workspace; + // The first position past a match the parse committed to. Nothing is + // priced from the positions before it. See [`NICE_MATCH_LENGTH`]. + let mut committed_through = 0usize; + + for index in 0..span { + let pos = start + index; + if index < committed_through { + continue; + } + // Every priced position extends a literal path to its successor; + // a committed match skips only to an already priced endpoint. Blocks + // are at most 1 MiB, so even all 15-bit literals stay below u32::MAX. + let here = price[index]; + let literal_cost = prices.map_or(ESTIMATED_LITERAL_COST, |prices| { + prices.literal(combined[pos]) as u32 + }); + let literal = here.saturating_add(literal_cost); + if literal < price[index + 1] { + price[index + 1] = literal; + arrive_length[index + 1] = 0; + arrive_distance[index + 1] = 0; + // A literal emits no distance, so it leaves the remembered ones + // exactly as it found them. + arrive_reps[index + 1] = arrive_reps[index]; + arrive_last_length[index + 1] = arrive_last_length[index]; + } + + let max_distance = pos.min(options.max_match_distance); + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if options.max_match_candidates == 0 || max_distance == 0 || max_length < 4 { + continue; + } + + let state = EncoderMatchState { + reps: arrive_reps[index].map(|distance| distance as usize), + last_length: arrive_last_length[index] as usize, + }; + + let mut reaches_len = 0; + let mut longest = 0usize; + + // A match at a remembered distance is priced out of the main table + // alone, a handful of bits against twenty for a fresh distance, so it + // earns its place even when it is shorter than anything the collector + // found. The collector only reports a candidate that beats the longest + // found so far, so these have to be asked for separately. + for repeat in state.reps { + if repeat == 0 || repeat > max_distance { + continue; + } + let length = match_length(combined, pos, repeat, max_length); + if length >= 4 { + reaches[reaches_len] = (4, length, repeat); + reaches_len += 1; + } + } + + // The collector reports nearest first, so the first distance to reach + // a length is the cheapest one that can. Each report that improves on + // the longest so far owns one run of lengths. The tree measures + // against the whole member where the chains stopped at the block, so + // a length is capped here to what this block can still hold. + for &(length, distance) in &runs[starts[index] as usize..starts[index + 1] as usize] { + let length = (length as usize).min(max_length); + if length > longest { + reaches[reaches_len] = (longest + 1, length, distance as usize); + reaches_len += 1; + longest = length; + } + } + + let reaches = &reaches[..reaches_len]; + + // Equal token prices do not make shorter matches redundant: their + // endpoints can expose a better continuation. Price every endpoint + // unless the explicit long-match heuristic commits past all of them. + let committed_reach = reaches + .iter() + .map(|&(_, end, _)| end) + .max() + .filter(|&length| length >= NICE_MATCH_LENGTH); + for &(run_start, run_end, distance) in reaches.iter() { + let mut length = run_start.max(4); + if let Some(committed) = committed_reach { + if run_end < committed { + continue; + } + length = committed; + } + while length <= run_end { + let reach = length; + let cost = match prices { + Some(prices) => prices.match_cost(&state, reach, distance, distance_size), + None => estimated_match_cost(&state, reach, distance, distance_size), + }; + if let Ok(cost) = cost { + let reached = here.saturating_add(cost as u32); + let target = index + reach; + if reached < price[target] { + price[target] = reached; + arrive_length[target] = reach as u32; + arrive_distance[target] = distance as u32; + let mut next = state; + next.remember(reach, distance); + arrive_reps[target] = next.reps.map(|distance| distance as u32); + arrive_last_length[target] = next.last_length as u32; + } + } + length = reach + 1; + } + } + + // A committed match is at least 512 bytes long. Even after the + // distance bonus, that length and every collected u32 distance fit + // their encoder tables, so the endpoint has already been priced. + let longest_reach = reaches.iter().map(|&(_, length, _)| length).max(); + if let Some(reach) = longest_reach { + if reach >= NICE_MATCH_LENGTH { + committed_through = index + reach; + } + } + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/pricing_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/pricing_tests.rs new file mode 100644 index 00000000..55b25e11 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/pricing_tests.rs @@ -0,0 +1,344 @@ +use super::*; + +#[test] +fn prepared_run_prices_preserve_every_length_and_distance_boundary() { + let distances = [0, 1, 3, 4, 5, 256, 257, 8192, 8193, 262144, 262145, 1 << 22]; + let states = [ + EncoderMatchState::default(), + EncoderMatchState { + reps: [1, 257, 8193, 262145], + last_length: 511, + }, + EncoderMatchState { + reps: [3, 3, 7, 3], + last_length: 4, + }, + ]; + for seed in [0, 1, 7] { + let codes = |size: usize| { + (0..size) + .map(|symbol| ((symbol * seed + seed) & 15) as u8) + .collect::>() + }; + let main = codes(MAIN_TABLE_SIZE); + let distance = codes(DISTANCE_TABLE_SIZE_70); + let align = codes(ALIGN_TABLE_SIZE); + let length = codes(LENGTH_TABLE_SIZE); + let actual = TokenPrices { + lengths: LengthSlices { + main: &main, + distance: &distance, + align: &align, + length: &length, + }, + }; + for prices in [None, Some(&actual)] { + let prepared = OptimalMatchPrices::new(prices); + for distance_size in [DISTANCE_TABLE_SIZE_50, DISTANCE_TABLE_SIZE_70] { + for state in states { + for distance in distances { + let run = prepared.run(&state, distance, distance_size, prices).ok(); + for length in 1..=MAX_ENCODER_MATCH_LENGTH { + let expected = match prices { + Some(prices) => { + prices.match_cost(&state, length, distance, distance_size) + } + None => { + estimated_match_cost(&state, length, distance, distance_size) + } + } + .ok() + .map(|cost| cost as u32); + assert_eq!( + run.as_ref().and_then(|run| run.cost(length)), + expected, + "seed={seed}, actual={}, state={state:?}, distance={distance}, length={length}, table={distance_size}", + prices.is_some(), + ); + } + } + } + } + } + } +} + +#[test] +#[cfg(target_pointer_width = "64")] +fn prepared_run_prices_refuse_unencodable_fresh_distances() { + let prepared = OptimalMatchPrices::new(None); + let state = EncoderMatchState::default(); + for (distance, size) in [ + (1usize << 40, DISTANCE_TABLE_SIZE_50), + ((1usize << 33) + 1, DISTANCE_TABLE_SIZE_70), + ] { + assert!(prepared.run(&state, distance, size, None).is_err()); + assert!(estimated_match_cost(&state, 5, distance, size).is_err()); + } +} + +#[derive(Debug, PartialEq)] +struct Paths { + price: Vec, + length: Vec, + distance: Vec, + reps: Vec<[u32; 4]>, + last_length: Vec, +} + +impl Paths { + fn new(span: usize) -> Self { + let mut price = vec![u32::MAX; span + 1]; + price[0] = 0; + Self { + price, + length: vec![0; span + 1], + distance: vec![0; span + 1], + reps: vec![[0; 4]; span + 1], + last_length: vec![0; span + 1], + } + } + + fn slices(&mut self) -> OptimalSlices<'_> { + OptimalSlices { + price: &mut self.price, + arrive_length: &mut self.length, + arrive_distance: &mut self.distance, + arrive_reps: &mut self.reps, + arrive_last_length: &mut self.last_length, + } + } +} + +#[test] +fn prepared_pricing_preserves_all_paths_and_remembered_state() { + let mut random = 0x1234_5678u32; + let noise: Vec<_> = (0..4096) + .map(|_| { + random = random.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (random >> 31) as u8 + }) + .collect(); + let mut text = Vec::new(); + for i in 0..80 { + text.extend_from_slice(b"the same match exposes different continuations "); + text.push(i); + } + for data in [ + vec![0; 4096], + b"abc".repeat(1366), + text, + noise, + (0..1024u32) + .flat_map(|word| (word * 71).to_le_bytes()) + .collect(), + ] { + for start in [0, 256] { + let block = start..data.len(); + for candidates in [0, 16, 128] { + let options = EncodeOptions::new(candidates).with_optimal_parse(true); + let mut collector = OptimalCollector::new(&data, start, options); + let matches = collector.collect(&data, block.clone(), options).unwrap(); + let capacity = matches + .starts + .windows(2) + .map(|pair| (pair[1] - pair[0]) as usize) + .max() + .unwrap_or(0) + + 4; + for distance_size in [DISTANCE_TABLE_SIZE_50, DISTANCE_TABLE_SIZE_70] { + let mut workspace = OptimalWorkspace::new(&Allowance::default()); + let tokens = optimal_tokens_in_workspace( + &data, + block.clone(), + options, + distance_size, + None, + &matches, + &mut workspace, + ) + .unwrap(); + let lengths = table_lengths_for_tokens(&tokens, distance_size).unwrap(); + let actual = TokenPrices { + lengths: lengths.slices(), + }; + for prices in [None, Some(&actual)] { + let mut expected = Paths::new(block.len()); + let mut result = Paths::new(block.len()); + let mut reaches = vec![(0, 0, 0); capacity]; + reference_price_optimal_paths( + &data, + block.clone(), + options, + distance_size, + prices, + &matches.runs, + &matches.starts, + expected.slices(), + &mut reaches, + ); + price_optimal_paths( + &data, + block.clone(), + options, + distance_size, + prices, + &matches.runs, + &matches.starts, + result.slices(), + &mut reaches, + ); + assert_eq!(result, expected, "start={start}, candidates={candidates}, table={distance_size}, actual={}", prices.is_some()); + } + } + } + } + } +} + +// Independent pre-hoisting implementation retained for differential checks. +#[allow(clippy::too_many_arguments)] +fn reference_price_optimal_paths( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + runs: &[(u32, u32)], + starts: &[u32], + workspace: OptimalSlices<'_>, + reaches: &mut [(usize, usize, usize)], +) { + let start = block.start; + let end = block.end; + let span = end - start; + let OptimalSlices { + price, + arrive_length, + arrive_distance, + arrive_reps, + arrive_last_length, + } = workspace; + // The first position past a match the parse committed to. Nothing is + // priced from the positions before it. See [`NICE_MATCH_LENGTH`]. + let mut committed_through = 0usize; + + for index in 0..span { + let pos = start + index; + if index < committed_through { + continue; + } + // Every priced position extends a literal path to its successor; + // a committed match skips only to an already priced endpoint. Blocks + // are at most 1 MiB, so even all 15-bit literals stay below u32::MAX. + let here = price[index]; + let literal_cost = prices.map_or(ESTIMATED_LITERAL_COST, |prices| { + prices.literal(combined[pos]) as u32 + }); + let literal = here.saturating_add(literal_cost); + if literal < price[index + 1] { + price[index + 1] = literal; + arrive_length[index + 1] = 0; + arrive_distance[index + 1] = 0; + // A literal emits no distance, so it leaves the remembered ones + // exactly as it found them. + arrive_reps[index + 1] = arrive_reps[index]; + arrive_last_length[index + 1] = arrive_last_length[index]; + } + + let max_distance = pos.min(options.max_match_distance); + let max_length = (end - pos).min(MAX_ENCODER_MATCH_LENGTH); + if options.max_match_candidates == 0 || max_distance == 0 || max_length < 4 { + continue; + } + + let state = EncoderMatchState { + reps: arrive_reps[index].map(|distance| distance as usize), + last_length: arrive_last_length[index] as usize, + }; + + let mut reaches_len = 0; + let mut longest = 0usize; + + // A match at a remembered distance is priced out of the main table + // alone, a handful of bits against twenty for a fresh distance, so it + // earns its place even when it is shorter than anything the collector + // found. The collector only reports a candidate that beats the longest + // found so far, so these have to be asked for separately. + for repeat in state.reps { + if repeat == 0 || repeat > max_distance { + continue; + } + let length = match_length(combined, pos, repeat, max_length); + if length >= 4 { + reaches[reaches_len] = (4, length, repeat); + reaches_len += 1; + } + } + + // The collector reports nearest first, so the first distance to reach + // a length is the cheapest one that can. Each report that improves on + // the longest so far owns one run of lengths. The tree measures + // against the whole member where the chains stopped at the block, so + // a length is capped here to what this block can still hold. + for &(length, distance) in &runs[starts[index] as usize..starts[index + 1] as usize] { + let length = (length as usize).min(max_length); + if length > longest { + reaches[reaches_len] = (longest + 1, length, distance as usize); + reaches_len += 1; + longest = length; + } + } + + let reaches = &reaches[..reaches_len]; + + // Equal token prices do not make shorter matches redundant: their + // endpoints can expose a better continuation. Price every endpoint + // unless the explicit long-match heuristic commits past all of them. + let committed_reach = reaches + .iter() + .map(|&(_, end, _)| end) + .max() + .filter(|&length| length >= NICE_MATCH_LENGTH); + for &(run_start, run_end, distance) in reaches.iter() { + let mut length = run_start.max(4); + if let Some(committed) = committed_reach { + if run_end < committed { + continue; + } + length = committed; + } + while length <= run_end { + let reach = length; + let cost = match prices { + Some(prices) => prices.match_cost(&state, reach, distance, distance_size), + None => estimated_match_cost(&state, reach, distance, distance_size), + }; + if let Ok(cost) = cost { + let reached = here.saturating_add(cost as u32); + let target = index + reach; + if reached < price[target] { + price[target] = reached; + arrive_length[target] = reach as u32; + arrive_distance[target] = distance as u32; + let mut next = state; + next.remember(reach, distance); + arrive_reps[target] = next.reps.map(|distance| distance as u32); + arrive_last_length[target] = next.last_length as u32; + } + } + length = reach + 1; + } + } + + // A committed match is at least 512 bytes long. Even after the + // distance bonus, that length and every collected u32 distance fit + // their encoder tables, so the endpoint has already been priced. + let longest_reach = reaches.iter().map(|&(_, length, _)| length).max(); + if let Some(reach) = longest_reach { + if reach >= NICE_MATCH_LENGTH { + committed_through = index + reach; + } + } + } +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/streaming_copy_tests.rs b/arca-rar/vendor/rars/src/codec/rar50/streaming_copy_tests.rs new file mode 100644 index 00000000..d777f16d --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/streaming_copy_tests.rs @@ -0,0 +1,227 @@ +use super::*; +use std::convert::Infallible; + +fn append_chunk(decoded: &mut Vec, chunk: DecodedChunk<'_>) { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => decoded.resize(decoded.len() + len, byte), + } +} + +fn reference_match(history: &[u8], pending: &[u8], distance: usize, length: usize) -> Vec { + let mut bytes = [history, pending].concat(); + let missing = distance == 0 || distance > bytes.len(); + for _ in 0..length { + let byte = if missing { + 0 + } else { + bytes[bytes.len() - distance] + }; + bytes.push(byte); + } + bytes[history.len()..].to_vec() +} + +#[test] +fn buffered_matches_equal_scalar_with_history_pending_overlap_and_missing_history() { + for history_len in 0..=6 { + let history: Vec<_> = (0..history_len).map(|n| n as u8 + 11).collect(); + let mut decoder = ReaderState::new(&Allowance::default()); + decoder.history = Buffer::from_vec(history.clone()); + for pending_len in 0..=6 { + let pending: Vec<_> = (0..pending_len).map(|n| n as u8 + 31).collect(); + for distance in 0..=history_len + pending_len + 2 { + for length in [0, 1, 2, 3, 7, 32, 129] { + let mut output = Buffer::from_vec(pending.clone()); + decoder + .copy_match(&mut output, distance, length, pending_len + length, 32) + .unwrap(); + assert_eq!( + &*output, + reference_match(&history, &pending, distance, length), + "history={history_len}, pending={pending_len}, distance={distance}, length={length}" + ); + } + } + } + } +} + +#[test] +fn buffered_overlap_preserves_output_limit_and_workspace_refusal() { + let budget = Allowance::limited(128); + let decoder = ReaderState::new(&budget); + let mut output = Buffer::with_capacity(128, &budget).unwrap(); + output.extend_from_slice(b"ab").unwrap(); + decoder.copy_match(&mut output, 2, 126, 128, 128).unwrap(); + assert_eq!(&*output, b"ab".repeat(64)); + assert_eq!(budget.used(), 128); + assert!(matches!( + decoder.copy_match(&mut output, 2, 1, 128, 128), + Err(Error::InvalidData("RAR 5 match exceeds output limit")) + )); + assert!(matches!( + decoder.copy_match(&mut output, 2, 1, 129, 128), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(&*output, b"ab".repeat(64)); + assert_eq!(budget.used(), 128); + drop(output); + drop(decoder); + assert_eq!(budget.used(), 0); +} + +#[test] +fn streaming_matches_equal_scalar_with_history_pending_overlap_and_missing_history() { + for history_len in 0..=6 { + let history: Vec<_> = (0..history_len).map(|n| n as u8 + 11).collect(); + for pending_len in 0..=6 { + let pending: Vec<_> = (0..pending_len).map(|n| n as u8 + 31).collect(); + for distance in 1..=history_len + pending_len + 2 { + for length in [0, 1, 2, 3, 7, 32, 129] { + let mut output = StreamingOutput::new( + Buffer::from_vec(history.clone()), + pending_len + length, + 32, + 32, + ) + .unwrap(); + let mut decoded = Vec::new(); + let mut sink = |chunk: DecodedChunk<'_>| { + append_chunk(&mut decoded, chunk); + Ok::<(), Infallible>(()) + }; + for &byte in &pending { + output.push(byte, &mut sink).unwrap(); + } + output.copy_match(distance, length, &mut sink).unwrap(); + output.finish(&mut sink).unwrap(); + assert_eq!( + decoded, + reference_match(&history, &pending, distance, length), + "history={history_len}, pending={pending_len}, distance={distance}, length={length}" + ); + } + } + } + } +} + +#[test] +fn streaming_match_crosses_both_wrapped_history_slices_and_multiple_flushes() { + let count = 2 * STREAM_FLUSH_THRESHOLD + 37; + let mut history = Buffer::with_capacity(16, &Allowance::default()).unwrap(); + history + .extend_from_slice(&(1..=12).collect::>()) + .unwrap(); + let mut output = StreamingOutput::new(history, count + 3, 16, 16).unwrap(); + output.history.discard_prefix(8); + output.history.extend_admitted(31..39); + assert!(!output.history.as_slices().1.is_empty()); + let history: Vec<_> = output.history.iter().copied().collect(); + let pending = [91, 92, 93]; + let mut decoded = Vec::new(); + let mut chunks = Vec::new(); + let mut sink = |chunk: DecodedChunk<'_>| { + chunks.push(match chunk { + DecodedChunk::Bytes(bytes) => bytes.len(), + DecodedChunk::Repeated { len, .. } => len, + }); + append_chunk(&mut decoded, chunk); + Ok::<(), Infallible>(()) + }; + for byte in pending { + output.push(byte, &mut sink).unwrap(); + } + output.copy_match(12, count, &mut sink).unwrap(); + output.finish(&mut sink).unwrap(); + assert_eq!(decoded, reference_match(&history, &pending, 12, count)); + assert_eq!(chunks, [STREAM_FLUSH_THRESHOLD, STREAM_FLUSH_THRESHOLD, 40]); + assert_eq!(output.written(), decoded.len()); + assert_eq!( + output.into_history().into_vec(), + decoded[decoded.len() - 16..] + ); +} + +#[test] +fn streaming_match_stops_on_the_first_failed_flush() { + let mut output = StreamingOutput::new( + Buffer::from_vec(b"abc".to_vec()), + STREAM_FLUSH_THRESHOLD + 7, + 3, + 3, + ) + .unwrap(); + let mut calls = 0; + assert!(matches!( + output.copy_match(3, STREAM_FLUSH_THRESHOLD + 7, &mut |_chunk| { + calls += 1; + Err("failed sink") + }), + Err(StreamDecodeError::Sink("failed sink")) + )); + assert_eq!(calls, 1); + assert_eq!(output.written(), STREAM_FLUSH_THRESHOLD); + assert_eq!(output.pending.len(), STREAM_FLUSH_THRESHOLD); + assert_eq!( + &*output.history, + &std::collections::VecDeque::from(b"abc".to_vec()) + ); +} + +#[test] +fn streaming_match_uses_only_its_admitted_pending_and_history_storage() { + let budget = Allowance::limited((STREAM_FLUSH_THRESHOLD + 16) as u64); + let mut history = Buffer::with_capacity(16, &budget).unwrap(); + history.extend_from_slice(b"abcdefghijklmnop").unwrap(); + let count = 2 * STREAM_FLUSH_THRESHOLD + 5; + let mut output = StreamingOutput::new(history, count, 16, 16).unwrap(); + let mut observed = 0; + let mut sink = |chunk: DecodedChunk<'_>| { + observed += match chunk { + DecodedChunk::Bytes(bytes) => bytes.len(), + DecodedChunk::Repeated { len, .. } => len, + }; + Ok::<(), Infallible>(()) + }; + output.copy_match(7, count, &mut sink).unwrap(); + output.finish(&mut sink).unwrap(); + assert_eq!(observed, count); + assert_eq!(budget.used(), (STREAM_FLUSH_THRESHOLD + 16) as u64); + drop(output); + assert_eq!(budget.used(), 0); +} + +#[cfg(feature = "write")] +#[test] +fn streaming_match_flush_preserves_decoder_cancellation() { + let data: Vec<_> = b"abc" + .iter() + .copied() + .cycle() + .take(3 * STREAM_FLUSH_THRESHOLD) + .collect(); + let input = Unpack50Encoder::new().encode_member(&data, 0).unwrap(); + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack50Decoder::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + let mut calls = 0; + let result = decoder.decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + data.len(), + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| { + calls += 1; + token.cancel(); + Ok::<(), Infallible>(()) + }, + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::Cancelled)) + )); + assert_eq!(calls, 1); +} diff --git a/arca-rar/vendor/rars/src/codec/rar50/tests.rs b/arca-rar/vendor/rars/src/codec/rar50/tests.rs new file mode 100644 index 00000000..c7d38c88 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rar50/tests.rs @@ -0,0 +1,5770 @@ +use crate::codec::workspace::RefusingBudget; + +fn assert_each_allocation_refusal( + mut work: impl FnMut(&RefusingBudget) -> crate::codec::Result, +) -> usize { + let baseline = RefusingBudget::new(usize::MAX); + drop(work(&baseline).unwrap()); + let attempts = baseline.attempts(); + assert!(attempts > 0); + assert_eq!(baseline.used(), 0); + + for fail_at in 0..attempts { + let budget = RefusingBudget::new(fail_at); + assert!( + matches!(work(&budget), Err(Error::Cancelled)), + "failure at allocation {fail_at}" + ); + assert_eq!(budget.used(), 0, "failure at allocation {fail_at}"); + } + attempts +} + +#[test] +fn reader_workspace_refusals_release_buffered_filters_and_checkpoints() { + let data = b"raw reader dictionary and filtered output\n".repeat(32); + for kind in [ + crate::FilterKind::E8, + crate::FilterKind::Delta { channels: 2 }, + ] { + let packed = Unpack50Encoder::new() + .encode_member_with_filter(&data, 0, crate::FilterSpec::whole(kind)) + .unwrap(); + let attempts = assert_each_allocation_refusal(|budget| { + let mut state = ReaderState::new(budget); + let decoded = state.decode_member_with_dictionary( + &packed, + 0, + data.len(), + 64, + false, + DecodeMode::Lz, + )?; + assert_eq!(&*decoded, &data); + let checkpoint = state.try_clone()?; + assert_eq!(checkpoint.history, state.history); + assert_eq!( + checkpoint.tables.as_ref().unwrap().main.state.len(), + state.tables.as_ref().unwrap().main.state.len() + ); + Ok(decoded) + }); + assert!(attempts > 15); + } +} + +#[test] +fn reader_workspace_refusals_release_streaming_history_and_input() { + let data = b"streaming reader window\n".repeat(4000); + let packed = encode_literal_only(&data, 0).unwrap(); + let attempts = assert_each_allocation_refusal(|budget| { + let mut state = ReaderState::new(budget); + let mut emitted = 0; + let result = state.decode_member_from_reader_with_dictionary_to_sink( + &mut packed.as_slice(), + 0, + data.len(), + 1024, + false, + |chunk| { + emitted += match chunk { + DecodedChunk::Bytes(bytes) => bytes.len(), + DecodedChunk::Repeated { len, .. } => len, + }; + Ok::<(), std::convert::Infallible>(()) + }, + ); + match result { + Ok(()) => { + assert_eq!(emitted, data.len()); + assert_eq!(&*state.history, &data[data.len() - 1024..]); + Ok(()) + } + Err(StreamDecodeError::Decode(error)) => Err(error), + Err(_) => panic!("unexpected streaming failure"), + } + }); + assert!(attempts > 8); +} + +#[test] +fn reader_workspace_retained_output_and_checkpoint_remain_charged() { + let data = b"retained reader result".repeat(20); + let packed = encode_literal_only(&data, 0).unwrap(); + let ledger = Allowance::limited(256 * 1024); + let mut state = ReaderState::new(&ledger); + let output = state + .decode_member_with_dictionary(&packed, 0, data.len(), 64, false, DecodeMode::Lz) + .unwrap(); + let owners = ledger.used(); + let checkpoint = state.try_clone().unwrap(); + assert!(ledger.used() > owners); + drop(state); + assert!(ledger.used() > output.capacity() as u64); + drop(checkpoint); + assert_eq!(ledger.used(), output.capacity() as u64); + assert_eq!(&*output, &data); + drop(output); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn reader_workspace_history_replacement_admits_overlap_before_mutating() { + let ledger = Allowance::limited(8); + let mut state = ReaderState::new(&ledger); + state.remember_history(b"ABCDEFGH", 8).unwrap(); + assert_eq!(ledger.used(), 8); + assert!(matches!( + state.remember_history(b"xy", 2), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(&*state.history, b"ABCDEFGH"); + assert_eq!(ledger.used(), 8); + drop(state); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn bounded_reader_admits_input_before_reading_and_releases_on_errors() { + struct Source<'a> { + bytes: std::io::Cursor<&'a [u8]>, + reads: usize, + } + impl Read for Source<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.reads += 1; + self.bytes.read(buffer) + } + } + let data: Vec<_> = (0..8192).map(|i| ((i * 71) ^ (i >> 5)) as u8).collect(); + let options = EncodeOptions::new(16).with_max_match_distance(1024); + let denied = Allowance::limited(0); + let mut source = Source { + bytes: std::io::Cursor::new(&data), + reads: 0, + }; + let error = reader_to_with_allowance( + &mut source, + data.len() as u64, + &mut std::io::sink(), + 0, + options, + 256, + None, + &denied, + ) + .unwrap_err(); + assert!(matches!( + error, + crate::Error::Codec(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(source.reads, 0); + assert_eq!(denied.used(), 0); + + let allowance = Allowance::limited(8 * 1024 * 1024); + let mut output = Vec::new(); + reader_to_with_allowance( + &mut std::io::Cursor::new(&data), + data.len() as u64, + &mut output, + 0, + options, + 256, + None, + &allowance, + ) + .unwrap(); + assert_eq!(decode_lz(&output, 0, data.len()).unwrap(), data); + assert_eq!(allowance.used(), 0); + for declared in [data.len() as u64 - 1, data.len() as u64 + 1] { + assert!(reader_to_with_allowance( + &mut std::io::Cursor::new(&data), + declared, + &mut std::io::sink(), + 0, + options, + 256, + None, + &allowance + ) + .is_err()); + assert_eq!(allowance.used(), 0); + } + let mut prefix = Vec::new(); + assert!(matches!( + reader_to_with_allowance( + &mut std::io::Cursor::new(&data), + data.len() as u64, + &mut prefix, + 0, + options, + 256, + Some(&mut |_| false), + &allowance + ), + Err(crate::Error::Cancelled) + )); + assert!( + !prefix.is_empty(), + "streaming refusal can leave an emitted prefix" + ); + assert_eq!(allowance.used(), 0); +} + +#[test] +fn filtered_state_owns_transform_scratch_history_and_outputs() { + let data: Vec<_> = (0..FILTERED_LZ_BLOCK_SIZE + 257) + .map(|i| ((i * 17) ^ (i >> 9)) as u8) + .collect(); + for version in [0, 1] { + for optimal in [false, true] { + let options = EncodeOptions::new(16) + .with_optimal_parse(optimal) + .with_max_match_distance(8192); + for size in [4096, data.len()] { + for kind in [ + crate::FilterKind::Delta { channels: 3 }, + crate::FilterKind::E8E9, + crate::FilterKind::Arm, + ] { + let filters = [crate::FilterSpec::whole(kind)]; + let allowance = Allowance::limited(16 * 1024 * 1024); + let mut state = EncoderState::new(options, &allowance); + let mut reference = Unpack50Encoder::with_options(options); + let expected = reference + .encode_member_with_filters(&data[..size], version, &filters) + .unwrap(); + let packed = state + .encode(&data[..size], version, Some(&filters), None) + .unwrap(); + assert_eq!(&*packed, expected); + assert_eq!(&*state.history, &data[size.saturating_sub(8192)..size]); + drop(packed); + let history_charge = allowance.used(); + assert!(history_charge >= state.history.len() as u64); + let following = state.encode(b"next member", version, None, None).unwrap(); + assert_eq!( + &*following, + reference.encode_member(b"next member", version).unwrap() + ); + drop(state); + assert!(allowance.used() >= following.len() as u64); + drop(following); + assert_eq!(allowance.used(), 0); + } + } + } + } +} + +#[test] +fn delta_scratch_refusal_does_not_modify_caller_input_or_leak_records() { + let input = [7u8; 512]; + let filters = [crate::FilterSpec::whole(crate::FilterKind::Delta { + channels: 3, + })]; + let records = std::mem::size_of::() as u64; + for limit in [1024 + records - 1, 1024 + records] { + let allowance = Allowance::limited(limit); + let result = filtered_member_with_allowance(&input, &filters, &allowance); + assert_eq!(input, [7; 512]); + if limit == 1024 + records - 1 { + assert!(matches!(result, Err(Error::WorkspaceLimitExceeded(_)))); + assert_eq!(allowance.used(), 0); + } else { + let (data, descriptors) = result.unwrap(); + assert_eq!(allowance.used(), 512 + records); + assert_eq!(descriptors.len(), 1); + assert_eq!( + filters::delta_decode_with_control( + &data, + 3, + rar50_delta_messages(), + &crate::read_control::ReadControl::default() + ) + .unwrap(), + input + ); + drop((data, descriptors)); + assert_eq!(allowance.used(), 0); + } + } +} + +#[test] +fn failed_stateful_encodes_preserve_history_and_allow_reuse() { + let allowance = Allowance::limited(4 * 1024 * 1024); + let options = EncodeOptions::new(16) + .with_optimal_parse(true) + .with_max_match_distance(128); + let mut state = EncoderState::new(options, &allowance); + drop(state.encode(b"existing history", 0, None, None).unwrap()); + let previous = state.history.to_vec(); + let retained = allowance.used(); + let blocker = + Buffer::filled((4 * 1024 * 1024 - retained - 64) as usize, 0u8, &allowance).unwrap(); + assert!(matches!( + state.encode(b"refused input", 0, None, None), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(&*state.history, previous); + drop(blocker); + assert_eq!(allowance.used(), retained); + let filters = [crate::FilterSpec::whole(crate::FilterKind::E8)]; + assert!(matches!( + state.encode(b"cancelled input", 0, Some(&filters), Some(&mut |_| false)), + Err(Error::Cancelled) + )); + assert_eq!(&*state.history, previous); + assert_eq!(allowance.used(), retained); + let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = state.encode( + b"callback panic", + 0, + Some(&filters), + Some(&mut |_| panic!("callback")), + ); + })); + assert!(unwind.is_err()); + assert_eq!(&*state.history, previous); + assert_eq!(allowance.used(), retained); + drop(state.encode(b"successful retry", 0, None, None).unwrap()); + assert!(state.history.ends_with(b"successful retry")); + drop(state); + assert_eq!(allowance.used(), 0); +} + +#[test] +fn streaming_results_remain_owned_and_cancelled_runs_release_all_blocks() { + let data: Vec<_> = (0..8192).map(|i| (i * 71) as u8).collect(); + let blocks = [(2048, false), (4096, false), (8192, true)]; + for optimal in [false, true] { + let allowance = Allowance::limited(8 * 1024 * 1024); + let options = EncodeOptions::new(16) + .with_optimal_parse(optimal) + .with_max_match_distance(1024); + let expected = + encode_lz_streaming_blocks(&data, b"prior history", &blocks, 0, options, None).unwrap(); + let outputs = streaming_blocks_with_allowance( + &data, + b"prior history", + &blocks, + 0, + options, + None, + &allowance, + ) + .unwrap(); + for (actual, expected) in outputs.iter().zip(expected) { + assert_eq!(&**actual, expected); + } + let mut outputs = outputs.into_iter(); + let first = outputs.next().unwrap(); + drop(outputs); + assert!(allowance.used() >= first.len() as u64); + drop(first); + assert_eq!(allowance.used(), 0); + assert!(matches!( + streaming_blocks_with_allowance( + &data, + &[], + &blocks, + 0, + options, + Some(&mut |_| false), + &allowance + ), + Err(Error::Cancelled) + )); + assert_eq!(allowance.used(), 0); + assert!(streaming_blocks_with_allowance( + &data, + &[], + &[(9000, true)], + 0, + options, + None, + &allowance + ) + .is_err()); + assert_eq!(allowance.used(), 0); + } +} + +#[test] +fn streaming_block_plan_rejects_gaps_and_invalid_ends() { + let data = b"ABCD"; + let options = EncodeOptions::new(0); + let allowance = Allowance::default(); + for blocks in [ + vec![(0, false)], + vec![(5, true)], + vec![(2, false), (2, true)], + ] { + assert!(matches!( + streaming_blocks_with_allowance(data, &[], &blocks, 0, options, None, &allowance,), + Err(Error::InvalidData("RAR 5 streaming block range is invalid")) + )); + } + assert!(matches!( + streaming_blocks_with_allowance(data, &[], &[(2, false)], 0, options, None, &allowance,), + Err(Error::InvalidData( + "RAR 5 streaming blocks do not cover input" + )) + )); +} + +#[test] +fn streaming_block_encode_errors_release_first_and_shared_search_state() { + let data = b"ABCD"; + let blocks = [(2, false), (4, true)]; + let expected = Error::InvalidData("RAR 5 unknown compression algorithm version"); + for (optimal, history) in [(true, &b""[..]), (true, &b"prior"[..]), (false, &b""[..])] { + let allowance = Allowance::limited(32 * 1024 * 1024); + let options = EncodeOptions::new(0).with_optimal_parse(optimal); + assert_eq!( + streaming_blocks_with_allowance(data, history, &blocks, 2, options, None, &allowance,), + Err(expected.clone()), + "optimal={optimal}, history={history:?}" + ); + assert_eq!(allowance.used(), 0); + } +} + +#[test] +fn empty_streaming_member_has_no_blocks_or_progress() { + let options = EncodeOptions::new(0).with_optimal_parse(true); + let allowance = Allowance::limited(0); + let mut called = false; + let mut progress = |_: usize| { + called = true; + true + }; + let output = + streaming_blocks_with_allowance(&[], &[], &[], 0, options, Some(&mut progress), &allowance) + .unwrap(); + assert!(output.is_empty()); + assert!(!called); + assert_eq!(allowance.used(), 0); +} + +#[test] +fn member_allowance_covers_history_tables_and_retained_block_output() { + let data = b"bounded member with repeated words and short matches\n".repeat(3000); + let history = b"short matches and remembered history\n".repeat(1000); + for version in [0, 1] { + for optimal in [false, true] { + let options = EncodeOptions::new(16) + .with_optimal_parse(optimal) + .with_max_match_distance(65536); + for history in [&[][..], &history[..]] { + let expected = + encode_lz_member_inner(&data, history, version, options, None).unwrap(); + let allowance = Allowance::limited(32 * 1024 * 1024); + let output = encode_member_with_allowance( + &data, history, version, options, None, &allowance, + ) + .unwrap(); + assert_eq!(&*output, expected); + assert!(allowance.used() >= output.len() as u64); + let retained = allowance.used(); + let second = encode_member_with_allowance( + b"another member", + &[], + version, + options, + None, + &allowance, + ) + .unwrap(); + assert!(allowance.used() >= retained + second.len() as u64); + drop(second); + assert_eq!(allowance.used(), retained); + drop(output); + assert_eq!(allowance.used(), 0); + } + } + } +} + +#[test] +fn large_member_search_setup_failures_release_the_allowance() { + let data = vec![b'A'; LZ_BLOCK_SIZE + 1]; + let filters = [crate::FilterSpec::whole(crate::FilterKind::E8)]; + for optimal in [false, true] { + let options = EncodeOptions::new(0).with_optimal_parse(optimal); + + let unfiltered = Allowance::limited(0); + assert!(matches!( + encode_member_with_allowance(&data, &[], 0, options, None, &unfiltered,), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(unfiltered.used(), 0); + + for limit in [128, data.len() as u64 + 4096] { + let filtered = Allowance::limited(limit); + assert!(matches!( + filtered_lz_blocks(&data, &filters, &[], 0, options, None, &filtered), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(filtered.used(), 0); + } + } +} + +#[test] +fn optimal_parse_pass_failures_release_the_allowance() { + let data: Vec = (0..512).map(|i| (i * 71) as u8).collect(); + let options = EncodeOptions::new(16).with_optimal_parse(true); + // The first limit fails while building the initial optimal path; the + // second permits that path but fails while repricing it. + for limit in [1_080_000, 1_105_000] { + let allowance = Allowance::limited(limit); + assert!(matches!( + encode_member_with_allowance(&data, &[], 0, options, None, &allowance), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(allowance.used(), 0); + } +} + +#[test] +fn member_refusal_cancellation_and_unwind_release_the_whole_pipeline() { + let data = b"member cancellation and refusal\n".repeat(100); + let history = b"history".repeat(100); + let options = EncodeOptions::new(16).with_optimal_parse(true); + for limit in [0, 4096, 65536, 512 * 1024] { + let allowance = Allowance::limited(limit); + assert!(matches!( + encode_member_with_allowance(&data, &history, 0, options, None, &allowance), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(allowance.used(), 0); + } + let allowance = Allowance::limited(16 * 1024 * 1024); + assert!(matches!( + encode_member_with_allowance( + &data, + &history, + 0, + options, + Some(&mut |_| false), + &allowance + ), + Err(Error::Cancelled) + )); + assert_eq!(allowance.used(), 0); + let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = encode_member_with_allowance( + &data, + &history, + 0, + options, + Some(&mut |_| panic!("progress callback panic")), + &allowance, + ); + })); + assert!(unwind.is_err()); + assert_eq!(allowance.used(), 0); +} + +#[test] +fn bit_growth_and_frame_copy_reserve_before_mutation() { + let allowance = Allowance::limited(8); + let mut writer = BitWriter::with_allowance(&allowance); + writer.try_write_bits(0, 64).unwrap(); + let filled = writer.bytes.to_vec(); + let filled_bits = writer.bit_pos; + assert!(writer.try_write_bits(1, 1).is_err()); + assert_eq!(&*writer.bytes, filled); + assert_eq!(writer.bit_pos, filled_bits); + drop(writer); + assert_eq!(allowance.used(), 0); + + for limit in [18, 19] { + let allowance = Allowance::limited(limit); + let payload = Buffer::filled(8, 0u8, &allowance).unwrap(); + let frame = encode_compressed_block_with_allowance(&payload, 64, true, true, &allowance); + if limit == 18 { + assert!(matches!(frame, Err(Error::WorkspaceLimitExceeded(_)))); + assert_eq!(allowance.used(), 8); + } else { + let frame = frame.unwrap(); + assert_eq!(allowance.used(), 19); + drop(payload); + assert_eq!(allowance.used(), 11); + drop(frame); + assert_eq!(allowance.used(), 0); + } + } +} + +#[test] +fn token_block_budget_refusals_release_all_encoder_owners() { + let base = [ + EncodeToken::Literal(b'A'), + EncodeToken::Literal(b'B'), + EncodeToken::Match { + length: 2, + distance: 2, + }, + EncodeToken::Match { + length: 2, + distance: 2, + }, + EncodeToken::Match { + length: 3, + distance: 2, + }, + ]; + + for filtered in [false, true] { + let mut tokens = Vec::new(); + if filtered { + tokens.push(EncodeToken::Filter(EncodeFilter { + offset: 0, + length: 9, + filter_type: FilterType::E8, + channels: 0, + })); + } + tokens.extend_from_slice(&base); + tokens.extend(std::iter::repeat_n(EncodeToken::Literal(b'A'), 100)); + let expected = encode_token_block(&tokens, 0, DISTANCE_TABLE_SIZE_50, true).unwrap(); + let mut decoded = b"ABABABABA".to_vec(); + decoded.extend(std::iter::repeat_n(b'A', 100)); + assert_eq!(decode_lz(&expected, 0, decoded.len()).unwrap(), decoded); + + let mut limit = 0; + let mut refusals = 0; + let mut admitted = false; + for _ in 0..256 { + let allowance = Allowance::limited(limit); + match encode_token_block_with_allowance( + &tokens, + 0, + DISTANCE_TABLE_SIZE_50, + true, + &allowance, + ) { + Err(Error::WorkspaceLimitExceeded(details)) => { + refusals += 1; + assert_eq!(allowance.used(), 0); + let next = details.used + details.required; + assert!(next > limit); + limit = next; + } + Ok(encoded) => { + assert_eq!(&*encoded, expected); + drop(encoded); + assert_eq!(allowance.used(), 0); + admitted = true; + break; + } + Err(error) => panic!("unexpected token block error: {error:?}"), + } + } + assert!( + admitted, + "token block did not fit after 256 admission thresholds" + ); + assert!(refusals > 1); + + let baseline_budget = RefusingBudget::new(usize::MAX); + let baseline = encode_token_block_with_allowance( + &tokens, + 0, + DISTANCE_TABLE_SIZE_50, + true, + &baseline_budget, + ) + .unwrap(); + assert_eq!(&*baseline, expected); + let attempts = baseline_budget.attempts(); + drop(baseline); + assert_eq!(baseline_budget.used(), 0); + assert!(attempts > 1); + + for fail_at in 0..attempts { + let budget = RefusingBudget::new(fail_at); + assert_eq!( + encode_token_block_with_allowance( + &tokens, + 0, + DISTANCE_TABLE_SIZE_50, + true, + &budget, + ), + Err(Error::Cancelled), + "filtered={filtered}, failure at allocation {fail_at}" + ); + assert_eq!(budget.used(), 0); + } + } +} + +#[test] +fn member_allocation_refusals_release_search_and_output_owners() { + let data = b"ABCDABCDABCDABCD".repeat(16); + let filters = [EncodeFilter { + offset: 0, + length: data.len(), + filter_type: FilterType::E8, + channels: 0, + }]; + + for optimal in [false, true] { + let options = EncodeOptions::new(16) + .with_max_match_distance(256) + .with_optimal_parse(optimal); + let expected = + encode_member_with_allowance(&data, &[], 0, options, None, &Allowance::default()) + .unwrap(); + assert_eq!(decode_lz(&expected, 0, data.len()).unwrap(), data); + let attempts = assert_each_allocation_refusal(|budget| { + encode_member_with_allowance(&data, &[], 0, options, None, budget) + }); + assert!(attempts > 1); + + let expected = encode_filtered_member_with_allowance( + &data, + &[], + 0, + &filters, + options, + None, + &Allowance::default(), + ) + .unwrap(); + assert_eq!(decode_lz(&expected, 0, data.len()).unwrap(), data); + let attempts = assert_each_allocation_refusal(|budget| { + encode_filtered_member_with_allowance(&data, &[], 0, &filters, options, None, budget) + }); + assert!(attempts > 1); + + let blocks = [(data.len() / 2, false), (data.len(), true)]; + let expected = streaming_blocks_with_allowance( + &data, + &[], + &blocks, + 0, + options, + None, + &Allowance::default(), + ) + .unwrap(); + let packed: Vec<_> = expected + .iter() + .flat_map(|block| block.iter().copied()) + .collect(); + assert_eq!(decode_lz(&packed, 0, data.len()).unwrap(), data); + let attempts = assert_each_allocation_refusal(|budget| { + streaming_blocks_with_allowance(&data, &[], &blocks, 0, options, None, budget) + }); + assert!(attempts > 1); + + let specs = [crate::FilterSpec::whole(crate::FilterKind::E8)]; + let expected = + filtered_lz_blocks(&data, &specs, &[], 0, options, None, &Allowance::default()) + .unwrap(); + assert_eq!(decode_lz(&expected, 0, data.len()).unwrap(), data); + let attempts = assert_each_allocation_refusal(|budget| { + filtered_lz_blocks(&data, &specs, &[], 0, options, None, budget) + }); + assert!(attempts > 1); + } +} + +#[test] +fn filtered_multiblock_output_growth_refusals_release_storage() { + let data = vec![b'A'; FILTERED_LZ_BLOCK_SIZE + 1]; + let filters = [crate::FilterSpec::whole(crate::FilterKind::E8)]; + let options = EncodeOptions::new(16).with_max_match_distance(256); + let expected = filtered_lz_blocks( + &data, + &filters, + &[], + 0, + options, + None, + &Allowance::default(), + ) + .unwrap(); + assert_eq!(decode_lz(&expected, 0, data.len()).unwrap(), data); + assert_each_allocation_refusal(|budget| { + filtered_lz_blocks(&data, &filters, &[], 0, options, None, budget) + }); +} + +#[test] +fn member_history_and_large_output_refusals_release_storage() { + let data: Vec = (0..2048) + .map(|index| ((index * 71) ^ (index >> 3)) as u8) + .collect(); + let history = vec![0xee; 256]; + let blocks = [(data.len() / 2, false), (data.len(), true)]; + let filters = [EncodeFilter { + offset: 0, + length: data.len(), + filter_type: FilterType::E8, + channels: 0, + }]; + + for optimal in [false, true] { + let options = EncodeOptions::new(16) + .with_max_match_distance(history.len()) + .with_optimal_parse(optimal); + assert_each_allocation_refusal(|budget| { + streaming_blocks_with_allowance(&data, &history, &blocks, 0, options, None, budget) + }); + assert_each_allocation_refusal(|budget| { + encode_filtered_member_with_allowance( + &data, &history, 0, &filters, options, None, budget, + ) + }); + assert_each_allocation_refusal(|budget| { + let mut state = EncoderState::new(options, budget); + state.encode(&data, 0, None, None) + }); + } + + let large = vec![b'A'; LZ_BLOCK_SIZE + 1]; + let options = EncodeOptions::new(16).with_max_match_distance(256); + let expected = + encode_member_with_allowance(&large, &[], 0, options, None, &Allowance::default()).unwrap(); + assert_eq!(decode_lz(&expected, 0, large.len()).unwrap(), large); + assert_each_allocation_refusal(|budget| { + encode_member_with_allowance(&large, &[], 0, options, None, budget) + }); +} + +#[test] +fn parse_repricing_and_lazy_literal_refusals_release_storage() { + let optimal_data = wordy_text(4096); + let optimal = EncodeOptions::new(32).with_optimal_parse(true); + assert_each_allocation_refusal(|budget| { + encode_tokens_with_allowance( + &optimal_data, + 0..optimal_data.len(), + MemberSearch::Fresh, + optimal, + DISTANCE_TABLE_SIZE_50, + &[], + None, + budget, + ) + }); + + let pos = 160; + let mut lazy_data: Vec = (0..240u16) + .map(|value| value.wrapping_mul(91) as u8) + .collect(); + lazy_data[pos - 30..pos - 22].copy_from_slice(b"ABCDEFGH"); + lazy_data[pos - 80..pos - 70].copy_from_slice(b"CDEFGHIJKL"); + lazy_data[pos..pos + 12].copy_from_slice(b"ABCDEFGHIJKL"); + let lazy = EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(2); + let tokens = encode_tokens_with_allowance( + &lazy_data, + pos..lazy_data.len(), + MemberSearch::Fresh, + lazy, + DISTANCE_TABLE_SIZE_50, + &[], + None, + &Allowance::default(), + ) + .unwrap(); + assert!(matches!(tokens.first(), Some(EncodeToken::Literal(_)))); + assert_each_allocation_refusal(|budget| { + encode_tokens_with_allowance( + &lazy_data, + pos..lazy_data.len(), + MemberSearch::Fresh, + lazy, + DISTANCE_TABLE_SIZE_50, + &[], + None, + budget, + ) + }); + + assert_each_allocation_refusal(|budget| EncoderCodeTable::from_lengths(&[1, 1], budget)); +} + +#[test] +fn production_limited_budget_covers_residual_allocation_sites() { + let denied = Allowance::limited(0); + assert!(matches!( + EncoderCodeTable::from_lengths(&[1, 1], &denied), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(denied.used(), 0); + + let mut noise = 0x2545_f491_4f6c_dd1du64; + let data: Vec = (0..4096) + .map(|_| { + noise ^= noise << 13; + noise ^= noise >> 7; + noise ^= noise << 17; + b'A' + ((noise >> 40) as u8 & 1) + }) + .collect(); + let options = EncodeOptions::new(64).with_optimal_parse(true); + for start in [0, data.len() / 2] { + let probe = Allowance::limited(64 * 1024 * 1024); + let collector = OptimalCollector::with_allowance(&data, start, options, &probe).unwrap(); + let finder_bytes = probe.used(); + drop(collector); + assert_eq!(probe.used(), 0); + + let span = data.len() - start; + let run_bytes = span * std::mem::size_of::<(u32, u32)>(); + let initial_bytes = run_bytes + (span + 1) * std::mem::size_of::(); + for limit in [ + finder_bytes + run_bytes as u64 - 1, + finder_bytes + initial_bytes as u64 - 1, + finder_bytes + initial_bytes as u64, + ] { + let allowance = Allowance::limited(limit); + let mut collector = + OptimalCollector::with_allowance(&data, start, options, &allowance).unwrap(); + assert!(matches!( + collector.collect(&data, start..data.len(), options), + Err(Error::WorkspaceLimitExceeded(_)) + )); + drop(collector); + assert_eq!(allowance.used(), 0); + } + } + + let input = b"state history must be admitted after its packed output"; + let options = EncodeOptions::new(0).with_max_match_distance(32); + let mut limit = 0u64; + let mut refusals = 0; + loop { + let allowance = Allowance::limited(limit); + let mut state = EncoderState::new(options, &allowance); + match state.encode(input, 0, None, None) { + Err(Error::WorkspaceLimitExceeded(details)) => { + refusals += 1; + let next = details.used + details.required; + assert!(next > limit); + limit = next; + } + Ok(packed) => { + assert_eq!(&*state.history, &input[input.len() - 32..]); + drop((packed, state)); + assert_eq!(allowance.used(), 0); + break; + } + Err(error) => panic!("unexpected state error: {error:?}"), + } + } + assert!(refusals > 1); +} + +#[test] +fn token_block_allocation_refusals_cover_each_control_form() { + let filter = EncodeToken::Filter(EncodeFilter { + offset: 0, + length: 1, + filter_type: FilterType::E8, + channels: 0, + }); + let mut filters = vec![filter; 256]; + filters.push(EncodeToken::Literal(b'A')); + + let mut last_length = vec![ + EncodeToken::Literal(b'A'), + EncodeToken::Match { + length: 2, + distance: 1, + }, + ]; + last_length.extend(std::iter::repeat_n( + EncodeToken::Match { + length: 2, + distance: 1, + }, + 256, + )); + + let mut repeat_distance = last_length[..2].to_vec(); + repeat_distance.extend((0..256).map(|index| EncodeToken::Match { + length: 3 + index % 2, + distance: 1, + })); + + let mut new_distance = vec![EncodeToken::Literal(b'A'); 100]; + new_distance.extend((0..256).map(|index| EncodeToken::Match { + length: 10, + distance: 70 + index % 5, + })); + + for (tokens, output_size) in [ + (filters, 1), + (last_length, 1 + 2 * 257), + (repeat_distance, 3 + 128 * (3 + 4)), + (new_distance, 100 + 256 * 10), + ] { + let packed = encode_token_block(&tokens, 0, DISTANCE_TABLE_SIZE_50, true).unwrap(); + assert_eq!( + decode_lz(&packed, 0, output_size).unwrap(), + vec![b'A'; output_size] + ); + assert_each_allocation_refusal(|budget| { + encode_token_block_with_allowance(&tokens, 0, DISTANCE_TABLE_SIZE_50, true, budget) + }); + } +} + +#[test] +fn filter_writer_allocation_refusals_cover_filter_type_bits() { + for filter_type in [ + FilterType::Delta, + FilterType::E8, + FilterType::E8E9, + FilterType::Arm, + ] { + let filter = EncodeFilter { + offset: 0, + length: 1, + filter_type, + channels: 1, + }; + for prefix_bits in [9, 12] { + let mut expected = BitWriter::new(); + expected.write_bits(0, prefix_bits); + try_write_filter(&mut expected, filter).unwrap(); + let expected = expected.finish(); + + let baseline = RefusingBudget::new(usize::MAX); + let mut writer = BitWriter::with_allowance(&baseline); + writer.try_write_bits(0, prefix_bits).unwrap(); + try_write_filter(&mut writer, filter).unwrap(); + assert_eq!(&*writer.bytes, expected); + drop(writer); + assert_eq!(baseline.used(), 0); + + assert_each_allocation_refusal(|budget| { + let mut writer = BitWriter::with_allowance(budget); + writer.try_write_bits(0, prefix_bits)?; + try_write_filter(&mut writer, filter)?; + Ok(writer.bytes) + }); + } + } +} + +#[test] +fn table_level_writer_refusals_release_bit_storage() { + let mut isolated_zeroes = [1u8; LEVEL_TABLE_SIZE]; + for index in (1..LEVEL_TABLE_SIZE).step_by(2) { + isolated_zeroes[index] = 0; + } + let mut offset_zero_runs = [1u8; LEVEL_TABLE_SIZE]; + for start in (1..LEVEL_TABLE_SIZE).step_by(4) { + offset_zero_runs[start..(start + 3).min(LEVEL_TABLE_SIZE)].fill(0); + } + for lengths in [ + [0u8; LEVEL_TABLE_SIZE], + isolated_zeroes, + offset_zero_runs, + [15u8; LEVEL_TABLE_SIZE], + ] { + for prefix_bits in 0..=16 { + let mut expected = BitWriter::new(); + expected.write_bits(0, prefix_bits); + try_write_level_lengths(&mut expected, &lengths).unwrap(); + let expected = expected.finish(); + + let baseline = RefusingBudget::new(usize::MAX); + let mut writer = BitWriter::with_allowance(&baseline); + writer.try_write_bits(0, prefix_bits).unwrap(); + try_write_level_lengths(&mut writer, &lengths).unwrap(); + assert_eq!(&*writer.bytes, expected); + drop(writer); + assert_eq!(baseline.used(), 0); + + assert_each_allocation_refusal(|budget| { + let mut writer = BitWriter::with_allowance(budget); + writer.try_write_bits(0, prefix_bits)?; + try_write_level_lengths(&mut writer, &lengths)?; + Ok(writer.bytes) + }); + } + } +} + +#[test] +fn table_run_refusals_release_token_storage() { + for count in [3, 10, 11, 138, 139, 1000] { + assert_each_allocation_refusal(|budget| { + let mut tokens = Buffer::new(budget); + emit_repeat_level_run(&mut tokens, count)?; + Ok(tokens) + }); + } + for count in [1, 3, 10, 11, 139, 1000] { + assert_each_allocation_refusal(|budget| { + let mut tokens = Buffer::new(budget); + emit_zero_level_run(&mut tokens, count)?; + Ok(tokens) + }); + } +} + +#[test] +fn filter_preparation_rejects_legacy_only_kinds() { + let data = b"plain bytes"; + let filters = [crate::FilterSpec::whole(crate::FilterKind::Itanium)]; + let expected = Error::InvalidData("RAR 5 has no builtin type for this filter"); + assert_eq!( + normalized_filter_specs(data.len(), &filters, &Allowance::default()), + Err(expected.clone()) + ); + assert_eq!( + filtered_member_with_allowance(data, &filters, &Allowance::default()), + Err(expected) + ); +} + +#[test] +fn parser_allowance_preserves_tokens_and_retains_the_returned_owner() { + let data: Vec<_> = (0..8192u32) + .map(|n| (n.wrapping_mul(71) ^ (n >> 4)) as u8) + .collect(); + for optimal in [false, true] { + let options = EncodeOptions::new(32) + .with_max_match_distance(65536) + .with_optimal_parse(optimal); + let expected = encode_tokens_with_progress( + &data, + 0..data.len(), + MemberSearch::Fresh, + options, + DISTANCE_TABLE_SIZE_50, + &[], + None, + ) + .unwrap(); + let allowance = Allowance::limited(16 * 1024 * 1024); + for _ in 0..3 { + let tokens = encode_tokens_with_allowance( + &data, + 0..data.len(), + MemberSearch::Fresh, + options, + DISTANCE_TABLE_SIZE_50, + &[], + None, + &allowance, + ) + .unwrap(); + assert_eq!(tokens, expected); + let retained = allowance.used(); + assert!(retained >= (tokens.len() * std::mem::size_of::()) as u64); + assert!(retained > 0); + assert!(matches!( + Buffer::::with_capacity( + (16 * 1024 * 1024 - retained + 1) as usize, + &allowance + ), + Err(Error::WorkspaceLimitExceeded(_)) + )); + drop(tokens); + assert_eq!(allowance.used(), 0); + } + } +} + +#[test] +fn parser_refusal_and_cancellation_release_finders_matches_and_parse_arrays() { + let data: Vec<_> = (0..16384u32) + .map(|n| (n.wrapping_mul(73) ^ (n >> 3)) as u8) + .collect(); + for optimal in [false, true] { + let options = EncodeOptions::new(32) + .with_max_match_distance(65536) + .with_optimal_parse(optimal); + for limit in [0, 524288] { + let allowance = Allowance::limited(limit); + let result = encode_tokens_with_allowance( + &data, + 0..data.len(), + MemberSearch::Fresh, + options, + DISTANCE_TABLE_SIZE_50, + &[], + None, + &allowance, + ); + assert!(matches!(result, Err(Error::WorkspaceLimitExceeded(_)))); + assert_eq!(allowance.used(), 0); + } + if optimal { + // Admit the tree itself, then refuse match/parse workspace. + let allowance = Allowance::limited(1_200_000); + assert!( + matches!(encode_tokens_with_allowance(&data, 0..data.len(), MemberSearch::Fresh, + options, DISTANCE_TABLE_SIZE_50, &[], None, &allowance), + Err(Error::WorkspaceLimitExceeded(details)) if details.used > 1_000_000) + ); + assert_eq!(allowance.used(), 0); + } + let allowance = Allowance::limited(16 * 1024 * 1024); + assert!(matches!( + encode_tokens_with_allowance( + &data, + 0..data.len(), + MemberSearch::Fresh, + options, + DISTANCE_TABLE_SIZE_50, + &[], + Some(&mut |_| false), + &allowance + ), + Err(Error::Cancelled) + )); + assert_eq!(allowance.used(), 0); + } +} + +#[test] +fn lazy_parser_honors_cancellation_at_final_progress_report() { + let allowance = Allowance::default(); + let data = b"AB"; + let mut reports = 0; + let result = encode_tokens_with_allowance( + data, + 0..data.len(), + MemberSearch::Fresh, + EncodeOptions::new(0), + DISTANCE_TABLE_SIZE_50, + &[], + Some(&mut |_| { + reports += 1; + reports == 1 + }), + &allowance, + ); + + assert_eq!(result, Err(Error::Cancelled)); + assert_eq!(reports, 2); +} + +#[test] +fn configured_x86_scanning_matches_default_across_poll_boundaries() { + let mut input = vec![0; 192 * 1024]; + for pos in [65530, 65535, 65541, 131070, 131080] { + input[pos] = 0xe9; + input[pos + 1..pos + 5].copy_from_slice(&123456u32.to_le_bytes()); + } + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + let mut expected = input.clone(); + e8e9_decode(&mut expected, 4096, true); + e8e9_decode_with_control(&mut input, 4096, true, &control).unwrap(); + assert_eq!(input, expected); +} + +#[test] +fn e8_decoder_adjusts_wrapped_negative_addresses() { + let mut data = [0xe8, 0xff, 0xff, 0xff, 0xff]; + + e8e9_decode(&mut data, 0, false); + + assert_eq!( + u32::from_le_bytes(data[1..5].try_into().unwrap()), + 0x00ff_ffff + ); +} + +#[test] +fn cancellation_interrupts_buffered_decode_and_filters() { + let data = b"cancellable RAR5 symbols ".repeat(16384); + let packed = encode_literal_only(&data, 0).unwrap(); + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack50Decoder::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + decoder.read_control.cancel_after_checks(16); + assert_eq!( + decoder + .decode_member(&packed, 0, data.len(), false, DecodeMode::Lz) + .unwrap_err(), + Error::Cancelled + ); + for arm in [false, true] { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(2); + let mut bytes = vec![0; 384 * 1024]; + let result = if arm { + arm_decode_with_control(&mut bytes, 0, &control) + } else { + e8e9_decode_with_control(&mut bytes, 0, true, &control) + }; + assert_eq!(result.unwrap_err(), Error::Cancelled); + } +} + +#[test] +fn delta_filter_cancellation_leaves_output_unchanged() { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(2); + let mut bytes = vec![0x5a; 384 * 1024]; + let filter = PendingFilter { + start: 0, + length: bytes.len(), + filter_type: FilterType::Delta, + channels: 3, + }; + + assert_eq!( + apply_filter_data(&mut bytes, &filter, &control), + Err(Error::Cancelled) + ); + assert!(bytes.iter().all(|&byte| byte == 0x5a)); +} + +#[test] +fn filter_application_propagates_range_and_cancellation_errors() { + let original = [0xe8, 0, 0, 0, 0]; + let control = crate::read_control::ReadControl::default(); + for (start, length, message) in [ + (usize::MAX, 2, "RAR 5 filter range overflows"), + (4, 2, "RAR 5 filter range exceeds output"), + ] { + let mut output = original; + let filter = PendingFilter { + start, + length, + filter_type: FilterType::E8, + channels: 0, + }; + assert_eq!( + apply_filters_with_control(&mut output, &[filter], &control), + Err(Error::InvalidData(message)) + ); + assert_eq!(output, original); + } + + for kind in [FilterType::E8, FilterType::E8E9, FilterType::Arm] { + for checks in [0, 1, 2] { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(checks); + let mut output = original; + let filter = PendingFilter { + start: 0, + length: output.len(), + filter_type: kind, + channels: 0, + }; + assert_eq!( + apply_filters_with_control(&mut output, &[filter], &control), + Err(Error::Cancelled), + "kind={kind:?}, checks={checks}" + ); + assert_eq!(output, original); + } + } +} + +#[test] +fn repricing_keeps_the_smallest_actual_block_including_filters() { + for filtered in [false, true] { + let data = wordy_text(4096); + let filters = if filtered { + vec![EncodeFilter { + offset: 0, + length: data.len(), + filter_type: FilterType::E8, + channels: 0, + }] + } else { + vec![] + }; + let options = EncodeOptions::new(32).with_optimal_parse(true); + let distances = DISTANCE_TABLE_SIZE_50; + let selected = encode_tokens_with_progress( + &data, + 0..data.len(), + MemberSearch::Fresh, + options, + distances, + &filters, + None, + ) + .unwrap(); + let lengths = table_lengths_with_filters(&selected, &filters, distances).unwrap(); + let selected_bits = token_stream_bits(&selected, &filters, &lengths, distances).unwrap(); + let all: Vec<_> = filters + .iter() + .copied() + .map(EncodeToken::Filter) + .chain(selected.iter().copied()) + .collect(); + let packed = encode_token_block(&all, 0, distances, true).unwrap(); + let parsed = parse_compressed_block(&packed).unwrap(); + assert_eq!(selected_bits, parsed.header.payload_bits); + let mut collector = OptimalCollector::new(&data, 0, options); + let matches = collector.collect(&data, 0..data.len(), options).unwrap(); + let mut pass = + optimal_tokens(&data, 0..data.len(), options, distances, None, &matches).unwrap(); + for _ in 0..OPTIMAL_PARSE_PASSES { + let lengths = table_lengths_with_filters(&pass, &filters, distances).unwrap(); + assert!( + selected_bits <= token_stream_bits(&pass, &filters, &lengths, distances).unwrap() + ); + pass = optimal_tokens( + &data, + 0..data.len(), + options, + distances, + Some(&TokenPrices { + lengths: lengths.slices(), + }), + &matches, + ) + .unwrap(); + } + } +} + +#[test] +fn shorter_equal_price_match_can_expose_a_better_continuation() { + let history = b"abcdefghijklm!mnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let data = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let mut combined = history.to_vec(); + combined.extend_from_slice(data); + let mut matches = BlockMatches { + runs: Buffer::new(&Allowance::default()), + starts: Buffer::new(&Allowance::default()), + }; + for pos in 0..data.len() { + matches.starts.push(matches.runs.len() as u32).unwrap(); + if pos == 0 { + matches.runs.push((13, history.len() as u32)).unwrap(); + } + if pos == 12 { + matches + .runs + .push((40, (history.len() + 12 - 14) as u32)) + .unwrap(); + } + } + matches.starts.push(matches.runs.len() as u32).unwrap(); + let tokens = optimal_tokens( + &combined, + history.len()..combined.len(), + EncodeOptions::new(32), + DISTANCE_TABLE_SIZE_50, + None, + &matches, + ) + .unwrap(); + assert_eq!(tokens.len(), 2); + assert!(matches!(tokens[0], EncodeToken::Match { length: 12, .. })); + assert!(matches!(tokens[1], EncodeToken::Match { length: 40, .. })); +} + +#[test] +fn shared_streaming_finder_preserves_separate_block_bytes() { + // Short blocks exercise seam hashes and wrap the dictionary repeatedly. + let data: Vec = (0..8192).map(|i| ((i * 17 + i / 31) % 251) as u8).collect(); + for optimal in [false, true] { + let options = EncodeOptions::new(32) + .with_optimal_parse(optimal) + .with_max_match_distance(128); + let blocks: Vec<_> = (1..=32).map(|i| (i * 256, i % 4 == 0)).collect(); + let grouped = encode_lz_streaming_blocks(&data, &[], &blocks, 0, options, None).unwrap(); + let mut start = 0; + for ((end, last), bytes) in blocks.into_iter().zip(grouped) { + let reference = + encode_lz_streaming_block(&data[start..end], &data[..start], 0, options, last) + .unwrap(); + assert_eq!(bytes, reference, "optimal={optimal}, start={start}"); + start = end; + } + } +} + +#[test] +fn optimal_streaming_blocks_allow_zero_match_distance() { + let data: Vec = (0..512).map(|i| (i * 71) as u8).collect(); + let options = EncodeOptions::new(16) + .with_optimal_parse(true) + .with_max_match_distance(0); + let blocks = + encode_lz_streaming_blocks(&data, &[], &[(256, false), (512, true)], 0, options, None) + .unwrap(); + let packed: Vec = blocks.into_iter().flatten().collect(); + assert_eq!(decode_lz(&packed, 0, data.len()).unwrap(), data); +} +use super::*; + +/// The flat code charged four bits for a symbol used once and four for one +/// used two hundred times, over an alphabet that is nothing like even. +#[test] +fn the_level_code_spends_fewer_bits_on_the_common_symbol() { + let mut tokens = vec![LevelToken::plain(0); 200]; + for symbol in [4, 7, 9, 11] { + tokens.push(LevelToken::plain(symbol)); + } + let lengths = level_code_lengths_for_tokens(&tokens); + assert!( + lengths[0] < lengths[7], + "the symbol used 200 times costs {} bits and one used once costs {}", + lengths[0], + lengths[7] + ); +} + +/// Strict decoders rebuild the pre-table and reject one that does not fill +/// its code space, so whichever coding wins has to satisfy Kraft equality. +#[test] +fn every_level_code_fills_its_code_space() { + let shapes: Vec> = vec![ + vec![LevelToken::plain(3); 8], + (0..20).map(LevelToken::plain).collect(), + { + let mut skewed = vec![LevelToken::plain(0); 900]; + skewed.extend((1..20).map(LevelToken::plain)); + skewed + }, + (0..20) + .flat_map(|symbol| std::iter::repeat_n(LevelToken::plain(symbol), 1 << symbol.min(9))) + .collect(), + ]; + for tokens in shapes { + let lengths = level_code_lengths_for_tokens(&tokens); + let used: Vec = lengths.iter().copied().filter(|&len| len != 0).collect(); + let kraft: f64 = used.iter().map(|&len| 0.5f64.powi(i32::from(len))).sum(); + assert!( + (kraft - 1.0).abs() < 1e-9, + "code over {} symbols fills {kraft} of its space: {lengths:?}", + used.len() + ); + assert!( + used.iter().all(|&len| len <= 15), + "{lengths:?} exceeds four bits" + ); + assert!(HuffmanTable::from_lengths(&lengths).is_ok()); + } +} + +/// The cost model has to agree with the writer, or it picks the wrong +/// coding whenever the two disagree about the table header. +#[test] +fn the_level_code_cost_matches_what_the_writer_emits() { + let cases: Vec<[u8; LEVEL_TABLE_SIZE]> = vec![ + [4; LEVEL_TABLE_SIZE], + { + let mut lengths = [0u8; LEVEL_TABLE_SIZE]; + lengths[0] = 1; + lengths[1] = 2; + lengths[19] = 15; + lengths + }, + { + let mut lengths = [0u8; LEVEL_TABLE_SIZE]; + lengths[2] = 15; + lengths[3] = 15; + lengths[9] = 3; + lengths + }, + ]; + for lengths in cases { + let mut writer = BitWriter::new(); + write_level_lengths(&mut writer, &lengths); + let header = level_code_cost(&lengths, &[0; LEVEL_TABLE_SIZE]); + assert_eq!( + header, writer.bit_pos, + "header cost {header} against {} bits written for {lengths:?}", + writer.bit_pos + ); + } +} + +fn encode_tokens( + input: &[u8], + history: &[u8], + options: EncodeOptions, + distance_size: usize, +) -> Vec { + let (combined, start) = + member_window_with_allowance(input, history, options, &Allowance::default()).unwrap(); + encode_tokens_with_progress( + &combined, + start..combined.len(), + MemberSearch::Fresh, + options, + distance_size, + &[], + None, + ) + .expect("encoding without cancellation cannot be cancelled") + .into_vec() +} + +fn optimal_tokens( + combined: &[u8], + block: std::ops::Range, + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, + matches: &BlockMatches, +) -> Result> { + optimal_tokens_in_workspace( + combined, + block, + options, + distance_size, + prices, + matches, + &mut OptimalWorkspace::new(&Allowance::default()), + ) + .map(Buffer::into_vec) +} + +/// One block through the optimal parse, collecting its matches first the +/// way [`encode_tokens_with_progress`] does. +fn collected_optimal_tokens( + data: &[u8], + options: EncodeOptions, + distance_size: usize, + prices: Option<&TokenPrices<'_>>, +) -> Vec { + let mut collector = OptimalCollector::new(data, 0, options); + let matches = collector.collect(data, 0..data.len(), options).unwrap(); + optimal_tokens( + data, + 0..data.len(), + options, + distance_size, + prices, + &matches, + ) + .unwrap() +} +fn should_lazy_emit_literal( + input: &[u8], + pos: usize, + finder: &Rar50MatchFinder, + options: EncodeOptions, + state: &EncoderMatchState, + distance_size: usize, + current: MatchCandidate, +) -> bool { + lazy_match_decision( + input, + pos, + input.len(), + finder, + options, + state, + distance_size, + current, + ) + .0 +} + +fn checksum(flags: u8, size_bytes: &[u8]) -> u8 { + size_bytes + .iter() + .fold(0x5a ^ flags, |acc, &byte| acc ^ byte) +} + +#[test] +fn parses_one_byte_size_block_header() { + let flags = 0xc7; + let size = [3]; + let input = [flags, checksum(flags, &size), size[0], 0xaa, 0xbb, 0xcc]; + + let block = parse_compressed_block(&input).unwrap(); + assert_eq!(block.header_len, 3); + assert_eq!(block.payload, 3..6); + assert_eq!(block.header.flags, flags); + assert!(block.header.is_last); + assert!(block.header.has_tables); + assert_eq!(block.header.final_byte_bits, 8); + assert_eq!(block.header.payload_size, 3); + assert_eq!(block.header.payload_bits, 24); +} + +#[test] +fn parses_three_byte_size_block_header_with_partial_final_byte() { + let flags = 0x94; + let size = [0x34, 0x12, 0x00]; + let mut input = vec![flags, checksum(flags, &size), size[0], size[1], size[2]]; + input.resize(0x1234 + 5, 0); + + let block = parse_compressed_block(&input).unwrap(); + assert_eq!(block.header_len, 5); + assert_eq!(block.payload, 5..0x1239); + assert!(!block.header.is_last); + assert!(block.header.has_tables); + assert_eq!(block.header.final_byte_bits, 5); + assert_eq!(block.header.payload_size, 0x1234); + assert_eq!(block.header.payload_bits, (0x1234 - 1) * 8 + 5); +} + +#[test] +fn rejects_reserved_size_length_selector() { + let input = [0x18, 0x42, 0x00]; + + assert_eq!( + parse_compressed_block(&input), + Err(Error::InvalidData("RAR 5 block size length is invalid")) + ); +} + +#[test] +fn rejects_bad_block_header_checksum() { + let input = [0xc7, 0x00, 0x03, 0xaa, 0xbb, 0xcc]; + + assert_eq!( + parse_compressed_block(&input), + Err(Error::InvalidData("RAR 5 block header checksum mismatch")) + ); +} + +#[test] +fn rejects_truncated_block_payload() { + let flags = 0xc7; + let size = [3]; + let input = [flags, checksum(flags, &size), size[0], 0xaa, 0xbb]; + + assert_eq!(parse_compressed_block(&input), Err(Error::NeedMoreInput)); +} + +#[test] +fn reads_level_lengths_with_literal_fifteen() { + let mut nibbles = vec![1, 2, 15, 0, 3, 4]; + nibbles.resize(LEVEL_TABLE_SIZE + 1, 0); + + let (lengths, bits) = read_level_lengths(&pack_nibbles(&nibbles)).unwrap(); + + assert_eq!(&lengths[..6], &[1, 2, 15, 3, 4, 0]); + assert_eq!(bits, LEVEL_TABLE_SIZE * 4 + 4); +} + +#[test] +fn reads_level_lengths_with_zero_run_at_current_position() { + let mut nibbles = vec![7, 15, 3, 2]; + nibbles.resize(LEVEL_TABLE_SIZE - 3, 0); + + let (lengths, bits) = read_level_lengths(&pack_nibbles(&nibbles)).unwrap(); + + assert_eq!(lengths[0], 7); + assert_eq!(&lengths[1..6], &[0, 0, 0, 0, 0]); + assert_eq!(lengths[6], 2); + assert_eq!(bits, (LEVEL_TABLE_SIZE - 3) * 4); +} + +#[test] +fn level_zero_run_stops_at_level_table_boundary() { + let mut nibbles = vec![0; LEVEL_TABLE_SIZE - 1]; + nibbles.extend([15, 15]); // a seventeen-entry zero run at the last slot + let (lengths, bits) = read_level_lengths(&pack_nibbles(&nibbles)).unwrap(); + assert_eq!(lengths, [0; LEVEL_TABLE_SIZE]); + assert_eq!(bits, (LEVEL_TABLE_SIZE + 1) * 4); +} + +fn pack_nibbles(nibbles: &[u8]) -> Vec { + nibbles + .chunks(2) + .map(|chunk| { + let high = chunk[0] & 0x0f; + let low = chunk.get(1).copied().unwrap_or(0) & 0x0f; + (high << 4) | low + }) + .collect() +} + +#[test] +fn reads_rar50_second_level_table_lengths() { + let mut writer = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + writer.write_bits(5, 4); + } + for count in [138, 138, 138, 16] { + writer.write_bits(19, 5); + writer.write_bits(count - 11, 7); + } + let input = writer.finish(); + + let (lengths, bits) = read_table_lengths(&input, 0).unwrap(); + + assert_eq!(lengths.main.len(), MAIN_TABLE_SIZE); + assert_eq!(lengths.distance.len(), DISTANCE_TABLE_SIZE_50); + assert_eq!(lengths.align.len(), ALIGN_TABLE_SIZE); + assert_eq!(lengths.length.len(), LENGTH_TABLE_SIZE); + assert!(lengths.main.iter().all(|&length| length == 0)); + assert!(lengths.distance.iter().all(|&length| length == 0)); + assert!(lengths.align.iter().all(|&length| length == 0)); + assert!(lengths.length.iter().all(|&length| length == 0)); + assert_eq!(bits, LEVEL_TABLE_SIZE * 4 + 4 * (5 + 7)); +} + +#[test] +fn table_length_runs_stop_at_the_table_boundary() { + // The reference decoder clips an otherwise valid run to the table + // length. Both zero runs and previous-length runs can cross it. + for (first_symbol, run_symbol, expected) in [(19, 19, 0), (5, 17, 5)] { + let mut writer = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + writer.write_bits(5, 4); + } + if first_symbol != run_symbol { + writer.write_bits(first_symbol, 5); + } + for _ in 0..4 { + writer.write_bits(run_symbol, 5); + writer.write_bits(127, 7); // a 138-entry run + } + + let (lengths, _) = read_table_lengths(&writer.finish(), 0).unwrap(); + assert!(lengths.main.iter().all(|&length| length == expected)); + assert!(lengths.distance.iter().all(|&length| length == expected)); + assert!(lengths.align.iter().all(|&length| length == expected)); + assert!(lengths.length.iter().all(|&length| length == expected)); + } +} + +#[test] +fn truncated_table_run_extras_need_more_input() { + // The last run symbol ends with fewer padding bits than its extra + // field needs. Exercise both repeat-previous and zero runs at each + // extra-field width. + for (prefix, run) in [ + (&[][..], 19), + (&[1, 1][..], 18), + (&[1][..], 17), + (&[1, 1][..], 16), + ] { + let mut writer = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + writer.write_bits(5, 4); + } + for &symbol in prefix { + writer.write_bits(symbol, 5); + } + writer.write_bits(run, 5); + assert_eq!( + read_table_lengths(&writer.finish(), 0), + Err(Error::NeedMoreInput), + "run symbol {run}" + ); + } +} + +#[test] +fn malformed_table_prefixes_fail_before_table_symbols() { + assert_eq!(read_table_lengths(&[], 0), Err(Error::NeedMoreInput)); + + let mut invalid = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + invalid.write_bits(1, 4); + } + assert!(matches!( + read_table_lengths(&invalid.finish(), 0), + Err(Error::InvalidData(_)) + )); + + let mut no_symbols = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + no_symbols.write_bits(5, 4); + } + assert_eq!( + read_table_lengths(&no_symbols.finish(), 0), + Err(Error::NeedMoreInput) + ); +} + +#[test] +fn truncated_level_escape_needs_its_count_nibble() { + // The low nibble is an escape; its following count nibble is absent. + assert_eq!(read_level_lengths(&[0x0f]), Err(Error::NeedMoreInput)); +} + +#[test] +fn decoders_reject_oversubscribed_secondary_huffman_tables() { + for invalid_table in ["distance", "align", "length"] { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + let invalid_lengths = match invalid_table { + "distance" => &mut lengths.distance, + "align" => &mut lengths.align, + "length" => &mut lengths.length, + _ => unreachable!(), + }; + invalid_lengths[..3].fill(1); + let (payload, payload_bits) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let input = encode_compressed_block(&payload, payload_bits, true, true).unwrap(); + let expected = Error::InvalidData("RAR 5 oversubscribed Huffman table"); + + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(expected.clone()), + "{invalid_table}" + ); + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!( + matches!(result, Err(StreamDecodeError::Decode(error)) if error == expected), + "{invalid_table}" + ); + } +} + +#[test] +fn reads_rar70_table_length_count() { + assert_eq!( + table_length_count(1).unwrap(), + MAIN_TABLE_SIZE + DISTANCE_TABLE_SIZE_70 + ALIGN_TABLE_SIZE + LENGTH_TABLE_SIZE + ); +} + +#[test] +fn table_reader_rejects_unknown_algorithm_before_decoding() { + let expected = Error::InvalidData("RAR 5 unknown compression algorithm version"); + assert_eq!(table_length_count(2).unwrap_err(), expected); + assert_eq!(read_table_lengths(&[], 2).unwrap_err(), expected); +} + +#[test] +fn encoded_table_lengths_round_trip_with_bit_count() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[b'B' as usize] = 3; + lengths.main[262] = 3; + lengths.distance[1] = 1; + lengths.align[0] = 4; + lengths.length[0] = 1; + + let (encoded, bit_count) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let (decoded, decoded_bits) = read_table_lengths(&encoded, 0).unwrap(); + + assert_eq!(decoded, lengths); + assert_eq!(decoded_bits, bit_count); +} + +#[test] +fn table_encoder_validates_public_inputs() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[b'B' as usize] = 1; + let encoded = encode_table_lengths(&lengths, 0).unwrap(); + assert_eq!(read_table_lengths(&encoded, 0).unwrap().0, lengths); + + assert_eq!( + encode_table_lengths(&lengths, 2), + Err(Error::InvalidData( + "RAR 5 unknown compression algorithm version" + )) + ); + lengths.distance.pop(); + assert_eq!( + encode_table_lengths(&lengths, 0), + Err(Error::InvalidData("RAR 5 table length count mismatch")) + ); + lengths.distance.push(0); + lengths.main[0] = 16; + assert_eq!( + encode_table_lengths(&lengths, 0), + Err(Error::InvalidData("RAR 5 Huffman length is too large")) + ); +} + +#[test] +fn member_encoders_reject_unknown_algorithm_version() { + let expected = Error::InvalidData("RAR 5 unknown compression algorithm version"); + assert_eq!(encode_literal_only(b"AB", 2).unwrap_err(), expected); + assert_eq!(encode_lz_member(b"ABABABAB", 2).unwrap_err(), expected); +} + +#[test] +fn table_level_encoder_uses_rar5_run_symbols() { + let mut lengths = + vec![0u8; MAIN_TABLE_SIZE + DISTANCE_TABLE_SIZE_50 + ALIGN_TABLE_SIZE + LENGTH_TABLE_SIZE]; + lengths[..4].fill(6); + lengths[8..21].fill(0); + + let tokens = encode_table_level_tokens(&lengths); + + assert!(tokens.contains(&LevelToken::repeat_previous_short(3))); + assert!(tokens.iter().any(|token| token.symbol == 19)); +} + +#[test] +fn repeat_level_runs_cover_the_whole_requested_length() { + for count in 3..=1024 { + let mut tokens = Buffer::new(&Allowance::default()); + emit_repeat_level_run(&mut tokens, count).unwrap(); + let emitted: usize = tokens + .iter() + .map(|token| match token.symbol { + 16 => usize::from(token.extra_value) + 3, + 17 => usize::from(token.extra_value) + 11, + other => panic!("unexpected repeat symbol {other}"), + }) + .sum(); + assert_eq!(emitted, count, "repeat run of {count}"); + } +} + +#[test] +fn zero_level_runs_cover_the_whole_requested_length() { + for count in 1..=1024 { + let mut tokens = Buffer::new(&Allowance::default()); + emit_zero_level_run(&mut tokens, count).unwrap(); + let emitted: usize = tokens + .iter() + .map(|token| match token.symbol { + 0 => 1, + 18 => usize::from(token.extra_value) + 3, + 19 => usize::from(token.extra_value) + 11, + other => panic!("unexpected zero-run symbol {other}"), + }) + .sum(); + assert_eq!(emitted, count, "zero run of {count}"); + } +} + +#[test] +fn encoded_compressed_block_round_trips_header_fields() { + let payload = [0xaa, 0xbb, 0xc0]; + let block = encode_compressed_block(&payload, 18, true, true).unwrap(); + + let parsed = parse_compressed_block(&block).unwrap(); + + assert_eq!(parsed.payload, 3..6); + assert!(parsed.header.has_tables); + assert!(parsed.header.is_last); + assert_eq!(parsed.header.final_byte_bits, 2); + assert_eq!(parsed.header.payload_bits, 18); + assert_eq!(&block[parsed.payload], payload); +} + +#[test] +fn rejects_table_repeat_without_previous_length() { + let mut writer = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + writer.write_bits(5, 4); + } + writer.write_bits(16, 5); + writer.write_bits(0, 3); + + assert_eq!( + read_table_lengths(&writer.finish(), 0), + Err(Error::InvalidData( + "RAR 5 table repeats missing previous length" + )) + ); +} + +#[test] +fn rejects_invalid_encoded_block_bit_counts() { + assert_eq!( + encode_compressed_block(&[0], 0, true, true), + Err(Error::InvalidData("RAR 5 block has unused payload bytes")) + ); + assert_eq!( + encode_compressed_block(&[], 1, true, true), + Err(Error::InvalidData("RAR 5 block bit count exceeds payload")) + ); + let oversized = vec![0; 0x0100_0000]; + assert_eq!( + encode_compressed_block(&oversized, oversized.len() * 8, false, true), + Err(Error::InvalidData("RAR 5 block payload is too large")) + ); +} + +#[test] +fn builds_named_decode_tables_from_lengths() { + let lengths = TableLengths { + main: vec![1, 1], + distance: vec![1, 1], + align: vec![4; ALIGN_TABLE_SIZE], + length: vec![1, 1], + }; + + let tables = DecodeTables::from_lengths(&lengths).unwrap(); + + assert!(!tables.main.is_empty()); + assert!(!tables.distance.is_empty()); + assert!(!tables.align.is_empty()); + assert!(!tables.length.is_empty()); + assert!(!tables.align_mode); + let copied = tables.clone(); + drop(tables); + for table in [ + &copied.main, + &copied.distance, + &copied.align, + &copied.length, + ] { + assert!(!table.is_empty()); + assert_eq!(table.decode(&mut BitReader::new(&[0])).unwrap(), 0); + } + assert!(!copied.align_mode); +} + +#[test] +fn rejects_oversubscribed_rar50_huffman_tables() { + assert!(matches!( + HuffmanTable::from_lengths(&[1, 1, 1]), + Err(Error::InvalidData("RAR 5 oversubscribed Huffman table")) + )); + assert!(matches!( + HuffmanTable::from_lengths(&[16]), + Err(Error::InvalidData("RAR 5 Huffman length is too large")) + )); +} + +#[test] +fn decoders_reject_empty_and_invalid_main_huffman_codes() { + for (main_length, code, code_bits, message) in [ + (0, 1, 1, "RAR 5 empty Huffman table"), + (2, 0xffff, 16, "RAR 5 invalid Huffman code"), + ] { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = main_length; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + writer.write_bits(code, code_bits); + let payload_bits = writer.bit_pos; + let input = encode_compressed_block(&writer.finish(), payload_bits, true, true).unwrap(); + + assert_eq!(decode_lz(&input, 0, 1), Err(Error::InvalidData(message))); + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::InvalidData(error))) if error == message + )); + } +} + +#[test] +fn detects_rar50_align_mode_when_align_lengths_are_not_uniform_four() { + let mut align = vec![4; ALIGN_TABLE_SIZE]; + align[0] = 0; + align[3] = 3; + let lengths = TableLengths { + main: vec![1, 1], + distance: vec![1, 1], + align, + length: vec![1, 1], + }; + + let tables = DecodeTables::from_lengths(&lengths).unwrap(); + + assert!(tables.align_mode); +} + +#[test] +fn decodes_synthetic_literal_only_block() { + let payload = literal_only_payload(b"ABBA"); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + + let output = decode_literal_only(&input, 0, 4).unwrap(); + + assert_eq!(output, b"ABBA"); +} + +#[test] +fn encodes_literal_only_member_that_decoder_reads() { + let data = b"literal-only RAR5 codec stream\nwith repeated words words words"; + let input = encode_literal_only(data, 0).unwrap(); + + let output = decode_literal_only(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); +} + +#[test] +fn encodes_literal_only_rar70_table_shape_that_decoder_reads() { + let data = b"small RAR7-compatible literal block"; + let input = encode_literal_only(data, 1).unwrap(); + + let output = decode_literal_only(&input, 1, data.len()).unwrap(); + + assert_eq!(output, data); +} + +#[test] +fn encodes_empty_literal_only_member() { + let input = encode_literal_only(b"", 0).unwrap(); + + let output = decode_literal_only(&input, 0, 0).unwrap(); + + assert!(output.is_empty()); +} + +#[test] +fn encodes_lz_member_with_same_member_matches() { + let data = b"RAR5 match writer phrase. RAR5 match writer phrase. RAR5 match writer phrase."; + let lz = encode_lz_member(data, 0).unwrap(); + let literal = encode_literal_only(data, 0).unwrap(); + + let output = decode_lz(&lz, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert!(lz.len() < literal.len()); + assert!( + encode_tokens(data, &[], EncodeOptions::default(), DISTANCE_TABLE_SIZE_50) + .iter() + .any(|token| matches!(token, EncodeToken::Match { .. })) + ); +} + +#[test] +fn frequency_weighted_huffman_lengths_shorten_common_symbols() { + let mut frequencies = vec![1usize; 24]; + frequencies[3] = 1024; + + let lengths = huffman::lengths_for_frequencies(&frequencies, 15); + + assert!(lengths[3] < lengths[0]); + assert!(lengths.iter().all(|&length| length <= 15)); +} + +#[test] +fn lz_encoder_uses_frequency_weighted_huffman_lengths() { + let mut data = vec![b'a'; 200]; + data.extend_from_slice(b"bcdefghijklmnopqrstuvwxyz"); + let input = encode_lz_member_with_options(&data, 0, EncodeOptions::new(0)).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert!(lengths.main[b'a' as usize] < lengths.main[b'z' as usize]); +} + +fn code_is_complete(lengths: &[u8]) -> bool { + let max_len = lengths.iter().copied().max().unwrap_or(0); + if max_len == 0 { + return true; + } + let sum: u64 = lengths + .iter() + .filter(|&&len| len != 0) + .map(|&len| 1u64 << (max_len - len)) + .sum(); + sum == (1u64 << max_len) +} + +#[test] +fn degenerate_inputs_emit_complete_huffman_tables() { + // Highly repetitive data collapses the distance/length/align tables to a + // single symbol. Those tables must still be transmitted as *complete* + // prefix codes, or strict RAR 5 decoders (7-Zip / WinRAR, which build + // with `Full_or_Empty`) reject the archive with a spurious data error. + // See issue #19. + let inputs: &[Vec] = &[ + vec![b'a'; 4000], + b"ab".repeat(4000), + (0u8..16).cycle().take(50_000).collect(), + b"lorem ipsum dolor sit amet ".repeat(2000), + ]; + for data in inputs { + let input = encode_lz_member_with_options(data, 0, EncodeOptions::new(0)).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + assert!(code_is_complete(&lengths.main), "main table incomplete"); + assert!( + code_is_complete(&lengths.distance), + "distance table incomplete" + ); + assert!(code_is_complete(&lengths.length), "length table incomplete"); + assert!(code_is_complete(&lengths.align), "align table incomplete"); + + assert_eq!(&decode_lz(&input, 0, data.len()).unwrap(), data); + } +} + +#[test] +fn lazy_lz_parser_defers_short_match_for_longer_next_match() { + let input = b"abcdXbcdYYYYYYYYYYYYabcdYYYYYYYYYYYY"; + let greedy = encode_tokens( + input, + &[], + EncodeOptions::new(MAX_MATCH_CANDIDATES), + DISTANCE_TABLE_SIZE_50, + ); + let lazy = encode_tokens( + input, + &[], + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_lazy_matching(true), + DISTANCE_TABLE_SIZE_50, + ); + let packed = encode_lz_member_with_options( + input, + 0, + EncodeOptions::new(MAX_MATCH_CANDIDATES).with_lazy_matching(true), + ) + .unwrap(); + + assert!(greedy + .iter() + .any(|token| matches!(token, EncodeToken::Match { length: 4, .. }))); + assert!(lazy + .iter() + .any(|token| matches!(token, EncodeToken::Match { length, .. } if *length > 8))); + assert_eq!(decode_lz(&packed, 0, input.len()).unwrap(), input); +} + +#[test] +fn match_selection_uses_length_then_distance_for_equal_scores() { + let state = EncoderMatchState::default(); + let distance_size = DISTANCE_TABLE_SIZE_50; + assert_eq!( + estimated_match_cost(&state, 5, 1, distance_size).unwrap(), + 10 + ); + assert_eq!( + estimated_match_cost(&state, 6, 262_144, distance_size).unwrap(), + 26 + ); + assert_eq!( + estimated_match_cost(&state, 6, 262_143, distance_size).unwrap(), + 26 + ); + + let mut best = None; + consider_match_candidate(&mut best, &state, distance_size, 5, 1); + consider_match_candidate(&mut best, &state, distance_size, 6, 262_144); + assert_eq!((best.unwrap().length, best.unwrap().distance), (6, 262_144)); + consider_match_candidate(&mut best, &state, distance_size, 6, 262_143); + assert_eq!((best.unwrap().length, best.unwrap().distance), (6, 262_143)); + + let mut best = None; + let longer = std::hint::black_box((6, 262_144)); + let shorter = std::hint::black_box((5, 1)); + consider_match_candidate(&mut best, &state, distance_size, longer.0, longer.1); + consider_match_candidate(&mut best, &state, distance_size, shorter.0, shorter.1); + assert_eq!((best.unwrap().length, best.unwrap().distance), (6, 262_144)); +} + +#[test] +fn cost_aware_match_selection_prefers_repeat_distance_token() { + let pos = 64; + let pattern = b"abcdefgh"; + let mut input: Vec = (0..96u8).map(|byte| byte.wrapping_mul(37)).collect(); + input[pos - 30..pos - 22].copy_from_slice(pattern); + input[pos - 10..pos - 2].copy_from_slice(pattern); + input[pos..pos + 8].copy_from_slice(pattern); + input[pos + 8] = b'X'; + + let mut finder = Rar50MatchFinder::new(input.len()); + for candidate in 0..pos { + finder.insert(&input, candidate); + } + let state = EncoderMatchState { + reps: [30, 0, 0, 0], + last_length: 8, + }; + + let best = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &state, + DISTANCE_TABLE_SIZE_50, + ) + .unwrap(); + + assert_eq!((best.length, best.distance), (8, 30)); +} + +#[test] +fn lazy_parser_uses_match_cost_not_only_match_length() { + let pos = 600; + let mut input: Vec = (0..700u16) + .map(|value| value.wrapping_mul(73) as u8) + .collect(); + input[pos - 512..pos - 504].copy_from_slice(b"ABCDEFGH"); + input[pos - 504] = b'Z'; + input[pos - 29..pos - 21].copy_from_slice(b"BCDEFGHI"); + input[pos - 30] = b'x'; + input[pos..pos + 9].copy_from_slice(b"ABCDEFGHI"); + + let mut finder = Rar50MatchFinder::new(input.len()); + for candidate in 0..pos { + finder.insert(&input, candidate); + } + let state = EncoderMatchState { + reps: [30, 0, 0, 0], + last_length: 8, + }; + let current = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &state, + DISTANCE_TABLE_SIZE_50, + ) + .unwrap(); + + assert_eq!((current.length, current.distance), (8, 512)); + assert!(should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::default().with_lazy_matching(true), + &state, + DISTANCE_TABLE_SIZE_50, + current, + )); +} + +#[test] +fn lazy_parser_uses_bounded_cost_lookahead() { + let pos = 160; + let mut input: Vec = (0..240u16) + .map(|value| value.wrapping_mul(91) as u8) + .collect(); + input[pos - 30..pos - 22].copy_from_slice(b"ABCDEFGH"); + input[pos - 80..pos - 70].copy_from_slice(b"CDEFGHIJKL"); + input[pos..pos + 12].copy_from_slice(b"ABCDEFGHIJKL"); + + let mut finder = Rar50MatchFinder::new(input.len()); + for candidate in 0..pos { + finder.insert(&input, candidate); + } + let state = EncoderMatchState::default(); + let current = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &state, + DISTANCE_TABLE_SIZE_50, + ) + .unwrap(); + + assert_eq!((current.length, current.distance), (8, 30)); + assert!(!should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(1), + &state, + DISTANCE_TABLE_SIZE_50, + current, + )); + assert!(should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(2), + &state, + DISTANCE_TABLE_SIZE_50, + current, + )); +} + +#[test] +fn lazy_parser_charges_for_skipped_literals() { + let pos = 160; + let mut input: Vec = (0..240u16) + .map(|value| value.wrapping_mul(91) as u8) + .collect(); + input[pos - 30..pos - 22].copy_from_slice(b"ABCDEFGH"); + input[pos - 80..pos - 71].copy_from_slice(b"CDEFGHIJK"); + input[pos..pos + 12].copy_from_slice(b"ABCDEFGHIJKL"); + + let mut finder = Rar50MatchFinder::new(input.len()); + for candidate in 0..pos { + finder.insert(&input, candidate); + } + let state = EncoderMatchState::default(); + let current = best_match( + &input, + pos, + input.len(), + &finder, + EncodeOptions::default(), + &state, + DISTANCE_TABLE_SIZE_50, + ) + .unwrap(); + + let next = best_match( + &input, + pos + 2, + input.len(), + &finder, + EncodeOptions::default(), + &state, + DISTANCE_TABLE_SIZE_50, + ) + .unwrap(); + + assert!(next.score > current.score); + assert!(next.score <= current.score + 16); + assert!(!should_lazy_emit_literal( + &input, + pos, + &finder, + EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(2), + &state, + DISTANCE_TABLE_SIZE_50, + current, + )); +} + +#[test] +fn lazy_match_lookahead_stops_at_the_member_boundary() { + let input = b"abcdef"; + let mut finder = Rar50MatchFinder::new(input.len()); + for pos in 0..4 { + finder.insert(input, pos); + } + let current = MatchCandidate { + length: 4, + distance: 1, + score: 1000, + }; + + assert_eq!( + lazy_match_decision( + input, + 4, + input.len(), + &finder, + EncodeOptions::default() + .with_lazy_matching(true) + .with_lazy_lookahead(4), + &EncoderMatchState::default(), + DISTANCE_TABLE_SIZE_50, + current, + ), + (false, None) + ); +} + +fn encode_lz_member_with_filter(data: &[u8], kind: crate::FilterKind) -> Result> { + Unpack50Encoder::new().encode_member_with_filter(data, 0, crate::FilterSpec::whole(kind)) +} + +#[test] +fn encodes_lz_member_with_delta_filter_record() { + let data: Vec = (0..96).map(|index| (index * 7 + index / 3) as u8).collect(); + let input = + encode_lz_member_with_filter(&data, crate::FilterKind::Delta { channels: 3 }).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_ne!(lengths.main[256], 0); +} + +#[test] +fn rejects_invalid_delta_filter_channel_count() { + assert_eq!( + encode_lz_member_with_filter(b"abc", crate::FilterKind::Delta { channels: 0 }), + Err(Error::InvalidData( + "RAR 5 DELTA filter channel count is invalid" + )) + ); + assert_eq!( + encode_lz_member_with_filter(b"abc", crate::FilterKind::Delta { channels: 33 }), + Err(Error::InvalidData( + "RAR 5 DELTA filter channel count is invalid" + )) + ); +} + +#[test] +fn filter_record_integers_round_trip_at_every_encoded_width() { + for value in [0, 0xff, 0x100, 0xffff, 0x1_0000, 0xff_ffff, 0x100_0000] { + let mut writer = BitWriter::new(); + try_write_filter_data(&mut writer, value).unwrap(); + let encoded = writer.finish(); + assert_eq!( + read_filter_data(&mut BitReader::new(&encoded)).unwrap(), + value + ); + } +} + +#[test] +fn truncated_filter_fields_and_overflowing_start_fail() { + assert_eq!( + read_filter_data(&mut BitReader::new(&[])), + Err(Error::NeedMoreInput) + ); + assert_eq!( + read_filter_data(&mut BitReader::new(&[0])), + Err(Error::NeedMoreInput) + ); + assert_eq!( + read_filter(&mut BitReader::new(&[]), 0), + Err(Error::NeedMoreInput) + ); + assert_eq!( + read_filter(&mut BitReader::new(&[0, 0]), 0), + Err(Error::NeedMoreInput) + ); + + let mut type_missing = BitWriter::new(); + type_missing.write_bits(0, 4); // The preceding Huffman code's tail. + try_write_filter_data(&mut type_missing, 0).unwrap(); + try_write_filter_data(&mut type_missing, 1).unwrap(); + let data = type_missing.finish(); + let mut bits = BitReader::new(&data); + bits.bit_pos = 4; + assert_eq!(read_filter(&mut bits, 0), Err(Error::NeedMoreInput)); + + let mut channels_missing = BitWriter::new(); + try_write_filter_data(&mut channels_missing, 0).unwrap(); + try_write_filter_data(&mut channels_missing, 1).unwrap(); + channels_missing.write_bits(0, 3); // DELTA, without its channel field. + assert_eq!( + read_filter(&mut BitReader::new(&channels_missing.finish()), 0), + Err(Error::NeedMoreInput) + ); + + let mut start_overflow = BitWriter::new(); + try_write_filter( + &mut start_overflow, + EncodeFilter { + offset: 1, + length: 1, + filter_type: FilterType::E8, + channels: 0, + }, + ) + .unwrap(); + assert_eq!( + read_filter(&mut BitReader::new(&start_overflow.finish()), usize::MAX), + Err(Error::InvalidData("RAR 5 filter start overflows")) + ); +} + +#[test] +fn filter_record_writer_rejects_invalid_delta_channels() { + for channels in [0, MAX_DELTA_CHANNELS + 1] { + let mut writer = BitWriter::new(); + assert_eq!( + try_write_filter( + &mut writer, + EncodeFilter { + offset: 0, + length: 1, + filter_type: FilterType::Delta, + channels, + }, + ), + Err(Error::InvalidData( + "RAR 5 DELTA filter channel count is invalid" + )) + ); + } +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn filter_record_writer_rejects_fields_beyond_u32() { + for (offset, length, message) in [ + (u32::MAX as usize + 1, 1, "RAR 5 filter offset is too large"), + (0, u32::MAX as usize + 1, "RAR 5 filter length is too large"), + ] { + let mut writer = BitWriter::new(); + assert_eq!( + try_write_filter( + &mut writer, + EncodeFilter { + offset, + length, + filter_type: FilterType::E8, + channels: 0, + }, + ), + Err(Error::InvalidData(message)) + ); + } +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn filter_transform_rejects_offset_beyond_record_width() { + let mut data = [0xe8, 0, 0, 0, 0]; + assert_eq!( + encode_filter_data( + Rar50Filter::E8, + &mut data, + u32::MAX as usize + 1, + &Allowance::default(), + ), + Err(Error::InvalidData("RAR 5 filter offset is too large")) + ); + assert_eq!(data, [0xe8, 0, 0, 0, 0]); +} + +#[test] +fn encodes_lz_member_with_e8_filter_record() { + let mut data = b"\xe8\0\0\0\0plain text after call".to_vec(); + data.extend_from_slice(&[0xe8, 3, 0, 0, 0, b'X']); + let input = encode_lz_member_with_filter(&data, crate::FilterKind::E8).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_ne!(lengths.main[256], 0); +} + +#[test] +fn short_e8_filter_range_round_trips_without_rewriting_bytes() { + let data = [0xe8, 0, 0, 0]; + let packed = encode_lz_member_with_filter(&data, crate::FilterKind::E8).unwrap(); + assert_eq!(decode_lz(&packed, 0, data.len()).unwrap(), data); +} + +#[test] +fn rar50_e8_filter_wraps_file_offset_modulo_16m() { + let file_offset = 0x0110_0000; + let mut encoded = vec![0xe8]; + encoded.extend_from_slice(&0x0010_0c08u32.to_le_bytes()); + + let mut decoded = encoded.clone(); + e8e9_decode(&mut decoded, file_offset, false); + + assert_eq!(&decoded[1..5], &0x0000_0c07u32.to_le_bytes()); + e8e9_encode(&mut decoded, file_offset, false); + assert_eq!(decoded, encoded); +} + +#[test] +fn streaming_decode_reports_filtered_member_with_typed_sentinel() { + let data = b"\xe8\0\0\0\0plain text after call".to_vec(); + let input = encode_lz_member_with_filter(&data, crate::FilterKind::E8).unwrap(); + let mut reader = input.as_slice(); + let mut decoder = Unpack50Decoder::new(); + + let error = decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut reader, + 0, + data.len(), + 128 * 1024, + false, + |_chunk| Ok::<_, std::convert::Infallible>(()), + ) + .unwrap_err(); + + assert!(matches!(error, StreamDecodeError::FilteredMember)); +} + +#[test] +fn streaming_filter_records_reconstruct_buffered_output() { + let cases = [ + ( + (0..96).map(|index| (index * 7 + index / 3) as u8).collect(), + crate::FilterKind::Delta { channels: 3 }, + ), + ( + b"\xe8\0\0\0\0plain text after call".to_vec(), + crate::FilterKind::E8, + ), + ( + b"\xe9\0\0\0\0jump target through e9".to_vec(), + crate::FilterKind::E8E9, + ), + ( + vec![0x04, 0x00, 0x00, 0xeb, b'A', b'R', b'M', b'!'], + crate::FilterKind::Arm, + ), + ]; + + for (expected, kind) in cases { + let packed = encode_lz_member_with_filter(&expected, kind).unwrap(); + let mut raw = Vec::new(); + let mut filters = Vec::new(); + Unpack50Decoder::new() + .decode_to_sink_with_filters( + &mut packed.as_slice(), + 0, + expected.len(), + DEFAULT_DICTIONARY_SIZE, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => raw.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + raw.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + Some(&mut |filter| { + filters.push(filter); + Ok::<(), std::convert::Infallible>(()) + }), + ) + .unwrap(); + + assert_eq!(filters.len(), 1); + apply_filters_with_control( + &mut raw, + &filters, + &crate::read_control::ReadControl::default(), + ) + .unwrap(); + assert_eq!(raw, expected); + } +} + +#[test] +fn encodes_lz_member_with_e8e9_filter_record() { + let data = b"\xe9\0\0\0\0jump target through e9".to_vec(); + let input = encode_lz_member_with_filter(&data, crate::FilterKind::E8E9).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_ne!(lengths.main[256], 0); +} + +#[test] +fn encodes_lz_member_with_ranged_e8e9_filter_record() { + let mut data = b"\xe8\0\0\0\0plain prefix outside filter range".to_vec(); + let range_start = data.len(); + for _ in 0..16 { + let operand_pos = data.len() + 1; + data.push(0xe8); + let relative = 0x7000u32.wrapping_sub(operand_pos as u32); + data.extend_from_slice(&relative.to_le_bytes()); + data.extend_from_slice(b" code "); + } + let range = range_start..data.len(); + data.extend_from_slice(b"\xe9\0\0\0\0plain suffix outside filter range"); + + let input = Unpack50Encoder::new() + .encode_member_with_filter( + &data, + 0, + crate::FilterSpec::range(crate::FilterKind::E8E9, range), + ) + .unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_ne!(lengths.main[256], 0); +} + +#[test] +fn encodes_lz_member_with_multiple_filter_records() { + let mut data = b"\xe8\0\0\0\0plain prefix outside filters".to_vec(); + let first_start = data.len(); + data.extend_from_slice(b"\xe8\0\0\0\0first filtered cluster"); + let first_end = data.len(); + data.extend_from_slice(b"large plain middle outside filters"); + let second_start = data.len(); + data.extend_from_slice(b"\xe8\0\0\0\0second filtered cluster"); + let second_end = data.len(); + + let input = Unpack50Encoder::new() + .encode_member_with_filters( + &data, + 0, + &[ + crate::FilterSpec::range(crate::FilterKind::E8, first_start..first_end), + crate::FilterSpec::range(crate::FilterKind::E8, second_start..second_end), + ], + ) + .unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, table_bits) = read_table_lengths(&input[block.payload.clone()], 0).unwrap(); + let tables = DecodeTables::from_lengths(&lengths).unwrap(); + let mut bits = BitReader { + input: &input[block.payload], + bit_pos: table_bits, + }; + assert_eq!(tables.main.decode(&mut bits).unwrap(), 256); + let first = read_filter(&mut bits, 0).unwrap(); + assert_eq!(tables.main.decode(&mut bits).unwrap(), 256); + let second = read_filter(&mut bits, 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_eq!(first.start, first_start); + assert_eq!(second.start, second_start); +} + +#[test] +fn encodes_lz_member_with_arm_filter_record() { + let data = [0x04, 0x00, 0x00, 0xeb, b'A', b'R', b'M', b'!']; + let input = encode_lz_member_with_filter(&data, crate::FilterKind::Arm).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_ne!(lengths.main[256], 0); +} + +#[test] +fn arm_filter_uses_wrapping_address_arithmetic_at_u32_boundary() { + let original = [0x04, 0x00, 0x00, 0xeb, 0x08, 0x00, 0x00, 0xeb]; + let mut filtered = original; + + arm_encode(&mut filtered, u32::MAX - 3); + assert_ne!(filtered, original); + arm_decode(&mut filtered, u32::MAX - 3); + + assert_eq!(filtered, original); +} + +#[test] +fn solid_encoder_emits_rar50_matches_against_previous_member_history() { + let first = b"RAR5 solid shared phrase alpha beta gamma\n".repeat(16); + let second = b"RAR5 solid shared phrase alpha beta gamma\nsecond\n".repeat(4); + let solid = encode_lz_member_with_history(&second, &first, 0).unwrap(); + let standalone = encode_lz_member(&second, 0).unwrap(); + let mut decoder = Unpack50Decoder::new(); + + assert_eq!( + decoder + .decode_member( + &encode_lz_member(&first, 0).unwrap(), + 0, + first.len(), + false, + DecodeMode::Lz + ) + .unwrap(), + first + ); + assert_eq!( + decoder + .decode_member(&solid, 0, second.len(), true, DecodeMode::Lz) + .unwrap(), + second + ); + assert!(solid.len() < standalone.len()); +} + +#[test] +fn public_decoder_clone_keeps_independent_solid_history_and_tables() { + let first = b"RAR5 independent decoder history ".repeat(32); + let second = b"RAR5 independent decoder history ".repeat(8); + let packed_first = encode_lz_member(&first, 0).unwrap(); + let packed_second = encode_lz_member_with_history(&second, &first, 0).unwrap(); + let mut original = Unpack50Decoder::new(); + assert_eq!( + original + .decode_member(&packed_first, 0, first.len(), false, DecodeMode::Lz) + .unwrap(), + first + ); + let mut copied = original.clone(); + let other = b"a different non-solid member"; + assert_eq!( + original + .decode_member( + &encode_lz_member(other, 0).unwrap(), + 0, + other.len(), + false, + DecodeMode::Lz + ) + .unwrap(), + other + ); + drop(original); + assert_eq!( + copied + .decode_member(&packed_second, 0, second.len(), true, DecodeMode::Lz) + .unwrap(), + second + ); +} + +#[test] +fn large_lz_members_are_split_into_multiple_compressed_blocks() { + // Two halves that do not look alike, so the second cannot be spelled + // with the first's tables and gets its own block. + let mut data = wordy_text(LZ_BLOCK_SIZE); + data.extend((0..LZ_BLOCK_SIZE).map(|index| (index as u8).wrapping_mul(37))); + let encoded = encode_lz_member_with_options(&data, 0, EncodeOptions::new(16)).unwrap(); + let mut cursor = std::io::Cursor::new(encoded.as_slice()); + let first = read_compressed_block(&mut cursor).unwrap(); + let second = read_compressed_block(&mut cursor).unwrap(); + let mut decoder = Unpack50Decoder::new(); + + assert!(!first.header.is_last); + assert!(second.header.is_last); + assert_eq!( + decoder + .decode_member(&encoded, 0, data.len(), false, DecodeMode::Lz) + .unwrap(), + data + ); +} + +#[test] +fn a_member_that_does_not_move_is_left_in_one_block() { + // Every chunk of this has the same byte distribution, so a fresh table + // set per chunk would describe nothing the first one did not. + let data = vec![0u8; LZ_BLOCK_SIZE * 4]; + let encoded = encode_lz_member_with_options(&data, 0, EncodeOptions::new(16)).unwrap(); + let mut cursor = std::io::Cursor::new(encoded.as_slice()); + let only = read_compressed_block(&mut cursor).unwrap(); + let mut decoder = Unpack50Decoder::new(); + + assert!(only.header.is_last, "a still member was cut up anyway"); + assert_eq!( + decoder + .decode_member(&encoded, 0, data.len(), false, DecodeMode::Lz) + .unwrap(), + data + ); +} + +#[test] +fn a_block_never_grows_past_the_cap() { + let data = vec![0u8; MAX_LZ_BLOCK_SIZE * 2 + LZ_BLOCK_SIZE]; + let encoded = encode_lz_member_with_options(&data, 0, EncodeOptions::new(16)).unwrap(); + let mut cursor = std::io::Cursor::new(encoded.as_slice()); + let mut blocks = 0; + loop { + let block = read_compressed_block(&mut cursor).unwrap(); + blocks += 1; + if block.header.is_last { + break; + } + } + + // Uniform to the last byte, so only the cap can be ending these. + assert_eq!(blocks, 3, "the cap stopped ending blocks"); +} + +#[test] +fn splitter_refuses_to_extend_an_empty_block_or_with_an_empty_chunk() { + let mut splitter = BlockSplitter::new(); + assert!(!splitter.extends(b"AAAA")); + splitter.accept(b"AAAA"); + assert!(!splitter.extends(b"")); + assert!(splitter.extends(b"AAAA")); +} + +#[test] +fn the_splitter_reads_only_what_both_writers_can_see() { + // The streaming writer decides with the open block's bytes and the next + // chunk, and nothing else. Same bytes in, same answer, however the + // chunks were handed over. + let mut whole = BlockSplitter::new(); + whole.accept(&vec![7u8; 4096]); + let mut piecemeal = BlockSplitter::new(); + for _ in 0..4 { + piecemeal.accept(&vec![7u8; 1024]); + } + + let next = vec![7u8; 4096]; + assert!(whole.extends(&next)); + assert_eq!(whole.extends(&next), piecemeal.extends(&next)); + + let moved: Vec = (0..4096u32).map(|index| index as u8).collect(); + assert!(!whole.extends(&moved), "a block swallowed unlike data"); + assert_eq!(whole.extends(&moved), piecemeal.extends(&moved)); +} + +/// Words drawn from a small vocabulary in a repeating-but-not-periodic +/// order. Every position has several matches at different distances and +/// lengths, which is the case where taking the longest one and checking +/// two bytes ahead leaves bits on the floor. +fn wordy_text(len: usize) -> Vec { + const WORDS: [&str; 12] = [ + "the ", "quick ", "brown ", "fox ", "jumps ", "over ", "lazy ", "dog ", "and ", "then ", + "runs ", "away ", + ]; + let mut out = Vec::with_capacity(len + 16); + let mut noise = 0x2545_f491_4f6c_dd1du64; + while out.len() < len { + noise ^= noise << 13; + noise ^= noise >> 7; + noise ^= noise << 17; + out.extend_from_slice(WORDS[(noise >> 40) as usize % WORDS.len()].as_bytes()); + if (noise >> 20).is_multiple_of(11) { + out.push(b'\n'); + } + } + out.truncate(len); + out +} + +#[test] +fn the_optimal_parse_round_trips() { + for data in [ + Vec::new(), + b"a".to_vec(), + b"abcabcabcabc".to_vec(), + wordy_text(3), + wordy_text(LZ_BLOCK_SIZE + 4096), + vec![0u8; LZ_BLOCK_SIZE * 2], + ] { + let options = EncodeOptions::new(64).with_optimal_parse(true); + let encoded = encode_lz_member_with_options(&data, 0, options).unwrap(); + let decoded = Unpack50Decoder::new() + .decode_member(&encoded, 0, data.len(), false, DecodeMode::Lz) + .unwrap(); + assert_eq!(decoded, data, "{} bytes did not round trip", data.len()); + } +} + +#[test] +fn optimal_parse_with_matching_disabled_emits_literals() { + let data = b"ABABABAB"; + let tokens = collected_optimal_tokens( + data, + EncodeOptions::new(0).with_optimal_parse(true), + DISTANCE_TABLE_SIZE_50, + None, + ); + + assert_eq!( + tokens, + data.iter() + .copied() + .map(EncodeToken::Literal) + .collect::>() + ); +} + +#[test] +fn optimal_long_match_commitment_prunes_shorter_candidate_endpoints() { + let history = 2; + let span = NICE_MATCH_LENGTH + 8; + let combined = vec![0; history + span]; + let mut price = vec![u32::MAX; span + 1]; + let mut arrive_length = vec![0; span + 1]; + let mut arrive_distance = vec![0; span + 1]; + let mut arrive_reps = vec![[0; 4]; span + 1]; + let mut arrive_last_length = vec![0; span + 1]; + price[0] = 0; + arrive_reps[0] = [0, (history + 1) as u32, 0, 0]; + let runs = [(4, 1), (NICE_MATCH_LENGTH as u32, 2)]; + let mut starts = vec![2; span + 1]; + starts[0] = 0; + let mut reaches = [(0, 0, 0); 8]; + + price_optimal_paths( + &combined, + history..history + span, + EncodeOptions::new(32).with_optimal_parse(true), + DISTANCE_TABLE_SIZE_50, + None, + &runs, + &starts, + OptimalSlices { + price: &mut price, + arrive_length: &mut arrive_length, + arrive_distance: &mut arrive_distance, + arrive_reps: &mut arrive_reps, + arrive_last_length: &mut arrive_last_length, + }, + &mut reaches, + ); + + assert_eq!(price[4], u32::MAX); + assert_ne!(price[NICE_MATCH_LENGTH], u32::MAX); + assert_eq!(arrive_length[NICE_MATCH_LENGTH], NICE_MATCH_LENGTH as u32); + assert_eq!(arrive_distance[NICE_MATCH_LENGTH], 2); +} + +#[test] +fn the_optimal_parse_beats_lazy_matching_at_the_same_depth() { + let data = wordy_text(256 * 1024); + let base = EncodeOptions::new(64); + let lazy = encode_lz_member_with_options( + &data, + 0, + base.with_lazy_matching(true).with_lazy_lookahead(2), + ) + .unwrap(); + let optimal = encode_lz_member_with_options(&data, 0, base.with_optimal_parse(true)).unwrap(); + + assert!( + optimal.len() < lazy.len(), + "optimal parse packed {} against lazy matching's {}", + optimal.len(), + lazy.len(), + ); +} + +/// One chunk, copied once per round with fresh literals in between, so +/// every match after the first sits at the same distance and every one of +/// them is separated from the last by literals. +fn strided_repeats(gap: usize, rounds: usize) -> Vec { + let mut noise = 0x1234_5678u32; + let mut byte = move || { + noise = noise.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (noise >> 24) as u8 + }; + let chunk: Vec = (0..48).map(|_| byte()).collect(); + let mut data = chunk.clone(); + for _ in 0..rounds { + data.extend((0..gap).map(|_| byte())); + data.extend_from_slice(&chunk); + } + data +} + +#[test] +fn the_optimal_parse_reuses_a_distance_across_the_literals_between_matches() { + let data = strided_repeats(8, 3000); + let options = EncodeOptions::new(64).with_optimal_parse(true); + let distance_size = DISTANCE_TABLE_SIZE_50; + let tokens = collected_optimal_tokens(&data, options, distance_size, None); + + let mut state = EncoderMatchState::default(); + let (mut repeated, mut fresh) = (0, 0); + for token in &tokens { + if let EncodeToken::Match { length, distance } = *token { + match state.encode_match(length, distance, distance_size).unwrap() { + EncodedMatch::New { .. } => fresh += 1, + _ => repeated += 1, + } + state.remember(length, distance); + } + } + + // Every match here is at the one distance the data repeats at, so all + // but the first should be priced against what the path remembers. + // Carrying only the arriving match instead left this at 244 repeated + // against 2,756 fresh, because a literal wiped the distance before the + // next match could be priced against it. + assert!( + fresh * 10 < repeated, + "{repeated} matches reused a distance against {fresh} that did not", + ); +} + +#[test] +fn reusing_a_remembered_distance_packs_strided_data_smaller() { + let data = strided_repeats(8, 3000); + let options = EncodeOptions::new(64).with_optimal_parse(true); + let packed = encode_lz_member_with_options(&data, 0, options).unwrap(); + + // 26,178 bytes when the path carries its remembered distances, 28,447 + // when it does not. The bound sits between the two. + assert!( + packed.len() < 27_000, + "{} bytes packed from {}", + packed.len(), + data.len(), + ); + let decoded = decode_lz(&packed, 0, data.len()).unwrap(); + assert_eq!(decoded, data); +} + +/// Once the parse commits to a match it steps over the bytes that match +/// covers, so a block that repeats is emitted as whole matches back to back +/// rather than as the cheapest split of each one. That is the difference +/// between the parse pricing 4096 positions per match and pricing one, which +/// is worth 53.41s against 1.19s on a mebibyte of this. +#[test] +fn the_optimal_parse_steps_over_a_match_it_has_committed_to() { + let block: Vec = (0..4096u32) + .map(|index| (index * 7 + (index >> 5)) as u8) + .collect(); + let data = block.repeat(16); + let options = EncodeOptions::new(64).with_optimal_parse(true); + let tokens = collected_optimal_tokens(&data, options, DISTANCE_TABLE_SIZE_50, None); + + let first = tokens + .iter() + .position(|token| { + matches!(token, EncodeToken::Match { length, .. } if *length >= NICE_MATCH_LENGTH) + }) + .expect("no match reached the length the parse commits at"); + for token in &tokens[first..] { + let EncodeToken::Match { length, .. } = *token else { + panic!("{token:?} interrupts the committed matches"); + }; + assert!( + length >= NICE_MATCH_LENGTH, + "a {length}-byte match interrupts the committed ones", + ); + } + // The first block is the only one with nothing behind it to match. + assert_eq!(tokens.len() - first, data.len() / block.len() - 1); + + let packed = encode_lz_member_with_options(&data, 0, options).unwrap(); + assert_eq!(decode_lz(&packed, 0, data.len()).unwrap(), data); +} + +#[test] +fn repricing_against_the_first_pass_beats_the_flat_guess() { + let data = wordy_text(256 * 1024); + let options = EncodeOptions::new(64).with_optimal_parse(true); + let distance_size = DISTANCE_TABLE_SIZE_50; + let guessed = collected_optimal_tokens(&data, options, distance_size, None); + let lengths = table_lengths_for_tokens(&guessed, distance_size).unwrap(); + let prices = TokenPrices { + lengths: lengths.slices(), + }; + let repriced = collected_optimal_tokens(&data, options, distance_size, Some(&prices)); + + let bits = |tokens: &[EncodeToken]| -> usize { + let lengths = table_lengths_for_tokens(tokens, distance_size).unwrap(); + let prices = TokenPrices { + lengths: lengths.slices(), + }; + let mut state = EncoderMatchState::default(); + let mut total = 0; + for token in tokens { + match *token { + EncodeToken::Filter(_) => {} + EncodeToken::Literal(byte) => total += prices.literal(byte), + EncodeToken::Match { length, distance } => { + total += prices + .match_cost(&state, length, distance, distance_size) + .unwrap(); + state.remember(length, distance); + } + } + } + total + }; + + assert!( + bits(&repriced) < bits(&guessed), + "repriced {} bits against the guess's {}", + bits(&repriced), + bits(&guessed), + ); +} + +#[test] +fn blocks_are_short_enough_to_refit_the_tables_when_the_data_changes() { + // Four stretches, one block each, every one drawing from its own + // sixteen bytes. A table per block codes sixteen symbols; one table + // over the member codes sixty-four. Raising LZ_BLOCK_SIZE trades the + // first for the second, which is what cost the corpus 6.4% until the + // block came down from a mebibyte. + let stretch_len = 64 * 1024; + let mut data = Vec::with_capacity(stretch_len * 4); + let mut noise = 0x2545_f491_4f6c_dd1du64; + for stretch in 0..4u8 { + for _ in 0..stretch_len { + noise ^= noise << 13; + noise ^= noise >> 7; + noise ^= noise << 17; + data.push(stretch * 16 + (noise >> 40) as u8 % 16); + } + } + let options = EncodeOptions::new(0); + + let blocked = encode_lz_member_with_options(&data, 0, options).unwrap(); + let single = encode_lz_block(&data, &[], 0, &[], options, true, None).unwrap(); + + assert!( + blocked.len() * 100 < single.len() * 90, + "member blocks {} did not beat one block over the lot {single}", + blocked.len(), + single = single.len(), + ); +} + +#[test] +fn large_filtered_lz_members_split_filter_records_by_block() { + let last_block_start = FILTERED_LZ_BLOCK_SIZE * 2; + let mut data: Vec<_> = (0..last_block_start + 512) + .map(|index| index as u8) + .collect(); + data[256] = 0xe8; + data[257..261].copy_from_slice(&0x20u32.to_le_bytes()); + data[last_block_start + 64] = 0xe8; + data[last_block_start + 65..last_block_start + 69].copy_from_slice(&0x40u32.to_le_bytes()); + + let encoded = Unpack50Encoder::with_options(EncodeOptions::new(0)) + .encode_member_with_filter( + &data, + 0, + crate::FilterSpec::range(crate::FilterKind::E8, 0..data.len()), + ) + .unwrap(); + let mut cursor = std::io::Cursor::new(encoded.as_slice()); + let first = read_compressed_block(&mut cursor).unwrap(); + let mut blocks = 1usize; + let mut last_is_last = first.header.is_last; + while cursor.position() < encoded.len() as u64 { + last_is_last = read_compressed_block(&mut cursor).unwrap().header.is_last; + blocks += 1; + } + let mut decoder = Unpack50Decoder::new(); + + assert!(!first.header.is_last); + assert!(last_is_last); + assert!(blocks > 2); + assert_eq!( + decoder + .decode_member(&encoded, 0, data.len(), false, DecodeMode::Lz) + .unwrap(), + data + ); +} + +/// The window a filtered member is parsed in reaches back into the +/// history the solid stream carries, and keeps reaching for every block +/// of it rather than only the first. Each block used to be handed its own +/// copy of that history and its own finder rebuilt from it, which cost +/// the member a re-insert per block and reached exactly this far. +#[test] +fn a_solid_filtered_member_matches_history_from_every_block() { + // Bytes that do not compress on their own, so matching the history is + // the only way this member gets small. + let mut state = 0x2545_f491_4f6c_dd1du64; + let mut earlier = Vec::with_capacity(FILTERED_LZ_BLOCK_SIZE * 3); + while earlier.len() < FILTERED_LZ_BLOCK_SIZE * 3 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + earlier.extend_from_slice(&state.to_le_bytes()); + } + earlier[512] = 0xe8; + earlier[513..517].copy_from_slice(&0x30u32.to_le_bytes()); + // Repeating what came before means every block of this member, not + // just its first, has something to match against the history. + let data = earlier.clone(); + let filter = crate::FilterSpec::range(crate::FilterKind::E8, 0..data.len()); + + let mut solid = Unpack50Encoder::with_options(EncodeOptions::new(16)); + solid + .state + .history + .remember(&earlier, solid.state.options.max_match_distance) + .unwrap(); + let against_history = solid + .encode_member_with_filter(&data, 0, filter.clone()) + .unwrap(); + let alone = Unpack50Encoder::with_options(EncodeOptions::new(16)) + .encode_member_with_filter(&data, 0, filter) + .unwrap(); + + assert!( + against_history.len() * 20 < alone.len(), + "a member repeating its history packed to {} against {alone} on its own", + against_history.len(), + alone = alone.len(), + ); +} + +#[test] +fn filters_are_split_before_rar_reader_filter_limit() { + let data = vec![0u8; FILTERED_LZ_BLOCK_SIZE + 1]; + let encoded = + Unpack50Encoder::with_options(EncodeOptions::new(0).with_max_match_distance(128 * 1024)) + .encode_member_with_filter( + &data, + 0, + crate::FilterSpec::whole(crate::FilterKind::Delta { channels: 4 }), + ) + .unwrap(); + let mut cursor = std::io::Cursor::new(encoded.as_slice()); + let first = read_compressed_block(&mut cursor).unwrap(); + let second = read_compressed_block(&mut cursor).unwrap(); + let mut decoder = Unpack50Decoder::new(); + + assert!(!first.header.is_last); + assert!(second.header.is_last); + assert_eq!( + decoder + .decode_member(&encoded, 0, data.len(), false, DecodeMode::Lz) + .unwrap(), + data + ); +} + +#[test] +fn solid_encoder_history_limit_follows_encode_options_dictionary() { + let mut encoder = Unpack50Encoder::with_options( + EncodeOptions::new(0).with_max_match_distance(DEFAULT_DICTIONARY_SIZE + 1024), + ); + encoder + .state + .history + .remember( + &vec![0x41; DEFAULT_DICTIONARY_SIZE + 512], + encoder.state.options.max_match_distance, + ) + .unwrap(); + + assert_eq!(encoder.state.history.len(), DEFAULT_DICTIONARY_SIZE + 512); + + let mut capped = + Unpack50Encoder::with_options(EncodeOptions::new(0).with_max_match_distance(1024)); + capped + .state + .history + .remember(&vec![0x42; 4096], capped.state.options.max_match_distance) + .unwrap(); + + assert_eq!(capped.state.history.len(), 1024); +} + +#[test] +fn cloned_encoder_keeps_an_independent_solid_history() { + let first = b"prefix repeated across solid members"; + let second = b"prefix repeated across solid members with a suffix"; + let mut original = Unpack50Encoder::with_options(EncodeOptions::new(16)); + let packed_first = original.encode_member(first, 0).unwrap(); + let mut cloned = original.clone(); + let packed_original = original.encode_member(second, 0).unwrap(); + let packed_clone = cloned.encode_member(second, 0).unwrap(); + assert_eq!(packed_clone, packed_original); + + let mut decoder = Unpack50Decoder::new(); + assert_eq!( + decoder + .decode_member(&packed_first, 0, first.len(), false, DecodeMode::Lz) + .unwrap(), + first + ); + assert_eq!( + decoder + .decode_member(&packed_clone, 0, second.len(), true, DecodeMode::Lz) + .unwrap(), + second + ); + + let clone_history = cloned.state.history.to_vec(); + original + .encode_member(b"a divergent third member", 0) + .unwrap(); + assert_eq!(cloned.state.history.to_vec(), clone_history); +} + +#[test] +fn encodes_lz_member_with_last_length_repeat_symbols() { + let data = b"abcdXabcdYabcdZabcd"; + let input = encode_lz_member(data, 0).unwrap(); + let block = parse_compressed_block(&input).unwrap(); + let (lengths, _) = read_table_lengths(&input[block.payload], 0).unwrap(); + + let output = decode_lz(&input, 0, data.len()).unwrap(); + + assert_eq!(output, data); + assert_ne!(lengths.main[257], 0); +} + +#[test] +fn encodes_lz_member_using_rar70_distance_table_shape() { + let data = b"RAR7-compatible repeated phrase repeated phrase repeated phrase"; + let input = encode_lz_member(data, 1).unwrap(); + + let output = decode_lz(&input, 1, data.len()).unwrap(); + + assert_eq!(output, data); +} + +#[test] +fn decode_member_from_reader_accepts_incremental_input() { + struct OneByteReader<'a> { + data: &'a [u8], + pos: usize, + } + + impl Read for OneByteReader<'_> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.pos >= self.data.len() { + return Ok(0); + } + out[0] = self.data[self.pos]; + self.pos += 1; + Ok(1) + } + } + + let payload = literal_only_payload(b"ABBA"); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + let mut reader = OneByteReader { + data: &input, + pos: 0, + }; + let mut decoder = Unpack50Decoder::new(); + + let output = decoder + .decode_member_from_reader(&mut reader, 0, 4, false, DecodeMode::LiteralOnly) + .unwrap(); + + assert_eq!(output, b"ABBA"); +} + +#[test] +fn decoders_stop_at_declared_size_before_a_nonfinal_block() { + let payload = literal_only_payload(b"AB"); + let block = encode_compressed_block(&payload, payload.len() * 8, true, false).unwrap(); + let mut input = block.clone(); + input.extend_from_slice(b"not another block"); + + assert_eq!(decode_literal_only(&input, 0, 2).unwrap(), b"AB"); + let mut reader = input.as_slice(); + let mut streamed = Vec::new(); + Unpack50Decoder::new() + .decode_member_from_reader_with_dictionary_to_sink( + &mut reader, + 0, + 2, + DEFAULT_DICTIONARY_SIZE, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => streamed.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + streamed.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + assert_eq!(streamed, b"AB"); + assert_eq!(reader, &input[block.len()..]); +} + +#[test] +fn decodes_synthetic_new_match_block() { + let payload = new_match_payload(); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + + let output = decode_lz(&input, 0, 4).unwrap(); + + assert_eq!(output, b"ABAB"); +} + +#[test] +fn decodes_synthetic_last_length_match_block() { + let payload = repeat_payload(257); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + + let output = decode_lz(&input, 0, 6).unwrap(); + + assert_eq!(output, b"ABABAB"); +} + +#[test] +fn decodes_synthetic_repeat_distance_match_block() { + let payload = repeat_payload(258); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + + let output = decode_lz(&input, 0, 6).unwrap(); + + assert_eq!(output, b"ABABAB"); +} + +#[test] +fn rejects_literal_only_block_without_tables() { + let input = encode_compressed_block(&[0], 8, false, true).unwrap(); + + assert_eq!( + decode_literal_only(&input, 0, 1), + Err(Error::InvalidData("RAR 5 block reuses missing tables")) + ); +} + +#[test] +fn decoders_propagate_malformed_table_errors() { + let mut incomplete = BitWriter::new(); + for _ in 0..LEVEL_TABLE_SIZE { + incomplete.write_bits(5, 4); + } + let incomplete = incomplete.finish(); + + let mut invalid = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + invalid.main[..3].fill(1); // Three one-bit codes overfill the tree. + let (invalid, invalid_bits) = encode_table_lengths_with_bit_count(&invalid, 0).unwrap(); + + for (payload, payload_bits, expected) in [ + ( + incomplete.as_slice(), + incomplete.len() * 8, + Error::NeedMoreInput, + ), + ( + invalid.as_slice(), + invalid_bits, + Error::InvalidData("RAR 5 oversubscribed Huffman table"), + ), + ] { + let input = encode_compressed_block(payload, payload_bits, true, true).unwrap(); + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::LiteralOnly, + ), + Err(expected.clone()) + ); + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!(result, Err(StreamDecodeError::Decode(error)) if error == expected)); + } +} + +#[test] +fn decoders_reject_invalid_packed_huffman_symbol() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; // Only the zero bit names a symbol. + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + writer.write_bits(0x7fff, 15); // No prefix can name 'A'. + let payload_bits = writer.bit_pos; + let input = encode_compressed_block(&writer.finish(), payload_bits, true, true).unwrap(); + let expected = Error::InvalidData("RAR 5 invalid Huffman code"); + + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::LiteralOnly, + ), + Err(expected.clone()) + ); + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!(result, Err(StreamDecodeError::Decode(error)) if error == expected)); +} + +#[test] +fn decoders_report_truncated_match_extra_bits() { + for (length_slot, distance_slot, packed) in [ + // Seven literals and a new match. Main symbol 270 means length + // slot 8, which needs one more bit. + (8, 0, 0b0000_0001), + // Six literals, a new match, and distance slot 4. Its extra bit + // is absent after the one-bit distance code. + (0, 4, 0b0000_0010), + ] { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[262 + length_slot] = 1; + lengths.distance[distance_slot] = 1; + let (tables, table_bits) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut input = encode_compressed_block(&tables, table_bits, true, false).unwrap(); + input.extend(encode_compressed_block(&[packed], 8, false, true).unwrap()); + + let mut buffered = Unpack50Decoder::new(); + let mut cursor = std::io::Cursor::new(&input); + assert_eq!( + buffered.decode_member_from_reader_with_dictionary( + &mut cursor, + 0, + 12, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(Error::NeedMoreInput), + "length slot {length_slot}, distance slot {distance_slot}" + ); + assert!(buffered.state.tables.is_none()); // Error occurred before the block completed. + let mut streaming = Unpack50Decoder::new(); + let result = streaming.decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 12, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::NeedMoreInput)) + )); + assert!(streaming.state.tables.is_none()); + } +} + +#[test] +fn decoders_reject_rar7_distance_slot_above_bit_limit() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_70], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[262] = 1; // New match with length slot 0. + lengths.distance[66] = 1; // (66 - 2) / 2 = 32 extra bits. + let (tables, table_bits) = encode_table_lengths_with_bit_count(&lengths, 1).unwrap(); + let mut input = encode_compressed_block(&tables, table_bits, true, false).unwrap(); + input.extend(encode_compressed_block(&[0], 2, false, true).unwrap()); + let expected = Error::InvalidData("RAR 5 distance slot is too large"); + + let mut buffered = Unpack50Decoder::new(); + assert_eq!( + buffered.decode_member_with_dictionary( + &input, + 1, + 2, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(expected.clone()) + ); + assert!(buffered.state.tables.is_none()); + let mut streaming = Unpack50Decoder::new(); + let result = streaming.decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 1, + 2, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!(result, Err(StreamDecodeError::Decode(error)) if error == expected)); + assert!(streaming.state.tables.is_none()); +} + +#[test] +fn solid_decoders_report_truncated_repeat_length() { + let bootstrap = new_match_payload(); + let bootstrap = encode_compressed_block(&bootstrap, bootstrap.len() * 8, true, true).unwrap(); + + for (length_slot, packed) in [ + (0, 0b0000_0001), // Seven literals and a repeat, without its length code. + (8, 0b0000_0010), // Six literals, repeat, length slot 8; extra bit absent. + ] { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[258] = 1; + lengths.length[length_slot] = 1; + let (tables, table_bits) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut input = encode_compressed_block(&tables, table_bits, true, false).unwrap(); + input.extend(encode_compressed_block(&[packed], 8, false, true).unwrap()); + + let mut buffered = Unpack50Decoder::new(); + assert_eq!( + buffered + .decode_member_with_dictionary( + &bootstrap, + 0, + 4, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ) + .unwrap(), + b"ABAB" + ); + assert_eq!( + buffered.decode_member_with_dictionary( + &input, + 0, + 20, + DEFAULT_DICTIONARY_SIZE, + true, + DecodeMode::Lz, + ), + Err(Error::NeedMoreInput) + ); + assert!(buffered.state.tables.is_none()); + + let mut streaming = Unpack50Decoder::new(); + streaming + .decode_member_from_reader_with_dictionary_to_sink( + &mut bootstrap.as_slice(), + 0, + 4, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ) + .unwrap(); + let result = streaming.decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 20, + DEFAULT_DICTIONARY_SIZE, + true, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::NeedMoreInput)) + )); + assert!(streaming.state.tables.is_none()); + } +} + +#[test] +fn decoders_report_malformed_new_match_distance_fields() { + let mut invalid_align = BitWriter::new(); + invalid_align.write_bits(0, 1); // New match, length slot 0. + invalid_align.write_bits(0, 1); // Distance slot 10. + invalid_align.write_bits(0x7fff, 15); // No alignment prefix names a symbol. + let cases = [ + ( + None, + false, + 0, + vec![0], + 1, + Error::InvalidData("RAR 5 empty Huffman table"), + ), + ( + Some(12), + true, + 6, + vec![0b0000_0010], + 8, + Error::NeedMoreInput, + ), + ( + Some(10), + true, + 0, + invalid_align.finish(), + 17, + Error::InvalidData("RAR 5 invalid Huffman code"), + ), + ]; + + for (distance_slot, align_mode, literal_count, packed, packed_bits, expected) in cases { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[262] = 1; + if literal_count != 0 { + lengths.main[b'A' as usize] = 1; + } + if let Some(slot) = distance_slot { + lengths.distance[slot] = 1; + } + if align_mode { + lengths.align[0] = 1; + } + let (tables, table_bits) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut input = encode_compressed_block(&tables, table_bits, true, false).unwrap(); + input.extend(encode_compressed_block(&packed, packed_bits, false, true).unwrap()); + + let mut buffered = Unpack50Decoder::new(); + assert_eq!( + buffered.decode_member_with_dictionary( + &input, + 0, + 20, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(expected.clone()), + "distance slot {distance_slot:?}" + ); + assert!(buffered.state.tables.is_none()); + let mut streaming = Unpack50Decoder::new(); + let result = streaming.decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 20, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!(result, Err(StreamDecodeError::Decode(error)) if error == expected)); + assert!(streaming.state.tables.is_none()); + } +} + +#[test] +fn decodes_length_slots() { + assert_eq!(slot_to_length(0, 0).unwrap(), 2); + assert_eq!(slot_to_length(7, 0).unwrap(), 9); + assert_eq!(slot_to_length(8, 0).unwrap(), 10); + assert_eq!(slot_to_length(8, 1).unwrap(), 11); + assert_eq!(slot_to_length(11, 1).unwrap(), 17); + assert_eq!(slot_to_length(12, 3).unwrap(), 21); + assert_eq!(length_slot_extra_bits(43), 9); +} + +#[test] +fn rejects_out_of_range_length_slots_and_extras() { + assert!(slot_to_length(103, (1 << 24) - 1).is_ok()); + assert_eq!( + slot_to_length(8, 2), + Err(Error::InvalidData("RAR 5 length extra bits exceed slot")) + ); + assert_eq!( + slot_to_length(104, 0), + Err(Error::InvalidData("RAR 5 length slot is too large")) + ); +} + +#[test] +fn encoder_code_table_rejects_oversubscribed_lengths() { + assert_eq!( + EncoderCodeTable::from_lengths(&[1, 1, 1], &Allowance::default()).err(), + Some(Error::InvalidData("RAR 5 oversubscribed Huffman table")) + ); +} + +#[test] +fn decodes_distance_slots() { + assert_eq!(slot_to_distance(0, 0).unwrap(), 1); + assert_eq!(slot_to_distance(3, 0).unwrap(), 4); + assert_eq!(distance_slot_bit_count(4).unwrap(), 1); + assert_eq!(slot_to_distance(4, 0).unwrap(), 5); + assert_eq!(slot_to_distance(4, 1).unwrap(), 6); + assert_eq!(distance_slot_bit_count(10).unwrap(), 4); + assert_eq!(slot_to_distance(10, 15).unwrap(), 48); +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn encoder_rejects_distance_beyond_rar50_table() { + // A caller may raise max_match_distance beyond the RAR 5.0 table's + // range. Such a match must fail before indexing the frequency table. + assert_eq!( + EncoderMatchState::default().encode_match(4, 1usize << 40, DISTANCE_TABLE_SIZE_50), + Err(Error::InvalidData("RAR 5 match distance is too large")) + ); +} + +#[test] +fn match_pricing_and_encoding_reject_invalid_internal_candidates() { + let mut repeated = EncoderMatchState::default(); + repeated.reps[0] = 1; + repeated.last_length = 2; + let too_short = Error::InvalidData("RAR 5 match length is too short"); + assert_eq!( + repeated + .encode_match(1, 1, DISTANCE_TABLE_SIZE_50) + .unwrap_err(), + too_short + ); + assert_eq!( + estimated_match_cost(&repeated, 1, 1, DISTANCE_TABLE_SIZE_50).unwrap_err(), + too_short + ); + + let fresh = EncoderMatchState::default(); + let zero_distance = Error::InvalidData("RAR 5 match distance is zero"); + assert_eq!( + fresh + .encode_match(4, 0, DISTANCE_TABLE_SIZE_50) + .unwrap_err(), + zero_distance + ); + assert_eq!( + estimated_match_cost(&fresh, 4, 0, DISTANCE_TABLE_SIZE_50).unwrap_err(), + zero_distance + ); + let underflow = Error::InvalidData("RAR 5 adjusted match length underflows"); + assert_eq!( + fresh + .encode_match(2, 0x40001, DISTANCE_TABLE_SIZE_50) + .unwrap_err(), + underflow + ); + assert_eq!( + estimated_match_cost(&fresh, 2, 0x40001, DISTANCE_TABLE_SIZE_50).unwrap_err(), + underflow + ); + + #[cfg(target_pointer_width = "64")] + { + let too_distant = Error::InvalidData("RAR 5 match distance is too large"); + assert_eq!( + estimated_match_cost(&fresh, 4, 1usize << 40, DISTANCE_TABLE_SIZE_50).unwrap_err(), + too_distant + ); + let invalid_slot = Error::InvalidData("RAR 5 distance slot is too large"); + assert_eq!( + fresh + .encode_match(5, (1usize << 33) + 1, DISTANCE_TABLE_SIZE_70) + .unwrap_err(), + invalid_slot + ); + assert_eq!( + estimated_match_cost(&fresh, 5, (1usize << 33) + 1, DISTANCE_TABLE_SIZE_70) + .unwrap_err(), + invalid_slot + ); + } +} + +#[test] +fn table_construction_and_stream_pricing_reject_invalid_tokens() { + let invalid = [EncodeToken::Match { + length: 1, + distance: 1, + }]; + assert_eq!( + table_lengths_for_tokens(&invalid, DISTANCE_TABLE_SIZE_50).err(), + Some(Error::InvalidData("RAR 5 match length is too short")) + ); + + let lengths = + table_lengths_for_tokens(&[EncodeToken::Literal(b'A')], DISTANCE_TABLE_SIZE_50).unwrap(); + assert_eq!( + token_stream_bits(&invalid, &[], &lengths, DISTANCE_TABLE_SIZE_50), + Err(Error::InvalidData("RAR 5 match length is too short")) + ); +} + +#[test] +fn optimal_match_collection_grows_beyond_one_run_per_position() { + let mut noise = 0x2545_f491_4f6c_dd1du64; + let data: Vec = (0..4096) + .map(|_| { + noise ^= noise << 13; + noise ^= noise >> 7; + noise ^= noise << 17; + b'A' + ((noise >> 40) as u8 & 1) + }) + .collect(); + let options = EncodeOptions::new(64).with_optimal_parse(true); + let baseline = RefusingBudget::new(usize::MAX); + let mut collector = OptimalCollector::with_allowance(&data, 0, options, &baseline).unwrap(); + let matches = collector.collect(&data, 0..data.len(), options).unwrap(); + assert!(matches.runs.len() > data.len()); + drop(matches); + drop(collector); + assert_eq!(baseline.used(), 0); + + assert_each_allocation_refusal(|budget| { + let mut collector = OptimalCollector::with_allowance(&data, 0, options, budget)?; + collector.collect(&data, 0..data.len(), options) + }); + + let start = data.len() / 2; + let baseline = RefusingBudget::new(usize::MAX); + let mut collector = OptimalCollector::with_allowance(&data, start, options, &baseline).unwrap(); + let matches = collector + .collect(&data, start..data.len(), options) + .unwrap(); + assert!(matches.runs.len() > data.len() - start); + drop(matches); + drop(collector); + assert_eq!(baseline.used(), 0); + + assert_each_allocation_refusal(|budget| { + let mut collector = OptimalCollector::with_allowance(&data, start, options, budget)?; + collector.collect(&data, start..data.len(), options) + }); +} + +#[test] +fn distance_slots_beyond_native_address_width_use_out_of_window_sentinel() { + for (slot, extra, distance) in [ + (63, (1u32 << 30) - 1, 1u64 << 32), + (65, (1u32 << 31) - 1, 1u64 << 33), + ] { + assert_eq!( + slot_to_distance(slot, extra).unwrap(), + usize::try_from(distance).unwrap_or(usize::MAX) + ); + } +} + +#[test] +fn rejects_out_of_range_distance_slots_and_extras() { + #[cfg(target_pointer_width = "64")] + assert!(slot_to_distance(65, (1 << 31) - 1).is_ok()); + assert_eq!( + slot_to_distance(4, 2), + Err(Error::InvalidData("RAR 5 distance extra bits exceed slot")) + ); + assert_eq!( + slot_to_distance(66, 0), + Err(Error::InvalidData("RAR 5 distance slot is too large")) + ); +} + +#[test] +fn bit_reader_accepts_large_rar5_distance_extras() { + let mut bits = BitReader::new(&[0xff, 0x00, 0xaa, 0x55]); + + assert_eq!(bits.read_bits(32).unwrap(), 0xff00_aa55); + assert_eq!( + bits.read_bits(1), + Err(Error::NeedMoreInput), + "32-bit reads must not leave a partial cursor state" + ); +} + +#[test] +fn copies_lz_matches_with_overlap() { + let decoder = Unpack50Decoder::new(); + let mut output = b"AB".to_vec(); + + decoder + .copy_match(&mut output, 2, 6, 8, DEFAULT_DICTIONARY_SIZE) + .unwrap(); + + assert_eq!(output, b"ABABABAB"); +} + +#[test] +fn zero_fills_a_match_that_reaches_past_the_window() { + let decoder = Unpack50Decoder::new(); + let mut output = b"AB".to_vec(); + + decoder + .copy_match(&mut output, 3, 1, 3, DEFAULT_DICTIONARY_SIZE) + .unwrap(); + + assert_eq!(output, b"AB\0"); +} + +#[test] +fn zero_fills_a_zero_distance_match_in_both_outputs() { + let decoder = Unpack50Decoder::new(); + let mut buffered = b"A".to_vec(); + decoder + .copy_match(&mut buffered, 0, 3, 4, DEFAULT_DICTIONARY_SIZE) + .unwrap(); + assert_eq!(buffered, b"A\0\0\0"); + + let mut streaming = StreamingOutput::new(Buffer::from_vec(b"A".to_vec()), 3, 1, 1).unwrap(); + let mut decoded = Vec::new(); + streaming + .copy_match(0, 3, &mut |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }) + .unwrap(); + streaming + .finish(&mut |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }) + .unwrap(); + assert_eq!(decoded, b"\0\0\0"); +} + +#[test] +fn rejects_a_match_that_runs_past_the_output_limit() { + let decoder = Unpack50Decoder::new(); + let mut output = b"AB".to_vec(); + + assert_eq!( + decoder.copy_match(&mut output, 1, 2, 3, DEFAULT_DICTIONARY_SIZE), + Err(Error::InvalidData("RAR 5 match exceeds output limit")) + ); + assert_eq!(output, b"AB"); +} + +#[test] +fn zero_fills_a_match_distance_beyond_the_dictionary() { + let decoder = Unpack50Decoder::new(); + let mut output = b"ABCD".to_vec(); + + decoder.copy_match(&mut output, 4, 1, 5, 3).unwrap(); + + assert_eq!(output, b"ABCD\0"); +} + +#[test] +fn buffered_reader_history_capacity_follows_the_active_dictionary() { + let data = b"ABBA".repeat(4096); + let packed = encode_literal_only(&data, 0).unwrap(); + let mut decoder = Unpack50Decoder::new(); + assert_eq!( + decoder + .decode_member_with_dictionary( + &packed, + 0, + data.len(), + 1024, + false, + DecodeMode::LiteralOnly + ) + .unwrap(), + data + ); + assert_eq!(&*decoder.state.history, &data[data.len() - 1024..][..]); + assert!( + decoder.state.history.capacity() <= 1024, + "retained {} bytes for a 1024-byte dictionary", + decoder.state.history.capacity() + ); + let next = b"next solid member"; + let packed = encode_literal_only(next, 0).unwrap(); + decoder + .decode_member_with_dictionary(&packed, 0, next.len(), 1024, true, DecodeMode::LiteralOnly) + .unwrap(); + let expected = [&data[data.len() - (1024 - next.len())..], next.as_slice()].concat(); + assert_eq!(decoder.state.history, expected); + assert!(decoder.state.history.capacity() <= 1024); + decoder + .decode_member_with_dictionary(&packed, 0, next.len(), 8, true, DecodeMode::LiteralOnly) + .unwrap(); + assert_eq!(&*decoder.state.history, &next[next.len() - 8..][..]); + assert!(decoder.state.history.capacity() <= 8); +} + +#[test] +fn filtered_reader_history_retains_only_the_raw_dictionary_tail() { + let data = b"\xe8\0\0\0\0abcdefgh".repeat(256); + let packed = encode_lz_member_with_filter(&data, crate::FilterKind::E8).unwrap(); + let raw = Unpack50Decoder::new() + .decode_member_with_dictionary(&packed, 0, data.len(), 64, false, DecodeMode::LzNoFilters) + .unwrap(); + let mut decoder = Unpack50Decoder::new(); + let decoded = decoder + .decode_member_with_dictionary(&packed, 0, data.len(), 64, false, DecodeMode::Lz) + .unwrap(); + assert_ne!(decoded, raw, "fixture must actually transform bytes"); + assert_eq!(&*decoder.state.history, &raw[raw.len() - 64..][..]); + assert!(decoder.state.history.capacity() <= 64); +} + +#[test] +fn failed_buffered_filter_preserves_previous_solid_history() { + let data = b"\xe8\0\0\0\0abcdefghijklmnop".to_vec(); + let packed = encode_lz_member_with_filter(&data, crate::FilterKind::E8).unwrap(); + let mut decoder = Unpack50Decoder::new(); + decoder.state.history = Buffer::from_vec(b"old raw history".to_vec()); + let history = decoder.state.history.to_vec(); + // The record spans the full member, but the advertised output stops + // one byte earlier: the failure happens during filter application. + assert_eq!( + decoder.decode_member_with_dictionary(&packed, 0, data.len() - 1, 64, true, DecodeMode::Lz), + Err(Error::InvalidData("RAR 5 filter range exceeds output")) + ); + assert_eq!(decoder.state.history, history); +} + +#[test] +fn solid_history_is_capped_to_dictionary_size() { + let mut decoder = Unpack50Decoder::new(); + let first_payload = literal_only_payload(b"ABBA"); + let first = + encode_compressed_block(&first_payload, first_payload.len() * 8, true, true).unwrap(); + let second_payload = literal_only_payload(b"BAAB"); + let second = + encode_compressed_block(&second_payload, second_payload.len() * 8, true, true).unwrap(); + + assert_eq!( + decoder + .decode_member_with_dictionary(&first, 0, 4, 6, false, DecodeMode::LiteralOnly) + .unwrap(), + b"ABBA" + ); + assert_eq!(&*decoder.state.history, &b"ABBA"[..]); + + assert_eq!( + decoder + .decode_member_with_dictionary(&second, 0, 4, 6, true, DecodeMode::LiteralOnly) + .unwrap(), + b"BAAB" + ); + assert_eq!(&*decoder.state.history, &b"BABAAB"[..]); +} + +#[test] +fn streaming_reader_history_capacity_follows_the_active_dictionary() { + let data = b"ABBA".repeat(65536); + let packed = encode_literal_only(&data, 0).unwrap(); + let mut decoder = Unpack50Decoder::new(); + let mut decoded = Vec::new(); + decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut packed.as_slice(), + 0, + data.len(), + 1024, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)) + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + assert_eq!(decoded, data); + assert_eq!(&*decoder.state.history, &data[data.len() - 1024..][..]); + assert!(decoder.state.history.capacity() <= 1024); + let next = b"next solid member"; + let packed = encode_literal_only(next, 0).unwrap(); + decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut packed.as_slice(), + 0, + next.len(), + 8, + true, + |_| Ok::<(), std::convert::Infallible>(()), + ) + .unwrap(); + assert_eq!(&*decoder.state.history, &next[next.len() - 8..][..]); + assert!(decoder.state.history.capacity() <= 8); +} + +#[test] +fn streaming_decoder_history_is_capped_without_reordering() { + let mut decoder = Unpack50Decoder::new(); + let first_payload = literal_only_payload(b"ABBA"); + let first = + encode_compressed_block(&first_payload, first_payload.len() * 8, true, true).unwrap(); + let second_payload = literal_only_payload(b"BAAB"); + let second = + encode_compressed_block(&second_payload, second_payload.len() * 8, true, true).unwrap(); + let mut decoded = Vec::new(); + + decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut std::io::Cursor::new(&first), + 0, + 4, + 6, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::io::Error>(()) + }, + ) + .unwrap(); + assert_eq!(decoded, b"ABBA"); + assert_eq!(&*decoder.state.history, &b"ABBA"[..]); + + decoded.clear(); + decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut std::io::Cursor::new(&second), + 0, + 4, + 6, + true, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::io::Error>(()) + }, + ) + .unwrap(); + assert_eq!(decoded, b"BAAB"); + assert_eq!(&*decoder.state.history, &b"BABAAB"[..]); +} + +#[test] +fn solid_streaming_decoder_trims_inherited_history_before_member() { + let payload = literal_only_payload(b"AB"); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + let mut decoder = Unpack50Decoder::new(); + decoder.state.history.extend_from_slice(b"123456").unwrap(); + let mut decoded = Vec::new(); + + decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 2, + 4, + true, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + + assert_eq!(decoded, b"AB"); + assert_eq!(&*decoder.state.history, &b"56AB"[..]); +} + +#[test] +fn streaming_decoder_handles_each_match_control() { + for (payload, expected) in [ + (new_match_payload(), b"ABAB".as_slice()), + (repeat_payload(257), b"ABABAB".as_slice()), + (repeat_payload(258), b"ABABAB".as_slice()), + ] { + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + let mut decoded = Vec::new(); + Unpack50Decoder::new() + .decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + expected.len(), + DEFAULT_DICTIONARY_SIZE, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + assert_eq!(decoded, expected); + } +} + +#[test] +fn buffered_and_streaming_decoders_reuse_tables_after_table_only_block() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[b'B' as usize] = 1; + let (tables, table_bits) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut input = encode_compressed_block(&tables, table_bits, true, false).unwrap(); + // A = 0, B = 1 with the two one-bit codes above. + input.extend(encode_compressed_block(&[0b0110_0000], 4, false, true).unwrap()); + + let buffered = Unpack50Decoder::new() + .decode_member_with_dictionary( + &input, + 0, + 4, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::LiteralOnly, + ) + .unwrap(); + assert_eq!(buffered, b"ABBA"); + + let mut streamed = Vec::new(); + Unpack50Decoder::new() + .decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 4, + DEFAULT_DICTIONARY_SIZE, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => streamed.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + streamed.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + assert_eq!(streamed, buffered); +} + +#[test] +fn compressed_block_reader_accepts_empty_final_marker_and_checks_header() { + let empty = encode_compressed_block(&[], 0, false, true).unwrap(); + let parsed = parse_compressed_block(&empty).unwrap(); + let streamed = read_compressed_block(&mut empty.as_slice()).unwrap(); + assert!(parsed.header.is_last); + assert_eq!(parsed.header.payload_bits, 0); + assert!(streamed.header.is_last); + assert_eq!(streamed.header.payload_bits, 0); + assert!(streamed.payload.is_empty()); + + assert!(matches!( + read_compressed_block(&mut empty[..2].as_ref()), + Err(Error::NeedMoreInput) + )); + + let mut bad_checksum = empty.clone(); + bad_checksum[1] ^= 1; + assert!(matches!( + read_compressed_block(&mut bad_checksum.as_slice()), + Err(Error::InvalidData("RAR 5 block header checksum mismatch")) + )); + assert_eq!( + parse_compressed_block(&bad_checksum), + Err(Error::InvalidData("RAR 5 block header checksum mismatch")) + ); + let mut invalid_size_width = empty; + invalid_size_width[0] |= 0b11 << 3; + assert!(matches!( + read_compressed_block(&mut invalid_size_width.as_slice()), + Err(Error::InvalidData("RAR 5 block size length is invalid")) + )); + assert_eq!( + parse_compressed_block(&invalid_size_width), + Err(Error::InvalidData("RAR 5 block size length is invalid")) + ); +} + +#[test] +fn parser_handles_largest_declared_block_size_without_overflow() { + let flags = 0b10 << 3; // three size bytes + let checksum = 0x5a ^ flags ^ 0xff ^ 0xff ^ 0xff; + let header = [flags, checksum, 0xff, 0xff, 0xff]; + assert_eq!(parse_compressed_block(&header), Err(Error::NeedMoreInput)); +} + +#[test] +fn reader_block_admits_capacity_before_reading_payload_and_releases_failures() { + let packed = encode_compressed_block(&[0x31; 8], 64, false, true).unwrap(); + let header_len = parse_compressed_block(&packed).unwrap().header_len; + let denied = Allowance::limited(7); + let mut input = std::io::Cursor::new(&packed); + let error = match read_compressed_block_with_allowance(&mut input, &denied) { + Ok(_) => panic!("payload must exceed the allowance"), + Err(error) => error, + }; + assert!(matches!(error, Error::WorkspaceLimitExceeded(ref details) + if details.limit == 7 && details.required == 8 && details.used == 0)); + assert_eq!(input.position(), header_len as u64); + assert_eq!(denied.used(), 0); + + let allowance = Allowance::limited(8); + let block = read_compressed_block_with_allowance(&mut packed.as_slice(), &allowance).unwrap(); + assert_eq!(&*block.payload, &[0x31; 8]); + assert_eq!(allowance.used(), 8); + drop(block); + assert_eq!(allowance.used(), 0); + + struct FailingPayload<'a> { + header: &'a [u8], + } + impl Read for FailingPayload<'_> { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + if self.header.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "payload read denied", + )); + } + self.header.read(output) + } + } + let mut failing = FailingPayload { + header: &packed[..header_len], + }; + let error = match read_compressed_block_with_allowance(&mut failing, &allowance) { + Ok(_) => panic!("payload read must fail"), + Err(error) => error, + }; + assert!(matches!(error, Error::Io(ref source) + if matches!(**source, crate::Error::Io(ref io) + if io.kind == std::io::ErrorKind::PermissionDenied))); + assert_eq!(allowance.used(), 0); + + let mut truncated = &packed[..packed.len() - 1]; + assert!(matches!( + read_compressed_block_with_allowance(&mut truncated, &allowance), + Err(Error::NeedMoreInput) + )); + assert_eq!(allowance.used(), 0); +} + +#[test] +fn reader_block_keeps_its_worker_charge_after_reservation_retirement() { + use crate::codec::workspace::RESERVATION_BYTES; + let packed = encode_compressed_block(&[0x31; 8], 64, false, true).unwrap(); + let ledger = Allowance::limited(128 + RESERVATION_BYTES); + let mut reservation = ledger.reserve(16).unwrap(); + let allowance = reservation.allowance(); + reservation.start(); + let block = read_compressed_block_with_allowance(&mut packed.as_slice(), &allowance).unwrap(); + let larger = encode_compressed_block(&[0x31; 9], 72, false, true).unwrap(); + assert!(matches!( + read_compressed_block_with_allowance(&mut larger.as_slice(), &allowance), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(allowance.used(), 8); + reservation.retire(); + assert_eq!(ledger.used(), 8 + RESERVATION_BYTES); + drop(block); + assert_eq!(ledger.used(), RESERVATION_BYTES); + drop(allowance); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn parser_rejects_truncated_block_header_and_payload() { + // A two-byte size field requires a fourth header byte. + assert_eq!( + parse_compressed_block(&[1 << 3, 0, 0]), + Err(Error::NeedMoreInput) + ); + let block = encode_compressed_block(&[0xaa, 0xbb], 16, false, true).unwrap(); + for length in 0..block.len() { + assert_eq!( + parse_compressed_block(&block[..length]), + Err(Error::NeedMoreInput), + "truncated at byte {length}" + ); + } + assert_eq!( + parse_compressed_block(&block).unwrap().payload, + 3..block.len() + ); +} + +#[test] +fn decoders_handle_uninitialized_match_controls() { + for (symbol, buffered_error, streaming_error) in [ + (257, Error::NeedMoreInput, Error::NeedMoreInput), + ( + 258, + Error::InvalidData("RAR 5 repeat distance is not initialized"), + Error::InvalidData("RAR 5 repeat distance is not initialized"), + ), + ] { + let input = control_only_block(symbol); + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(buffered_error) + ); + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!( + matches!(result, Err(StreamDecodeError::Decode(error)) if error == streaming_error) + ); + } +} + +#[test] +fn decoders_reject_match_that_exceeds_declared_output() { + for (payload, declared_size) in [ + (new_match_payload(), 3), // AB, then a two-byte match. + (repeat_payload(257), 5), // ABAB, then the previous match. + (repeat_payload(258), 5), // ABAB, then a repeat distance. + ] { + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + declared_size, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(Error::InvalidData("RAR 5 match exceeds output limit")) + ); + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + declared_size, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 match exceeds output limit" + ))) + )); + } +} + +#[test] +fn literal_only_decoder_rejects_every_lz_control_class() { + for symbol in [256, 257, 258, 262] { + let input = control_only_block(symbol); + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::LiteralOnly, + ), + Err(Error::InvalidData( + "RAR 5 literal-only decoder encountered non-literal symbol" + )), + "control symbol {symbol}" + ); + } +} + +#[test] +fn decoder_defaults_match_fresh_state() { + assert_eq!(BlockSplitter::default().counts, BlockSplitter::new().counts); + assert_eq!(BlockSplitter::default().total, BlockSplitter::new().total); + + let default = Unpack50Decoder::default(); + let fresh = Unpack50Decoder::new(); + assert!(default.state.tables.is_none()); + assert!(fresh.state.tables.is_none()); + assert_eq!(default.state.reps, fresh.state.reps); + assert_eq!(default.state.last_length, fresh.state.last_length); + assert_eq!(default.state.history, fresh.state.history); +} + +#[test] +fn decoders_reject_unsupported_filter_type() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[256] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + writer.write_bits(1, 1); // filter control symbol + writer.write_bits(0, 2); // one-byte offset + writer.write_bits(0, 8); + writer.write_bits(0, 2); // one-byte length + writer.write_bits(1, 8); + writer.write_bits(7, 3); // unsupported filter type + let payload_bits = writer.bit_pos; + let input = encode_compressed_block(&writer.finish(), payload_bits, true, true).unwrap(); + + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(Error::InvalidData("RAR 5 filter type is unsupported")) + ); + let result = Unpack50Decoder::new().decode_to_sink_with_filters( + &mut input.as_slice(), + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + Some(&mut |_filter| Ok::<(), std::convert::Infallible>(())), + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 filter type is unsupported" + ))) + )); +} + +#[test] +fn streaming_decoder_rejects_missing_tables_and_invalid_dictionary() { + let input = encode_compressed_block(&[0], 8, false, true).unwrap(); + assert_eq!( + Unpack50Decoder::new().decode_member_with_dictionary( + &input, + 0, + 1, + 0, + false, + DecodeMode::LiteralOnly, + ), + Err(Error::InvalidData("RAR 5 dictionary size is zero")) + ); + let decode = |dictionary_size| { + Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 1, + dictionary_size, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ) + }; + assert!(matches!( + decode(1), + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 block reuses missing tables" + ))) + )); + assert!(matches!( + decode(0), + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 dictionary size is zero" + ))) + )); +} + +#[test] +fn streaming_decoder_propagates_sink_failure() { + let payload = literal_only_payload(b"AB"); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + let error = Unpack50Decoder::new() + .decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 2, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Err("sink failed"), + ) + .unwrap_err(); + assert!(matches!(error, StreamDecodeError::Sink("sink failed"))); +} + +#[test] +fn streaming_decoder_propagates_sink_failure_during_literal_flush() { + let data = vec![b'A'; STREAM_FLUSH_THRESHOLD]; + let input = encode_literal_only(&data, 0).unwrap(); + let mut decoder = Unpack50Decoder::new(); + let mut calls = 0; + let error = decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + data.len(), + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| { + calls += 1; + Err("sink failed") + }, + ) + .unwrap_err(); + assert!(matches!(error, StreamDecodeError::Sink("sink failed"))); + assert_eq!(calls, 1); + assert!(decoder.state.tables.is_none()); // The flush failed inside the literal loop. +} + +#[test] +fn streaming_decoder_observes_cancellation_after_literal_flush() { + let data = vec![b'A'; STREAM_FLUSH_THRESHOLD + 1]; + let input = encode_literal_only(&data, 0).unwrap(); + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack50Decoder::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + let mut calls = 0; + + let result = decoder.decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + data.len(), + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| { + calls += 1; + token.cancel(); + Ok::<(), std::convert::Infallible>(()) + }, + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::Cancelled)) + )); + assert_eq!(calls, 1); + assert!(decoder.state.tables.is_none()); // The next symbol observed cancellation. +} + +#[test] +fn streaming_match_output_propagates_sink_failures() { + let mut repeated = StreamingOutput::new( + Buffer::new(&Allowance::default()), + STREAM_FLUSH_THRESHOLD, + 2, + 2, + ) + .unwrap(); + assert!(matches!( + repeated.push_repeated(b'A', STREAM_FLUSH_THRESHOLD, &mut |_chunk| Err("sink")), + Err(StreamDecodeError::Sink("sink")) + )); + + let mut zero_flush = StreamingOutput::new(Buffer::new(&Allowance::default()), 2, 2, 2).unwrap(); + zero_flush + .push(b'A', &mut |_chunk| Ok::<(), &str>(())) + .unwrap(); + assert!(matches!( + zero_flush.push_zeroes(1, &mut |_chunk| Err("sink")), + Err(StreamDecodeError::Sink("sink")) + )); + + let mut zero_chunk = StreamingOutput::new(Buffer::from_vec(vec![0]), 1, 2, 2).unwrap(); + assert!(matches!( + zero_chunk.push_zeroes(1, &mut |_chunk| Err("sink")), + Err(StreamDecodeError::Sink("sink")) + )); + + let mut copied = StreamingOutput::new( + Buffer::new(&Allowance::default()), + STREAM_FLUSH_THRESHOLD, + 2, + 2, + ) + .unwrap(); + copied + .push_repeated(b'A', STREAM_FLUSH_THRESHOLD - 1, &mut |_chunk| { + Ok::<(), &str>(()) + }) + .unwrap(); + assert!(matches!( + copied.copy_match(2, 1, &mut |_chunk| Err("sink")), + Err(StreamDecodeError::Sink("sink")) + )); +} + +#[test] +fn decoder_reader_entry_points_observe_existing_cancellation() { + let token = crate::ReadCancellation::new(); + token.cancel(); + let control = crate::read_control::ReadControl::new(Some(&token)); + let mut buffered = Unpack50Decoder::new(); + buffered.read_control = control.clone(); + assert_eq!( + buffered.decode_member_from_reader_with_dictionary( + &mut &[][..], + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(Error::Cancelled) + ); + + let mut streaming = Unpack50Decoder::new(); + streaming.read_control = control; + let result = streaming.decode_member_from_reader_with_dictionary_to_sink( + &mut &[][..], + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ); + assert!(matches!( + result, + Err(StreamDecodeError::Decode(Error::Cancelled)) + )); +} + +#[test] +fn buffered_decoder_wrappers_observe_existing_cancellation() { + let token = crate::ReadCancellation::new(); + token.cancel(); + let control = crate::read_control::ReadControl::new(Some(&token)); + let mut decoder = Unpack50Decoder::new(); + decoder.read_control = control.clone(); + assert_eq!( + decoder.decode_member(&[], 0, 1, false, DecodeMode::Lz), + Err(Error::Cancelled) + ); + + decoder.read_control = control.clone(); + assert_eq!( + decoder.decode_member_with_dictionary( + &[], + 0, + 1, + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(Error::Cancelled) + ); + + decoder.read_control = control; + assert_eq!( + decoder.decode_member_from_reader(&mut &[][..], 0, 1, false, DecodeMode::Lz), + Err(Error::Cancelled) + ); +} + +#[test] +fn buffered_decoder_observes_cancellation_before_and_during_filters() { + let data: Vec = (0..96).map(|index| (index * 7 + index / 3) as u8).collect(); + let input = + encode_lz_member_with_filter(&data, crate::FilterKind::Delta { channels: 3 }).unwrap(); + assert_eq!(decode_lz(&input, 0, data.len()).unwrap(), data); + + for successful_checks in [5, 6] { + let token = crate::ReadCancellation::new(); + let mut decoder = Unpack50Decoder::new(); + decoder.read_control = crate::read_control::ReadControl::new(Some(&token)); + decoder.read_control.cancel_after_checks(successful_checks); + assert_eq!( + decoder.decode_member_from_reader_with_dictionary( + &mut input.as_slice(), + 0, + data.len(), + DEFAULT_DICTIONARY_SIZE, + false, + DecodeMode::Lz, + ), + Err(Error::Cancelled), + "after {successful_checks} checks" + ); + assert!(decoder.state.tables.is_some()); // All packed symbols decoded first. + assert!(decoder.state.history.is_empty()); // Filtered output was not committed. + } +} + +#[test] +fn streaming_decoder_rejects_filter_beyond_declared_output() { + let data = b"\xe8\0\0\0\0plain text after call"; + let input = encode_lz_member_with_filter(data, crate::FilterKind::E8).unwrap(); + let mut record_count = 0; + let error = Unpack50Decoder::new() + .decode_to_sink_with_filters( + &mut input.as_slice(), + 0, + data.len() - 1, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + Some(&mut |_filter| { + record_count += 1; + Ok::<(), std::convert::Infallible>(()) + }), + ) + .unwrap_err(); + assert!(matches!( + error, + StreamDecodeError::Decode(Error::InvalidData("RAR 5 filter range exceeds output")) + )); + assert_eq!(record_count, 0); +} + +#[test] +fn streaming_decoder_propagates_filter_record_failure() { + let data = b"\xe8\0\0\0\0plain text after call"; + let input = encode_lz_member_with_filter(data, crate::FilterKind::E8).unwrap(); + let error = Unpack50Decoder::new() + .decode_to_sink_with_filters( + &mut input.as_slice(), + 0, + data.len(), + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), &str>(()), + Some(&mut |_filter| Err("filter failed")), + ) + .unwrap_err(); + assert!(matches!(error, StreamDecodeError::Sink("filter failed"))); +} + +#[test] +fn streaming_decoder_reports_truncated_block_and_incomplete_output() { + let payload = literal_only_payload(b"AB"); + let input = encode_compressed_block(&payload, payload.len() * 8, true, true).unwrap(); + for truncated in [&input[..1], &input[..input.len() - 1]] { + let mut reader = truncated; + let error = Unpack50Decoder::new() + .decode_member_from_reader_with_dictionary_to_sink( + &mut reader, + 0, + 2, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ) + .unwrap_err(); + assert!(matches!( + error, + StreamDecodeError::Decode(Error::NeedMoreInput) + )); + } + let error = Unpack50Decoder::new() + .decode_member_from_reader_with_dictionary_to_sink( + &mut input.as_slice(), + 0, + 3, + DEFAULT_DICTIONARY_SIZE, + false, + |_chunk| Ok::<(), std::convert::Infallible>(()), + ) + .unwrap_err(); + assert!(matches!( + error, + StreamDecodeError::Decode(Error::NeedMoreInput) + )); +} + +#[test] +fn streaming_output_rejects_overruns_without_partial_emission() { + let emitted = std::cell::RefCell::new(Vec::new()); + let mut sink = |chunk: DecodedChunk<'_>| { + match chunk { + DecodedChunk::Bytes(bytes) => emitted.borrow_mut().extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + emitted.borrow_mut().extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }; + + let mut literal = StreamingOutput::new(Buffer::new(&Allowance::default()), 1, 1, 1).unwrap(); + literal.push(b'A', &mut sink).unwrap(); + assert!(matches!( + literal.push(b'B', &mut sink), + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 match exceeds output limit" + ))) + )); + literal.finish(&mut sink).unwrap(); + assert_eq!(&*emitted.borrow(), b"A"); + + emitted.borrow_mut().clear(); + let mut repeated = StreamingOutput::new(Buffer::new(&Allowance::default()), 1, 1, 1).unwrap(); + assert!(matches!( + repeated.push_repeated(b'B', 2, &mut sink), + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 match exceeds output limit" + ))) + )); + assert_eq!(repeated.written(), 0); + assert!(emitted.borrow().is_empty()); + + let mut zeroes = StreamingOutput::new(Buffer::from_vec(vec![0]), 1, 1, 1).unwrap(); + assert!(matches!( + zeroes.push_zeroes(2, &mut sink), + Err(StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 match exceeds output limit" + ))) + )); + assert_eq!(zeroes.written(), 0); + assert!(emitted.borrow().is_empty()); + zeroes.push_zeroes(1, &mut sink).unwrap(); + assert_eq!(&*emitted.borrow(), &[0]); + assert_eq!(zeroes.into_history().into_vec(), [0]); +} + +#[test] +fn virtual_zero_history_does_not_overallocate_a_tiny_dictionary() { + let mut output = StreamingOutput::new(Buffer::from_vec(vec![0]), 100_000, 1, 1).unwrap(); + let mut emitted = 0; + output + .copy_match(1, 100_000, &mut |chunk| { + let DecodedChunk::Repeated { byte: 0, len } = chunk else { + panic!("expected virtual zeros") + }; + emitted += len; + Ok::<(), std::convert::Infallible>(()) + }) + .unwrap(); + assert_eq!(emitted, 100_000); + assert_eq!(output.history.iter().copied().collect::>(), [0]); + assert_eq!(output.history.capacity(), 1); +} + +#[test] +fn virtual_zero_history_flush_refusal_releases_streaming_charge() { + let budget = RefusingBudget::new(1); + { + let mut output = StreamingOutput::new(Buffer::new(&budget), 3, 1, 1).unwrap(); + output + .push(0, &mut |_| Ok::<(), std::convert::Infallible>(())) + .unwrap(); + let mut emitted = Vec::new(); + let error = output + .copy_match(1, 2, &mut |chunk| { + let DecodedChunk::Bytes(bytes) = chunk else { + panic!("expected initial literal") + }; + emitted.extend_from_slice(bytes); + Ok::<(), std::convert::Infallible>(()) + }) + .unwrap_err(); + assert!(matches!(error, StreamDecodeError::Decode(Error::Cancelled))); + // The initial literal is streamed before retaining its history fails. + assert_eq!(emitted, [0]); + assert_eq!(output.written(), 1); + assert!(output.history.is_empty()); + assert_eq!(budget.attempts(), 2); + } + assert_eq!(budget.used(), 0); +} + +#[test] +fn streaming_flush_preserves_dictionary_tail_with_excess_allocation_capacity() { + let mut output = StreamingOutput::new(Buffer::from_vec(b"ABC".to_vec()), 1, 3, 3).unwrap(); + // Model an allocator returning more capacity than requested. Vec's + // public contract permits this even though this host allocates exactly. + let mut history = Vec::with_capacity(8); + history.extend_from_slice(b"ABC"); + output.history = crate::codec::workspace::Deque::from_buffer(Buffer::from_vec(history)); + assert!(output.history.capacity() > output.history_limit); + let mut emitted = Vec::new(); + let mut sink = |chunk: DecodedChunk<'_>| { + let DecodedChunk::Bytes(bytes) = chunk else { + panic!("expected bytes") + }; + emitted.extend_from_slice(bytes); + Ok::<(), std::convert::Infallible>(()) + }; + output.push(b'D', &mut sink).unwrap(); + output.finish(&mut sink).unwrap(); + assert_eq!(emitted, b"D"); + assert_eq!(output.history.iter().copied().collect::>(), b"BCD"); +} + +#[test] +fn initial_repeat_distance_zero_is_rejected_before_streaming_output() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[258] = 1; + lengths.length[0] = 1; + lengths.length[1] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + writer.write_bits(1, 1); // Repeat the still-zero first distance. + writer.write_bits(0, 1); // Length two. + let payload_bits = writer.bit_pos; + let packed = encode_compressed_block(&writer.finish(), payload_bits, true, true).unwrap(); + assert_eq!( + decode_lz(&packed, 0, 2), + Err(Error::InvalidData( + "RAR 5 repeat distance is not initialized" + )) + ); + let mut decoder = Unpack50Decoder::new(); + let mut emitted = Vec::new(); + let error = decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut packed.as_slice(), + 0, + 2, + 1, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => emitted.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + emitted.extend(std::iter::repeat_n(byte, len)) + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap_err(); + assert!(matches!( + error, + StreamDecodeError::Decode(Error::InvalidData( + "RAR 5 repeat distance is not initialized" + )) + )); + assert!(emitted.is_empty()); +} + +#[test] +fn zero_literal_and_initialized_matches_stream_virtual_zero_history() { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[0] = 1; + lengths.main[258] = 2; + lengths.main[262] = 2; + lengths.distance[0] = 1; + lengths.distance[1] = 1; + lengths.length[0] = 1; + lengths.length[1] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + writer.write_bits(0, 1); // Zero literal. + writer.write_bits(3, 2); // New match: length two. + writer.write_bits(0, 1); // Distance one. + writer.write_bits(2, 2); // Repeat distance one. + writer.write_bits(0, 1); // Length two. + let payload_bits = writer.bit_pos; + let packed = encode_compressed_block(&writer.finish(), payload_bits, true, true).unwrap(); + assert_eq!(decode_lz(&packed, 0, 5).unwrap(), [0; 5]); + let mut decoder = Unpack50Decoder::new(); + let mut bytes = Vec::new(); + let mut virtual_bytes = 0; + decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut packed.as_slice(), + 0, + 5, + 1, + false, + |chunk| { + match chunk { + DecodedChunk::Bytes(literals) => bytes.extend_from_slice(literals), + DecodedChunk::Repeated { byte, len } => { + assert_eq!(byte, 0); + virtual_bytes += len; + bytes.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }, + ) + .unwrap(); + assert_eq!(bytes, [0; 5]); + assert_eq!(virtual_bytes, 4); +} + +#[test] +fn streaming_zero_history_emits_large_match_without_materializing_it() { + let mut output = StreamingOutput::new(Buffer::from_vec(vec![0, 0]), 100_000, 2, 2).unwrap(); + let mut chunks = Vec::new(); + output + .copy_match(2, 100_000, &mut |chunk| { + chunks.push(match chunk { + DecodedChunk::Bytes(bytes) => (bytes[0], bytes.len()), + DecodedChunk::Repeated { byte, len } => (byte, len), + }); + Ok::<(), std::convert::Infallible>(()) + }) + .unwrap(); + assert_eq!(chunks, [(0, 100_000)]); + assert_eq!(output.written(), 100_000); + assert_eq!(output.into_history().into_vec(), [0, 0]); +} + +#[test] +fn streaming_match_keeps_its_window_across_flushes() { + let count = STREAM_FLUSH_THRESHOLD + 11; + let mut output = StreamingOutput::new(Buffer::from_vec(b"abc".to_vec()), count, 3, 3).unwrap(); + let mut decoded = Vec::new(); + let mut sink = |chunk: DecodedChunk<'_>| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::convert::Infallible>(()) + }; + output.copy_match(3, count, &mut sink).unwrap(); + output.finish(&mut sink).unwrap(); + assert_eq!( + decoded, + b"abc" + .iter() + .copied() + .cycle() + .take(count) + .collect::>() + ); + assert!(output.history.capacity() <= 3); + assert_eq!(output.into_history().into_vec(), b"abc"); +} + +#[test] +fn streaming_window_accepts_match_beyond_old_64_mib_cap() { + const OLD_STREAM_HISTORY_LIMIT: usize = 64 * 1024 * 1024; + let distance = OLD_STREAM_HISTORY_LIMIT + 1; + let history = vec![b'A'; distance]; + let mut output = + StreamingOutput::new(Buffer::from_vec(history), 1, distance, distance).unwrap(); + let mut decoded = Vec::new(); + + output + .copy_match(distance, 1, &mut |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::io::Error>(()) + }) + .unwrap(); + output + .finish(&mut |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::io::Error>(()) + }) + .unwrap(); + + assert_eq!(decoded, b"A"); +} + +#[test] +fn streaming_window_zero_fills_match_beyond_declared_dictionary() { + let mut output = StreamingOutput::new(Buffer::from_vec(vec![b'A'; 8]), 1, 7, 8).unwrap(); + let mut decoded = Vec::new(); + + output + .copy_match(8, 1, &mut |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::io::Error>(()) + }) + .unwrap(); + output + .flush(&mut |chunk| { + match chunk { + DecodedChunk::Bytes(bytes) => decoded.extend_from_slice(bytes), + DecodedChunk::Repeated { byte, len } => { + decoded.extend(std::iter::repeat_n(byte, len)); + } + } + Ok::<(), std::io::Error>(()) + }) + .unwrap(); + + assert_eq!(decoded, b"\0"); +} + +fn literal_only_payload(data: &[u8]) -> Vec { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[b'B' as usize] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + for &byte in data { + match byte { + b'A' => writer.write_bits(0, 1), + b'B' => writer.write_bits(1, 1), + _ => panic!("test helper only encodes A/B"), + } + } + writer.finish() +} + +fn new_match_payload() -> Vec { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 2; + lengths.main[b'B' as usize] = 2; + lengths.main[262] = 2; + lengths.distance[1] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + + writer.write_bits(0b00, 2); // 'A' + writer.write_bits(0b01, 2); // 'B' + writer.write_bits(0b10, 2); // match length 2 + writer.write_bits(0, 1); // distance slot 1 + writer.finish() +} + +fn repeat_payload(repeat_symbol: usize) -> Vec { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 2; + lengths.main[b'B' as usize] = 2; + lengths.main[repeat_symbol] = 2; + lengths.main[262] = 2; + lengths.distance[1] = 1; + lengths.length[0] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + + writer.write_bits(0b00, 2); // 'A' + writer.write_bits(0b01, 2); // 'B' + writer.write_bits(0b11, 2); // match length 2 + writer.write_bits(0, 1); // distance slot 1 + writer.write_bits(0b10, 2); // repeat control symbol + if repeat_symbol == 258 { + writer.write_bits(0, 1); // length slot 0 + } + writer.finish() +} + +fn control_only_block(symbol: usize) -> Vec { + let mut lengths = TableLengths { + main: vec![0; MAIN_TABLE_SIZE], + distance: vec![0; DISTANCE_TABLE_SIZE_50], + align: vec![0; ALIGN_TABLE_SIZE], + length: vec![0; LENGTH_TABLE_SIZE], + }; + lengths.main[b'A' as usize] = 1; + lengths.main[symbol] = 1; + let (bytes, bit_pos) = encode_table_lengths_with_bit_count(&lengths, 0).unwrap(); + let mut writer = BitWriter { + bytes: Buffer::from_vec(bytes), + bit_pos, + }; + writer.write_bits(1, 1); + let payload_bits = writer.bit_pos; + encode_compressed_block(&writer.finish(), payload_bits, true, true).unwrap() +} diff --git a/arca-rar/vendor/rars/src/codec/rarvm.rs b/arca-rar/vendor/rars/src/codec/rarvm.rs new file mode 100644 index 00000000..f1b5e683 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rarvm.rs @@ -0,0 +1,1093 @@ +use super::workspace::{Allowance, Budget, Buffer}; +use super::{Error, Result}; + +const MEMORY_SIZE: usize = 0x40000; +const MEMORY_MASK: u32 = 0x3ffff; +const GLOBAL_BASE: usize = 0x3c000; +const SYSTEM_GLOBAL_SIZE: usize = 64; +const MAX_USER_GLOBAL: usize = 0x2000 - SYSTEM_GLOBAL_SIZE; +const MAX_STATIC_DATA: usize = MEMORY_SIZE - GLOBAL_BASE; +const MAX_INSTRUCTIONS: usize = 25_000_000; +const FLAG_C: u32 = 1; +const FLAG_Z: u32 = 2; +const FLAG_S: u32 = 0x8000_0000; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Program { + pub static_data: Vec, + pub instructions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Instruction { + pub opcode: Opcode, + pub byte_mode: bool, + pub operands: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum Opcode { + Mov = 0, + Cmp = 1, + Add = 2, + Sub = 3, + Jz = 4, + Jnz = 5, + Inc = 6, + Dec = 7, + Jmp = 8, + Xor = 9, + And = 10, + Or = 11, + Test = 12, + Js = 13, + Jns = 14, + Jb = 15, + Jbe = 16, + Ja = 17, + Jae = 18, + Push = 19, + Pop = 20, + Call = 21, + Ret = 22, + Not = 23, + Shl = 24, + Shr = 25, + Sar = 26, + Neg = 27, + Pusha = 28, + Popa = 29, + Pushf = 30, + Popf = 31, + Movzx = 32, + Movsx = 33, + Xchg = 34, + Mul = 35, + Div = 36, + Adc = 37, + Sbb = 38, + Print = 39, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Operand { + Register(u8), + Immediate(u32), + RegisterIndirect(u8), + Indexed { register: u8, base: u32 }, + Absolute(u32), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Invocation<'a> { + pub input: &'a [u8], + pub regs: [u32; 7], + pub global_data: &'a [u8], + pub file_offset: u64, + pub exec_count: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExecutionResult { + pub output: Vec, + pub globals: Vec, + pub regs: [u32; 8], +} + +impl Program { + pub fn parse(blob: &[u8]) -> Result { + OwnedProgram::parse(blob, &Allowance::default()).map(OwnedProgram::into_public) + } + + pub fn execute(&self, invocation: Invocation<'_>) -> Result { + self.execute_with_control(invocation, &crate::read_control::ReadControl::default()) + } + + pub(crate) fn execute_with_control( + &self, + invocation: Invocation<'_>, + control: &crate::read_control::ReadControl, + ) -> Result { + control.check_codec()?; + let mut vm = Vm::with_allowance(self, invocation, &Allowance::default())?; + vm.run(self, control).map(OwnedExecutionResult::into_public) + } +} + +#[derive(Debug)] +pub(crate) struct OwnedProgram { + static_data: Buffer, + instructions: Buffer, B>, +} +#[derive(Debug)] +struct OwnedInstruction { + opcode: Opcode, + byte_mode: bool, + operands: Buffer, +} +impl OwnedProgram { + pub(crate) fn try_clone(&self) -> Result { + let allowance = self.static_data.allowance(); + Ok(Self { + static_data: Buffer::copied(&self.static_data, &allowance)?, + instructions: Buffer::try_collect( + self.instructions.iter().map(|instruction| { + Ok(OwnedInstruction { + opcode: instruction.opcode, + byte_mode: instruction.byte_mode, + operands: Buffer::copied(&instruction.operands, &allowance)?, + }) + }), + &allowance, + )?, + }) + } + pub(crate) fn execute_with_control( + &self, + invocation: Invocation<'_>, + control: &crate::read_control::ReadControl, + ) -> Result> { + control.check_codec()?; + let mut vm = Vm::with_allowance(self, invocation, &self.static_data.allowance())?; + vm.run(self, control) + } + pub(crate) fn parse(blob: &[u8], allowance: &B) -> Result { + if blob.is_empty() { + return Err(Error::InvalidData("RARVM program blob is empty")); + } + if blob.iter().fold(0u8, |acc, &byte| acc ^ byte) != 0 { + return Err(Error::InvalidData("RARVM program checksum mismatch")); + } + + let mut bits = BitReader::new(&blob[1..]); + let mut static_data = Buffer::new(allowance); + if bits.read_bit()? != 0 { + let size = bits + .read_vm_number()? + .checked_add(1) + .ok_or(Error::InvalidData("RARVM static data size overflows"))? + as usize; + if size > MAX_STATIC_DATA { + return Err(Error::InvalidData("RARVM static data is too large")); + } + static_data = Buffer::with_capacity(size, allowance)?; + for _ in 0..size { + static_data.push_admitted(bits.read_bits(8)? as u8); + } + } + + let mut instructions = Buffer::new(allowance); + while bits.remaining_bits() >= 8 { + match parse_instruction_with_allowance(&mut bits, instructions.len(), allowance) { + Ok(instruction) => instructions.try_push(instruction)?, + // Instruction decoding only performs bounded bit reads. An + // incomplete final instruction is ignored for compatibility. + Err(Error::NeedMoreInput | Error::InvalidData(_)) => break, + Err(error) => return Err(error), + } + } + + if instructions + .last() + .is_none_or(|instruction| !instruction.opcode.is_unconditional_control_transfer()) + { + instructions.try_push(OwnedInstruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Buffer::new(allowance), + })?; + } + + Ok(Self { + static_data, + instructions, + }) + } +} +#[cfg(test)] +impl From for OwnedProgram { + fn from(program: Program) -> Self { + Self { + static_data: program.static_data.into(), + instructions: program + .instructions + .into_iter() + .map(|instruction| OwnedInstruction { + opcode: instruction.opcode, + byte_mode: instruction.byte_mode, + operands: instruction.operands.into(), + }) + .collect::>() + .into(), + } + } +} +impl OwnedProgram { + fn into_public(self) -> Program { + Program { + static_data: self.static_data.into_vec(), + instructions: self + .instructions + .into_vec() + .into_iter() + .map(|instruction| Instruction { + opcode: instruction.opcode, + byte_mode: instruction.byte_mode, + operands: instruction.operands.into_vec(), + }) + .collect(), + } + } +} +struct InstructionRef<'a> { + opcode: Opcode, + byte_mode: bool, + operands: &'a [Operand], +} +trait ProgramCode { + fn static_data(&self) -> &[u8]; + fn instruction(&self, index: usize) -> Option>; + fn instruction_count(&self) -> usize; +} +impl ProgramCode for Program { + fn static_data(&self) -> &[u8] { + &self.static_data + } + fn instruction(&self, index: usize) -> Option> { + self.instructions + .get(index) + .map(|instruction| InstructionRef { + opcode: instruction.opcode, + byte_mode: instruction.byte_mode, + operands: &instruction.operands, + }) + } + fn instruction_count(&self) -> usize { + self.instructions.len() + } +} +impl ProgramCode for OwnedProgram { + fn static_data(&self) -> &[u8] { + &self.static_data + } + fn instruction(&self, index: usize) -> Option> { + self.instructions + .get(index) + .map(|instruction| InstructionRef { + opcode: instruction.opcode, + byte_mode: instruction.byte_mode, + operands: &instruction.operands, + }) + } + fn instruction_count(&self) -> usize { + self.instructions.len() + } +} +#[derive(Debug)] +pub(crate) struct OwnedExecutionResult { + pub(crate) output: Buffer, + pub(crate) globals: Buffer, + pub(crate) regs: [u32; 8], +} +impl OwnedExecutionResult { + fn into_public(self) -> ExecutionResult { + ExecutionResult { + output: self.output.into_vec(), + globals: self.globals.into_vec(), + regs: self.regs, + } + } +} + +impl Opcode { + fn from_encoded(value: u8) -> Self { + const OPCODES: [Opcode; 40] = [ + Opcode::Mov, + Opcode::Cmp, + Opcode::Add, + Opcode::Sub, + Opcode::Jz, + Opcode::Jnz, + Opcode::Inc, + Opcode::Dec, + Opcode::Jmp, + Opcode::Xor, + Opcode::And, + Opcode::Or, + Opcode::Test, + Opcode::Js, + Opcode::Jns, + Opcode::Jb, + Opcode::Jbe, + Opcode::Ja, + Opcode::Jae, + Opcode::Push, + Opcode::Pop, + Opcode::Call, + Opcode::Ret, + Opcode::Not, + Opcode::Shl, + Opcode::Shr, + Opcode::Sar, + Opcode::Neg, + Opcode::Pusha, + Opcode::Popa, + Opcode::Pushf, + Opcode::Popf, + Opcode::Movzx, + Opcode::Movsx, + Opcode::Xchg, + Opcode::Mul, + Opcode::Div, + Opcode::Adc, + Opcode::Sbb, + Opcode::Print, + ]; + // The short form is three bits and the long form is five bits plus + // eight, so the wire representation can only produce 0..=39. + OPCODES[usize::from(value)] + } + + fn operand_count(self) -> usize { + match self { + Self::Ret | Self::Pusha | Self::Popa | Self::Pushf | Self::Popf | Self::Print => 0, + Self::Jz + | Self::Jnz + | Self::Inc + | Self::Dec + | Self::Jmp + | Self::Js + | Self::Jns + | Self::Jb + | Self::Jbe + | Self::Ja + | Self::Jae + | Self::Push + | Self::Pop + | Self::Call + | Self::Not + | Self::Neg => 1, + Self::Mov + | Self::Cmp + | Self::Add + | Self::Sub + | Self::Xor + | Self::And + | Self::Or + | Self::Test + | Self::Shl + | Self::Shr + | Self::Sar + | Self::Movzx + | Self::Movsx + | Self::Xchg + | Self::Mul + | Self::Div + | Self::Adc + | Self::Sbb => 2, + } + } + + fn supports_byte_mode(self) -> bool { + matches!( + self, + Self::Mov + | Self::Cmp + | Self::Add + | Self::Sub + | Self::Inc + | Self::Dec + | Self::Xor + | Self::And + | Self::Or + | Self::Test + | Self::Not + | Self::Shl + | Self::Shr + | Self::Sar + | Self::Neg + | Self::Xchg + | Self::Mul + | Self::Div + | Self::Adc + | Self::Sbb + ) + } + + fn is_jump_or_call(self) -> bool { + matches!( + self, + Self::Jz + | Self::Jnz + | Self::Jmp + | Self::Js + | Self::Jns + | Self::Jb + | Self::Jbe + | Self::Ja + | Self::Jae + | Self::Call + ) + } + + fn is_unconditional_control_transfer(self) -> bool { + matches!(self, Self::Jmp | Self::Ret) + } +} + +fn parse_instruction_with_allowance( + bits: &mut BitReader<'_>, + instruction_index: usize, + allowance: &B, +) -> Result> { + let opcode = if bits.read_bit()? == 0 { + Opcode::from_encoded(bits.read_bits(3)? as u8) + } else { + Opcode::from_encoded(bits.read_bits(5)? as u8 + 8) + }; + let byte_mode = opcode.supports_byte_mode() && bits.read_bit()? != 0; + let mut operands = Buffer::with_capacity(opcode.operand_count(), allowance)?; + for operand_index in 0..opcode.operand_count() { + let mut operand = parse_operand(bits, byte_mode)?; + if operand_index == 0 && opcode.is_jump_or_call() { + if let Operand::Immediate(value) = operand { + operand = Operand::Immediate(remap_jump_target(value, instruction_index)); + } + } + operands.push_admitted(operand); + } + Ok(OwnedInstruction { + opcode, + byte_mode, + operands, + }) +} + +fn parse_operand(bits: &mut BitReader<'_>, byte_mode: bool) -> Result { + if bits.read_bit()? != 0 { + return Ok(Operand::Register(bits.read_bits(3)? as u8)); + } + if bits.read_bit()? == 0 { + return if byte_mode { + Ok(Operand::Immediate(bits.read_bits(8)?)) + } else { + Ok(Operand::Immediate(bits.read_vm_number()?)) + }; + } + if bits.read_bit()? == 0 { + return Ok(Operand::RegisterIndirect(bits.read_bits(3)? as u8)); + } + if bits.read_bit()? == 0 { + Ok(Operand::Indexed { + register: bits.read_bits(3)? as u8, + base: bits.read_vm_number()?, + }) + } else { + Ok(Operand::Absolute(bits.read_vm_number()?)) + } +} + +fn remap_jump_target(value: u32, instruction_index: usize) -> u32 { + if value >= 256 { + return value - 256; + } + + let mut distance = value as i64; + if distance >= 136 { + distance -= 264; + } else if distance >= 16 { + distance -= 8; + } else if distance >= 8 { + distance -= 16; + } + (instruction_index as i64).wrapping_add(distance) as u32 +} + +struct Vm { + memory: Buffer, + regs: [u32; 8], + flags: u32, +} + +#[derive(Clone, Copy)] +enum ShiftKind { + Left, + Right, + ArithmeticRight, +} + +impl Vm { + fn with_allowance( + program: &impl ProgramCode, + invocation: Invocation<'_>, + allowance: &B, + ) -> Result { + if invocation.input.len() > GLOBAL_BASE { + return Err(Error::InvalidData("RARVM filter input is too large")); + } + + let mut memory = Buffer::filled(MEMORY_SIZE, 0, allowance)?; + memory[..invocation.input.len()].copy_from_slice(invocation.input); + let global_len = invocation.global_data.len().min(0x2000); + memory[GLOBAL_BASE..GLOBAL_BASE + global_len] + .copy_from_slice(&invocation.global_data[..global_len]); + let static_start = GLOBAL_BASE + global_len; + let static_len = program + .static_data() + .len() + .min(MEMORY_SIZE.saturating_sub(static_start)); + memory[static_start..static_start + static_len] + .copy_from_slice(&program.static_data()[..static_len]); + + write_u32( + &mut memory, + GLOBAL_BASE + 0x1c, + invocation.input.len() as u32, + ); + write_u32(&mut memory, GLOBAL_BASE + 0x20, 0); + write_u32( + &mut memory, + GLOBAL_BASE + 0x24, + invocation.file_offset as u32, + ); + write_u32( + &mut memory, + GLOBAL_BASE + 0x28, + (invocation.file_offset >> 32) as u32, + ); + write_u32(&mut memory, GLOBAL_BASE + 0x2c, invocation.exec_count); + + let mut regs = [0u32; 8]; + regs[..7].copy_from_slice(&invocation.regs); + regs[3] = GLOBAL_BASE as u32; + regs[4] = invocation.input.len() as u32; + regs[5] = invocation.exec_count; + regs[6] = invocation.file_offset as u32; + regs[7] = MEMORY_SIZE as u32; + + Ok(Self { + memory, + regs, + flags: 0, + }) + } + + fn run( + &mut self, + program: &impl ProgramCode, + control: &crate::read_control::ReadControl, + ) -> Result> { + self.run_with_limit(program, control, MAX_INSTRUCTIONS) + } + + fn run_with_limit( + &mut self, + program: &impl ProgramCode, + control: &crate::read_control::ReadControl, + instruction_limit: usize, + ) -> Result> { + let mut poller = control.poller(); + let mut ip = 0usize; + let mut terminated = false; + for _ in 0..instruction_limit { + poller.check_codec(0)?; + let Some(instruction) = program.instruction(ip) else { + terminated = true; + break; + }; + ip += 1; + if let Some(next_ip) = self.execute_instruction(&instruction, ip)? { + if next_ip >= program.instruction_count() { + terminated = true; + break; + } + ip = next_ip; + } + if instruction.opcode == Opcode::Ret && self.regs[7] >= MEMORY_SIZE as u32 { + terminated = true; + break; + } + } + if !terminated { + return Err(Error::InvalidData("RARVM instruction limit exceeded")); + } + + let mut output_pos = self.read_u32(GLOBAL_BASE + 0x20) as usize & MEMORY_MASK as usize; + let mut output_size = self.read_u32(GLOBAL_BASE + 0x1c) as usize & MEMORY_MASK as usize; + if output_pos + .checked_add(output_size) + .is_none_or(|end| end > MEMORY_SIZE) + { + output_pos = 0; + output_size = 0; + } + let output = Buffer::copied( + &self.memory[output_pos..output_pos + output_size], + &self.memory.allowance(), + )?; + + let user_global = (self.read_u32(GLOBAL_BASE + 0x30) as usize).min(MAX_USER_GLOBAL); + let globals = Buffer::copied( + &self.memory[GLOBAL_BASE..GLOBAL_BASE + SYSTEM_GLOBAL_SIZE + user_global], + &self.memory.allowance(), + )?; + Ok(OwnedExecutionResult { + output, + globals, + regs: self.regs, + }) + } + + fn execute_instruction( + &mut self, + instruction: &InstructionRef<'_>, + ip: usize, + ) -> Result> { + let byte_mode = instruction.byte_mode; + let op = |index| { + instruction + .operands + .get(index) + .ok_or(Error::InvalidData("RARVM instruction operand is missing")) + }; + match instruction.opcode { + Opcode::Mov => { + let value = self.read_operand(op(1)?, byte_mode); + self.write_operand(op(0)?, value, byte_mode)?; + } + Opcode::Cmp => { + let a = self.read_operand(op(0)?, byte_mode); + let b = self.read_operand(op(1)?, byte_mode); + self.set_sub_flags(a, b, 0, byte_mode); + } + Opcode::Add => { + let a = self.read_operand(op(0)?, byte_mode); + let b = self.read_operand(op(1)?, byte_mode); + let result = self.mask_width(a.wrapping_add(b), byte_mode); + self.write_operand(op(0)?, result, byte_mode)?; + self.set_add_flags(a, b, 0, result, byte_mode); + } + Opcode::Sub => { + let a = self.read_operand(op(0)?, byte_mode); + let b = self.read_operand(op(1)?, byte_mode); + let result = self.mask_width(a.wrapping_sub(b), byte_mode); + self.write_operand(op(0)?, result, byte_mode)?; + self.set_sub_flags(a, b, 0, byte_mode); + } + Opcode::Jz => return Ok(self.conditional_jump(op(0)?, self.flags & FLAG_Z != 0)), + Opcode::Jnz => return Ok(self.conditional_jump(op(0)?, self.flags & FLAG_Z == 0)), + Opcode::Inc => { + let value = self.read_operand(op(0)?, byte_mode).wrapping_add(1); + let result = self.mask_width(value, byte_mode); + self.write_operand(op(0)?, result, byte_mode)?; + self.set_zs(result, byte_mode); + } + Opcode::Dec => { + let value = self.read_operand(op(0)?, byte_mode).wrapping_sub(1); + let result = self.mask_width(value, byte_mode); + self.write_operand(op(0)?, result, byte_mode)?; + self.set_zs(result, byte_mode); + } + Opcode::Jmp => return Ok(Some(self.read_operand(op(0)?, false) as usize)), + Opcode::Xor | Opcode::And | Opcode::Or | Opcode::Test => { + let a = self.read_operand(op(0)?, byte_mode); + let b = self.read_operand(op(1)?, byte_mode); + let result = if instruction.opcode == Opcode::Xor { + a ^ b + } else if instruction.opcode == Opcode::Or { + a | b + } else { + a & b + }; + let result = self.mask_width(result, byte_mode); + if instruction.opcode != Opcode::Test { + self.write_operand(op(0)?, result, byte_mode)?; + } + self.set_zs(result, byte_mode); + } + Opcode::Js => return Ok(self.conditional_jump(op(0)?, self.flags & FLAG_S != 0)), + Opcode::Jns => return Ok(self.conditional_jump(op(0)?, self.flags & FLAG_S == 0)), + Opcode::Jb => return Ok(self.conditional_jump(op(0)?, self.flags & FLAG_C != 0)), + Opcode::Jbe => { + return Ok(self.conditional_jump(op(0)?, self.flags & (FLAG_C | FLAG_Z) != 0)); + } + Opcode::Ja => { + return Ok(self.conditional_jump(op(0)?, self.flags & (FLAG_C | FLAG_Z) == 0)); + } + Opcode::Jae => return Ok(self.conditional_jump(op(0)?, self.flags & FLAG_C == 0)), + Opcode::Push => self.push(self.read_operand(op(0)?, false)), + Opcode::Pop => { + let value = self.pop(); + self.write_operand(op(0)?, value, false)?; + } + Opcode::Call => { + self.push(ip as u32); + return Ok(Some(self.read_operand(op(0)?, false) as usize)); + } + Opcode::Ret => { + if self.regs[7] >= MEMORY_SIZE as u32 { + return Ok(Some(usize::MAX)); + } + return Ok(Some(self.pop() as usize)); + } + Opcode::Not => { + let result = self.mask_width(!self.read_operand(op(0)?, byte_mode), byte_mode); + self.write_operand(op(0)?, result, byte_mode)?; + } + Opcode::Shl => { + self.shift( + ShiftKind::Left, + op(0)?, + self.read_operand(op(1)?, byte_mode), + byte_mode, + )?; + } + Opcode::Shr => { + self.shift( + ShiftKind::Right, + op(0)?, + self.read_operand(op(1)?, byte_mode), + byte_mode, + )?; + } + Opcode::Sar => { + self.shift( + ShiftKind::ArithmeticRight, + op(0)?, + self.read_operand(op(1)?, byte_mode), + byte_mode, + )?; + } + Opcode::Neg => { + let value = self.read_operand(op(0)?, byte_mode); + let result = self.mask_width(0u32.wrapping_sub(value), byte_mode); + self.write_operand(op(0)?, result, byte_mode)?; + if result == 0 { + self.flags = FLAG_Z; + } else { + self.flags = FLAG_C | (result & self.sign_bit(byte_mode)); + } + } + Opcode::Pusha => { + let regs = self.regs; + for value in regs { + self.push(value); + } + } + Opcode::Popa => { + let mut stack = self.regs[7]; + for index in (0..8).rev() { + self.regs[index] = self.read_mem(stack, false); + stack = stack.wrapping_add(4); + } + } + Opcode::Pushf => self.push(self.flags), + Opcode::Popf => self.flags = self.pop(), + Opcode::Movzx => { + let value = self.read_operand(op(1)?, true) & 0xff; + self.write_operand(op(0)?, value, false)?; + } + Opcode::Movsx => { + let value = self.read_operand(op(1)?, true) as u8 as i8 as i32 as u32; + self.write_operand(op(0)?, value, false)?; + } + Opcode::Xchg => { + let a = self.read_operand(op(0)?, byte_mode); + let b = self.read_operand(op(1)?, byte_mode); + self.write_operand(op(0)?, b, byte_mode)?; + self.write_operand(op(1)?, a, byte_mode)?; + } + Opcode::Mul => { + let result = self + .read_operand(op(0)?, byte_mode) + .wrapping_mul(self.read_operand(op(1)?, byte_mode)); + self.write_operand(op(0)?, self.mask_width(result, byte_mode), byte_mode)?; + } + Opcode::Div => { + let divisor = self.read_operand(op(1)?, byte_mode); + if let Some(result) = self.read_operand(op(0)?, byte_mode).checked_div(divisor) { + self.write_operand(op(0)?, result, byte_mode)?; + } + } + Opcode::Adc | Opcode::Sbb => { + let a = self.read_operand(op(0)?, byte_mode); + let b = self.read_operand(op(1)?, byte_mode); + let carry = u32::from(self.flags & FLAG_C != 0); + let result = if instruction.opcode == Opcode::Adc { + self.mask_width(a.wrapping_add(b).wrapping_add(carry), byte_mode) + } else { + self.mask_width(a.wrapping_sub(b).wrapping_sub(carry), byte_mode) + }; + self.write_operand(op(0)?, result, byte_mode)?; + if instruction.opcode == Opcode::Adc { + self.set_add_flags(a, b, carry, result, byte_mode); + } else { + self.set_sub_flags(a, b, carry, byte_mode); + } + } + Opcode::Print => {} + } + Ok(None) + } + + fn conditional_jump(&self, operand: &Operand, condition: bool) -> Option { + condition.then_some(self.read_operand(operand, false) as usize) + } + + fn read_operand(&self, operand: &Operand, byte_mode: bool) -> u32 { + match *operand { + Operand::Register(index) => { + let value = self.regs[index as usize]; + if byte_mode { + value & 0xff + } else { + value + } + } + Operand::Immediate(value) => self.mask_width(value, byte_mode), + Operand::RegisterIndirect(index) => self.read_mem(self.regs[index as usize], byte_mode), + Operand::Indexed { register, base } => { + self.read_mem(base.wrapping_add(self.regs[register as usize]), byte_mode) + } + Operand::Absolute(address) => self.read_mem(address, byte_mode), + } + } + + fn write_operand(&mut self, operand: &Operand, value: u32, byte_mode: bool) -> Result<()> { + match *operand { + Operand::Register(index) => { + let slot = &mut self.regs[index as usize]; + if byte_mode { + *slot = (*slot & 0xffff_ff00) | (value & 0xff); + } else { + *slot = value; + } + } + Operand::RegisterIndirect(index) => { + self.write_mem(self.regs[index as usize], value, byte_mode) + } + Operand::Indexed { register, base } => { + self.write_mem( + base.wrapping_add(self.regs[register as usize]), + value, + byte_mode, + ); + } + Operand::Absolute(address) => self.write_mem(address, value, byte_mode), + Operand::Immediate(_) => { + return Err(Error::InvalidData("RARVM write to immediate operand")) + } + } + Ok(()) + } + + fn read_mem(&self, address: u32, byte_mode: bool) -> u32 { + let address = address & MEMORY_MASK; + if byte_mode { + u32::from(self.memory[address as usize]) + } else { + self.read_u32(address as usize) + } + } + + fn write_mem(&mut self, address: u32, value: u32, byte_mode: bool) { + let address = address & MEMORY_MASK; + if byte_mode { + self.memory[address as usize] = value as u8; + } else { + write_u32(&mut self.memory, address as usize, value); + } + } + + fn read_u32(&self, address: usize) -> u32 { + let address = address as u32; + u32::from_le_bytes([ + self.memory[(address & MEMORY_MASK) as usize], + self.memory[(address.wrapping_add(1) & MEMORY_MASK) as usize], + self.memory[(address.wrapping_add(2) & MEMORY_MASK) as usize], + self.memory[(address.wrapping_add(3) & MEMORY_MASK) as usize], + ]) + } + + fn push(&mut self, value: u32) { + self.regs[7] = self.regs[7].wrapping_sub(4); + self.write_mem(self.regs[7], value, false); + } + + fn pop(&mut self) -> u32 { + let value = self.read_mem(self.regs[7], false); + self.regs[7] = self.regs[7].wrapping_add(4); + value + } + + fn shift(&mut self, kind: ShiftKind, dst: &Operand, count: u32, byte_mode: bool) -> Result<()> { + if count == 0 { + return Ok(()); + } + let width = if byte_mode { 8 } else { 32 }; + let count = count.min(width); + let value = self.read_operand(dst, byte_mode); + let result = match kind { + ShiftKind::Left => { + if count == width { + 0 + } else { + value.wrapping_shl(count) + } + } + ShiftKind::Right => { + if count == width { + 0 + } else { + value.wrapping_shr(count) + } + } + ShiftKind::ArithmeticRight => { + if byte_mode { + if count >= 8 { + if value & 0x80 != 0 { + 0xff + } else { + 0 + } + } else { + ((value as u8 as i8) >> count) as u8 as u32 + } + } else if count >= 32 { + if value & 0x8000_0000 != 0 { + u32::MAX + } else { + 0 + } + } else { + ((value as i32) >> count) as u32 + } + } + }; + let carry = match kind { + ShiftKind::Left => value & (1 << (width - count)) != 0, + ShiftKind::Right | ShiftKind::ArithmeticRight => value & (1 << (count - 1)) != 0, + }; + let result = self.mask_width(result, byte_mode); + self.write_operand(dst, result, byte_mode)?; + self.set_zsc(result, carry, byte_mode); + Ok(()) + } + + fn set_add_flags(&mut self, a: u32, b: u32, carry: u32, result: u32, byte_mode: bool) { + let mask = self.value_mask(byte_mode) as u64; + let sum = (a as u64 & mask) + (b as u64 & mask) + u64::from(carry); + self.set_zsc(result, sum > mask, byte_mode); + } + + fn set_sub_flags(&mut self, a: u32, b: u32, borrow: u32, byte_mode: bool) { + let mask = self.value_mask(byte_mode) as u64; + let a = a as u64 & mask; + let subtrahend = (b as u64 & mask) + u64::from(borrow); + let result = self.mask_width((a as u32).wrapping_sub(subtrahend as u32), byte_mode); + self.set_zsc(result, a < subtrahend, byte_mode); + } + + fn set_zs(&mut self, result: u32, byte_mode: bool) { + self.flags = if result == 0 { + FLAG_Z + } else { + result & self.sign_bit(byte_mode) + }; + } + + fn set_zsc(&mut self, result: u32, carry: bool, byte_mode: bool) { + self.set_zs(result, byte_mode); + if carry { + self.flags |= FLAG_C; + } + } + + fn mask_width(&self, value: u32, byte_mode: bool) -> u32 { + value & self.value_mask(byte_mode) + } + + fn value_mask(&self, byte_mode: bool) -> u32 { + if byte_mode { + 0xff + } else { + u32::MAX + } + } + + fn sign_bit(&self, byte_mode: bool) -> u32 { + if byte_mode { + 0x80 + } else { + FLAG_S + } + } +} + +fn write_u32(memory: &mut [u8], address: usize, value: u32) { + let address = address as u32; + for (offset, byte) in value.to_le_bytes().into_iter().enumerate() { + memory[(address.wrapping_add(offset as u32) & MEMORY_MASK) as usize] = byte; + } +} + +#[cfg(test)] +impl Vm { + fn new(program: &impl ProgramCode, invocation: Invocation<'_>) -> Result { + Self::with_allowance(program, invocation, &Allowance::default()) + } +} + +#[derive(Debug, Clone)] +struct BitReader<'a> { + input: &'a [u8], + bit_pos: usize, +} + +impl<'a> BitReader<'a> { + fn new(input: &'a [u8]) -> Self { + Self { input, bit_pos: 0 } + } + + fn remaining_bits(&self) -> usize { + self.input.len() * 8 - self.bit_pos + } + + fn read_bit(&mut self) -> Result { + self.read_bits(1) + } + + fn read_bits(&mut self, count: usize) -> Result { + if self.remaining_bits() < count { + return Err(Error::NeedMoreInput); + } + let mut value = 0; + for _ in 0..count { + let byte = self.input[self.bit_pos / 8]; + let bit = (byte >> (7 - (self.bit_pos % 8))) & 1; + value = (value << 1) | u32::from(bit); + self.bit_pos += 1; + } + Ok(value) + } + + fn read_vm_number(&mut self) -> Result { + match self.read_bits(2)? { + 0 => self.read_bits(4), + 1 => { + let high = self.read_bits(8)?; + if high >= 16 { + Ok(high) + } else { + Ok(0xffff_ff00 | (high << 4) | self.read_bits(4)?) + } + } + 2 => self.read_bits(16), + _ => self.read_bits(32), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/arca-rar/vendor/rars/src/codec/rarvm/tests.rs b/arca-rar/vendor/rars/src/codec/rarvm/tests.rs new file mode 100644 index 00000000..db824f72 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/rarvm/tests.rs @@ -0,0 +1,1273 @@ +fn reader_vm_blob() -> Vec { + let mut bits = BitWriter::new(); + bits.write_bits(1, 1); + write_vm_number(&mut bits, 2); + for byte in [0xaa, 0xbb, 0xcc] { + bits.write_bits(byte, 8); + } + for (address, value) in [(0, 0x41), (GLOBAL_BASE as u32 + 0x30, 8)] { + write_opcode(&mut bits, Opcode::Mov); + bits.write_bits(0, 1); + write_absolute(&mut bits, address); + write_number_immediate(&mut bits, value); + } + write_opcode(&mut bits, Opcode::Ret); + with_xor(bits.finish()) +} +fn reader_vm_invocation() -> Invocation<'static> { + Invocation { + input: &[1, 2, 3, 4], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + } +} + +#[test] +fn reader_vm_workspace_refusals_release_program_operands_memory_and_results() { + use crate::codec::workspace::RefusingBudget; + let blob = reader_vm_blob(); + let run = |budget: &RefusingBudget| -> Result<()> { + let program = OwnedProgram::parse(&blob, budget)?; + let mut vm = Vm::with_allowance(&program, reader_vm_invocation(), budget)?; + let result = vm.run(&program, &crate::read_control::ReadControl::default())?; + assert_eq!(&*result.output, b"A\0\0\0"); + assert_eq!(result.globals.len(), SYSTEM_GLOBAL_SIZE + 8); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + let attempts = baseline.attempts(); + assert!(attempts >= 7); + assert_eq!(baseline.used(), 0); + for index in 0..attempts { + let budget = RefusingBudget::new(index); + assert!( + matches!(run(&budget), Err(Error::Cancelled)), + "allocation {index}" + ); + assert_eq!(budget.used(), 0); + } +} + +#[test] +fn reader_vm_workspace_results_remain_charged_after_worker_retirement() { + use crate::codec::workspace::RESERVATION_BYTES; + let ledger = Allowance::limited(1024 * 1024 + RESERVATION_BYTES); + let mut reservation = ledger.reserve(1024 * 1024).unwrap(); + let budget = reservation.allowance(); + reservation.start(); + let program = OwnedProgram::parse(&reader_vm_blob(), &budget).unwrap(); + let mut vm = Vm::with_allowance(&program, reader_vm_invocation(), &budget).unwrap(); + let result = vm + .run(&program, &crate::read_control::ReadControl::default()) + .unwrap(); + reservation.retire(); + assert!(ledger.used() >= MEMORY_SIZE as u64 + RESERVATION_BYTES); + drop(vm); + drop(program); + assert_eq!( + ledger.used(), + (result.output.capacity() + result.globals.capacity()) as u64 + RESERVATION_BYTES + ); + drop(result); + assert_eq!(ledger.used(), RESERVATION_BYTES); + drop(budget); + assert_eq!(ledger.used(), 0); +} + +#[test] +fn cancellation_interrupts_vm_work_without_output() { + let program = Program { + static_data: vec![], + instructions: vec![instr(Opcode::Jmp, false, vec![Operand::Immediate(0)])], + }; + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(2); + let err = program + .execute_with_control( + Invocation { + input: &[], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }, + &control, + ) + .unwrap_err(); + assert_eq!(err, Error::Cancelled); +} + +#[test] +fn rejects_a_program_that_exhausts_its_instruction_allowance() { + let program = Program { + static_data: vec![], + instructions: vec![instr(Opcode::Jmp, false, vec![Operand::Immediate(0)])], + }; + let invocation = Invocation { + input: &[], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }; + let mut vm = Vm::new(&program, invocation).unwrap(); + + assert!(matches!( + vm.run_with_limit(&program, &crate::read_control::ReadControl::default(), 3), + Err(Error::InvalidData("RARVM instruction limit exceeded")) + )); +} +use super::*; + +#[test] +fn rejects_bad_xor_checksum() { + assert_eq!( + Program::parse(&[0x12, 0x34]), + Err(Error::InvalidData("RARVM program checksum mismatch")) + ); +} + +#[test] +fn rejects_an_empty_program_blob() { + assert_eq!( + Program::parse(&[]), + Err(Error::InvalidData("RARVM program blob is empty")) + ); +} + +#[test] +fn an_explicitly_empty_program_terminates_without_changing_its_input() { + let result = Program { + static_data: Vec::new(), + instructions: Vec::new(), + } + .execute(Invocation { + input: b"unchanged", + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, b"unchanged"); +} + +#[test] +fn parses_static_data_and_appends_implicit_ret() { + let mut bits = BitWriter::new(); + bits.write_bits(1, 1); + write_vm_number(&mut bits, 2); + bits.write_bits(0xaa, 8); + bits.write_bits(0xbb, 8); + bits.write_bits(0xcc, 8); + let program = Program::parse(&with_xor(bits.finish())).unwrap(); + + assert_eq!(program.static_data, [0xaa, 0xbb, 0xcc]); + assert_eq!( + program.instructions, + [Instruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Vec::new(), + }] + ); +} + +#[test] +fn parses_register_immediate_and_memory_operands() { + let mut bits = BitWriter::new(); + bits.write_bits(0, 1); + write_opcode(&mut bits, Opcode::Mov); + bits.write_bits(0, 1); + write_reg(&mut bits, 2); + write_number_immediate(&mut bits, 0x1234); + write_opcode(&mut bits, Opcode::Add); + bits.write_bits(1, 1); + write_reg_indirect(&mut bits, 3); + write_byte_immediate(&mut bits, 0x7f); + write_opcode(&mut bits, Opcode::Sub); + bits.write_bits(0, 1); + write_indexed(&mut bits, 1, 0x44); + write_absolute(&mut bits, 0x3c000); + write_opcode(&mut bits, Opcode::Ret); + + let program = Program::parse(&with_xor(bits.finish())).unwrap(); + assert!(program.static_data.is_empty()); + assert_eq!(program.instructions.len(), 4); + assert_eq!(program.instructions[0].opcode, Opcode::Mov); + assert!(!program.instructions[0].byte_mode); + assert_eq!( + program.instructions[0].operands, + [Operand::Register(2), Operand::Immediate(0x1234)] + ); + assert_eq!(program.instructions[1].opcode, Opcode::Add); + assert!(program.instructions[1].byte_mode); + assert_eq!( + program.instructions[1].operands, + [Operand::RegisterIndirect(3), Operand::Immediate(0x7f)] + ); + assert_eq!( + program.instructions[2].operands, + [ + Operand::Indexed { + register: 1, + base: 0x44, + }, + Operand::Absolute(0x3c000), + ] + ); + assert_eq!(program.instructions[3].opcode, Opcode::Ret); +} + +#[test] +fn parses_every_long_opcode_used_by_generic_programs() { + for opcode in [ + Opcode::Xor, + Opcode::Sar, + Opcode::Pushf, + Opcode::Popf, + Opcode::Div, + Opcode::Adc, + Opcode::Sbb, + ] { + let mut bits = BitWriter::new(); + bits.write_bits(0, 1); // no static data + write_opcode(&mut bits, opcode); + if opcode.supports_byte_mode() { + bits.write_bits(0, 1); + } + for index in 0..opcode.operand_count() { + write_reg(&mut bits, index as u8); + } + write_opcode(&mut bits, Opcode::Ret); + + let program = Program::parse(&with_xor(bits.finish())).unwrap(); + assert_eq!(program.instructions[0].opcode, opcode); + } +} + +#[test] +fn parses_runtime_and_absolute_jump_targets() { + let mut runtime = BitWriter::new(); + runtime.write_bits(0, 1); // no static data + write_opcode(&mut runtime, Opcode::Jmp); + write_reg(&mut runtime, 3); + let program = Program::parse(&with_xor(runtime.finish())).unwrap(); + assert_eq!(program.instructions[0].operands, [Operand::Register(3)]); + + let mut absolute = BitWriter::new(); + absolute.write_bits(0, 1); // no static data + write_opcode(&mut absolute, Opcode::Jmp); + write_number_immediate(&mut absolute, 300); + let program = Program::parse(&with_xor(absolute.finish())).unwrap(); + assert_eq!(program.instructions[0].operands, [Operand::Immediate(44)]); +} + +#[test] +fn ignores_a_trailing_partial_instruction() { + let mut bits = BitWriter::new(); + bits.write_bits(0, 1); // no static data + write_opcode(&mut bits, Opcode::Mov); + bits.write_bits(0, 1); // word mode + write_reg(&mut bits, 0); // second operand is absent + + let program = Program::parse(&with_xor(bits.finish())).unwrap(); + assert_eq!( + program.instructions, + [instr(Opcode::Ret, false, Vec::new())] + ); +} + +#[test] +fn remaps_jump_immediates_to_instruction_indices() { + let mut bits = BitWriter::new(); + bits.write_bits(0, 1); + write_opcode(&mut bits, Opcode::Print); + write_opcode(&mut bits, Opcode::Jmp); + write_number_immediate(&mut bits, 15); + + let program = Program::parse(&with_xor(bits.finish())).unwrap(); + assert_eq!(program.instructions.len(), 2); + assert_eq!( + program.instructions[1], + Instruction { + opcode: Opcode::Jmp, + byte_mode: false, + operands: vec![Operand::Immediate(0)], + } + ); +} + +#[test] +fn executes_arithmetic_and_memory_writes() { + let program = Program { + static_data: Vec::new(), + instructions: vec![ + Instruction { + opcode: Opcode::Mov, + byte_mode: false, + operands: vec![Operand::Register(0), Operand::Immediate(7)], + }, + Instruction { + opcode: Opcode::Add, + byte_mode: false, + operands: vec![Operand::Register(0), Operand::Immediate(5)], + }, + Instruction { + opcode: Opcode::Mov, + byte_mode: true, + operands: vec![Operand::Absolute(0), Operand::Register(0)], + }, + Instruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Vec::new(), + }, + ], + }; + + let result = program + .execute(Invocation { + input: &[0], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, [12]); + assert_eq!(result.regs[0], 12); +} + +#[test] +fn executes_conditional_jump_and_stack_call() { + let program = Program { + static_data: Vec::new(), + instructions: vec![ + Instruction { + opcode: Opcode::Mov, + byte_mode: false, + operands: vec![Operand::Register(0), Operand::Immediate(1)], + }, + Instruction { + opcode: Opcode::Cmp, + byte_mode: false, + operands: vec![Operand::Register(0), Operand::Immediate(1)], + }, + Instruction { + opcode: Opcode::Jz, + byte_mode: false, + operands: vec![Operand::Immediate(4)], + }, + Instruction { + opcode: Opcode::Mov, + byte_mode: false, + operands: vec![Operand::Register(0), Operand::Immediate(99)], + }, + Instruction { + opcode: Opcode::Call, + byte_mode: false, + operands: vec![Operand::Immediate(6)], + }, + Instruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Vec::new(), + }, + Instruction { + opcode: Opcode::Add, + byte_mode: false, + operands: vec![Operand::Register(0), Operand::Immediate(41)], + }, + Instruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Vec::new(), + }, + ], + }; + + let result = program + .execute(Invocation { + input: &[0], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.regs[0], 42); +} + +#[test] +fn nested_calls_return_through_each_stack_frame() { + let result = execute_instructions(vec![ + instr(Opcode::Call, false, vec![Operand::Immediate(3)]), + instr(Opcode::Ret, false, Vec::new()), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(99)], + ), + instr(Opcode::Call, false, vec![Operand::Immediate(6)]), + instr( + Opcode::Add, + false, + vec![Operand::Register(0), Operand::Immediate(2)], + ), + instr(Opcode::Ret, false, Vec::new()), + instr( + Opcode::Add, + false, + vec![Operand::Register(0), Operand::Immediate(40)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 42); +} + +#[test] +fn executes_unconditional_jumps_and_mutating_unary_ops() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(1)], + ), + instr(Opcode::Inc, false, vec![Operand::Register(0)]), + instr(Opcode::Dec, false, vec![Operand::Register(0)]), + instr(Opcode::Not, false, vec![Operand::Register(0)]), + instr(Opcode::Neg, false, vec![Operand::Register(0)]), + instr(Opcode::Jmp, false, vec![Operand::Immediate(7)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(99)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 2); +} + +#[test] +fn executes_logic_ops_and_test_without_writing_destination() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0b1010)], + ), + instr( + Opcode::Xor, + false, + vec![Operand::Register(0), Operand::Immediate(0b1100)], + ), + instr( + Opcode::And, + false, + vec![Operand::Register(0), Operand::Immediate(0b0110)], + ), + instr( + Opcode::Or, + false, + vec![Operand::Register(0), Operand::Immediate(0b0001)], + ), + instr( + Opcode::Test, + false, + vec![Operand::Register(0), Operand::Immediate(0b0100)], + ), + instr(Opcode::Jnz, false, vec![Operand::Immediate(7)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(99)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0b0111); +} + +#[test] +fn executes_unsigned_conditional_jumps() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(1), Operand::Immediate(2)], + ), + instr(Opcode::Jb, false, vec![Operand::Immediate(5)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(99)], + ), + instr(Opcode::Ret, false, Vec::new()), + instr(Opcode::Jbe, false, vec![Operand::Immediate(7)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(98)], + ), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(3), Operand::Immediate(2)], + ), + instr(Opcode::Ja, false, vec![Operand::Immediate(10)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(97)], + ), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(3), Operand::Immediate(2)], + ), + instr(Opcode::Jae, false, vec![Operand::Immediate(13)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(96)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(42)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 42); +} + +#[test] +fn executes_signed_conditional_jumps() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr( + Opcode::Sub, + false, + vec![Operand::Register(0), Operand::Immediate(1)], + ), + instr(Opcode::Js, false, vec![Operand::Immediate(5)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(99)], + ), + instr(Opcode::Ret, false, Vec::new()), + instr( + Opcode::Add, + false, + vec![Operand::Register(0), Operand::Immediate(1)], + ), + instr(Opcode::Jns, false, vec![Operand::Immediate(8)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(98)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(42)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0); + assert_eq!(result.regs[1], 42); +} + +#[test] +fn executes_stack_register_and_flag_round_trips() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(10)], + ), + instr(Opcode::Push, false, vec![Operand::Register(0)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr(Opcode::Pop, false, vec![Operand::Register(1)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(10)], + ), + instr(Opcode::Pusha, false, Vec::new()), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(99)], + ), + instr(Opcode::Popa, false, Vec::new()), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(1), Operand::Immediate(2)], + ), + instr(Opcode::Pushf, false, Vec::new()), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(2), Operand::Immediate(2)], + ), + instr(Opcode::Popf, false, Vec::new()), + instr(Opcode::Jb, false, vec![Operand::Immediate(14)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(99)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 10); + assert_eq!(result.regs[1], 10); +} + +#[test] +fn executes_shifts_with_byte_and_word_modes() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0x81)], + ), + instr( + Opcode::Shl, + false, + vec![Operand::Register(0), Operand::Immediate(1)], + ), + instr( + Opcode::Shr, + false, + vec![Operand::Register(0), Operand::Immediate(2)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(0x80)], + ), + instr( + Opcode::Sar, + true, + vec![Operand::Register(1), Operand::Immediate(1)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0x40); + assert_eq!(result.regs[1], 0xc0); +} + +#[test] +fn byte_mode_sar_accepts_shift_count_equal_to_width() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0x80)], + ), + instr( + Opcode::Sar, + true, + vec![Operand::Register(0), Operand::Immediate(8)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(0x7f)], + ), + instr( + Opcode::Sar, + true, + vec![Operand::Register(1), Operand::Immediate(8)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0xff); + assert_eq!(result.regs[1], 0); +} + +#[test] +fn word_mode_sar_sign_extends_at_ordinary_and_full_width_counts() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0x8000_0000)], + ), + instr( + Opcode::Sar, + false, + vec![Operand::Register(0), Operand::Immediate(1)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(0x8000_0000)], + ), + instr( + Opcode::Sar, + false, + vec![Operand::Register(1), Operand::Immediate(32)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(2), Operand::Immediate(1)], + ), + instr( + Opcode::Sar, + false, + vec![Operand::Register(2), Operand::Immediate(32)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0xc000_0000); + assert_eq!(result.regs[1], u32::MAX); + assert_eq!(result.regs[2], 0); +} + +#[test] +fn neg_zero_sets_zero_and_divide_by_zero_keeps_the_destination() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr(Opcode::Neg, false, vec![Operand::Register(0)]), + instr(Opcode::Jz, false, vec![Operand::Immediate(4)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(99)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(42)], + ), + instr( + Opcode::Div, + false, + vec![Operand::Register(1), Operand::Immediate(0)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0); + assert_eq!(result.regs[1], 42); +} + +#[test] +fn full_width_shl_and_shr_clear_destination() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0x1234_5678)], + ), + instr( + Opcode::Shl, + false, + vec![Operand::Register(0), Operand::Immediate(32)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(0x8765_4321)], + ), + instr( + Opcode::Shr, + false, + vec![Operand::Register(1), Operand::Immediate(32)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(2), Operand::Immediate(0xff)], + ), + instr( + Opcode::Shl, + true, + vec![Operand::Register(2), Operand::Immediate(8)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(3), Operand::Immediate(0xff)], + ), + instr( + Opcode::Shr, + true, + vec![Operand::Register(3), Operand::Immediate(8)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0); + assert_eq!(result.regs[1], 0); + assert_eq!(result.regs[2] & 0xff, 0); + assert_eq!(result.regs[3] & 0xff, 0); +} + +#[test] +fn sbb_sets_borrow_flag_when_subtrahend_plus_carry_wraps_byte_width() { + let result = execute_instructions(vec![ + instr( + Opcode::Cmp, + true, + vec![Operand::Immediate(0), Operand::Immediate(1)], + ), + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr( + Opcode::Sbb, + true, + vec![Operand::Register(0), Operand::Immediate(0xff)], + ), + instr(Opcode::Jb, false, vec![Operand::Immediate(6)]), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(0xdead)], + ), + instr(Opcode::Ret, false, Vec::new()), + instr( + Opcode::Mov, + false, + vec![Operand::Register(1), Operand::Immediate(0xbeef)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0] & 0xff, 0); + assert_eq!(result.regs[1], 0xbeef); +} + +#[test] +fn zero_count_shifts_are_noops() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Register(0), Operand::Immediate(0x1234_5678)], + ), + instr( + Opcode::Shl, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr( + Opcode::Shr, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr( + Opcode::Sar, + false, + vec![Operand::Register(0), Operand::Immediate(0)], + ), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0x1234_5678); +} + +#[test] +fn output_range_accepts_exclusive_memory_end() { + let program = Program { + static_data: Vec::new(), + instructions: vec![ + instr( + Opcode::Mov, + false, + vec![ + Operand::Absolute((GLOBAL_BASE + 0x20) as u32), + Operand::Immediate((MEMORY_SIZE - 1) as u32), + ], + ), + instr( + Opcode::Mov, + false, + vec![ + Operand::Absolute((GLOBAL_BASE + 0x1c) as u32), + Operand::Immediate(1), + ], + ), + instr( + Opcode::Mov, + true, + vec![ + Operand::Absolute((MEMORY_SIZE - 1) as u32), + Operand::Immediate(0x5a), + ], + ), + instr(Opcode::Ret, false, Vec::new()), + ], + }; + + let result = program + .execute(Invocation { + input: &[0], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + + assert_eq!(result.output, [0x5a]); +} + +#[test] +fn output_range_past_memory_is_discarded() { + let program = Program { + static_data: Vec::new(), + instructions: vec![ + instr( + Opcode::Mov, + false, + vec![ + Operand::Absolute((GLOBAL_BASE + 0x20) as u32), + Operand::Immediate((MEMORY_SIZE - 1) as u32), + ], + ), + instr( + Opcode::Mov, + false, + vec![ + Operand::Absolute((GLOBAL_BASE + 0x1c) as u32), + Operand::Immediate(2), + ], + ), + instr(Opcode::Ret, false, Vec::new()), + ], + }; + + let result = program + .execute(Invocation { + input: &[0x5a], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap(); + assert!(result.output.is_empty()); +} + +#[test] +fn rejects_a_program_that_writes_to_an_immediate() { + let program = Program { + static_data: Vec::new(), + instructions: vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Immediate(0), Operand::Immediate(1)], + ), + instr(Opcode::Ret, false, Vec::new()), + ], + }; + + assert_eq!( + program.execute(Invocation { + input: &[], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }), + Err(Error::InvalidData("RARVM write to immediate operand")) + ); +} + +#[test] +fn parsed_shifts_reject_an_immediate_destination() { + for opcode in [Opcode::Shl, Opcode::Shr, Opcode::Sar] { + let mut bits = BitWriter::new(); + bits.write_bits(0, 1); // no static data + write_opcode(&mut bits, opcode); + bits.write_bits(0, 1); // word mode + write_number_immediate(&mut bits, 1); // destination + write_number_immediate(&mut bits, 1); // nonzero shift count + write_opcode(&mut bits, Opcode::Ret); + + let program = Program::parse(&with_xor(bits.finish())).unwrap(); + assert_eq!(program.instructions[0].opcode, opcode); + assert_eq!( + program.execute(Invocation { + input: &[], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }), + Err(Error::InvalidData("RARVM write to immediate operand")), + "{opcode:?} should reject an immediate destination", + ); + } +} + +#[test] +fn executes_extension_exchange_multiply_divide_and_carry_arithmetic() { + let result = execute_instructions(vec![ + instr( + Opcode::Mov, + false, + vec![Operand::Absolute(0), Operand::Immediate(0x80)], + ), + instr( + Opcode::Movzx, + false, + vec![Operand::Register(0), Operand::Absolute(0)], + ), + instr( + Opcode::Movsx, + false, + vec![Operand::Register(1), Operand::Absolute(0)], + ), + instr( + Opcode::Xchg, + false, + vec![Operand::Register(0), Operand::Register(1)], + ), + instr( + Opcode::Mul, + false, + vec![Operand::Register(1), Operand::Immediate(3)], + ), + instr( + Opcode::Div, + false, + vec![Operand::Register(1), Operand::Immediate(2)], + ), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(1), Operand::Immediate(2)], + ), + instr( + Opcode::Adc, + false, + vec![Operand::Register(1), Operand::Immediate(1)], + ), + instr( + Opcode::Cmp, + false, + vec![Operand::Immediate(1), Operand::Immediate(2)], + ), + instr( + Opcode::Sbb, + false, + vec![Operand::Register(1), Operand::Immediate(2)], + ), + instr(Opcode::Print, false, Vec::new()), + instr(Opcode::Ret, false, Vec::new()), + ]); + + assert_eq!(result.regs[0], 0xffff_ff80); + assert_eq!(result.regs[1], 0xbf); +} + +#[test] +fn preserves_requested_user_globals() { + let program = Program { + static_data: b"static".to_vec(), + instructions: vec![ + Instruction { + opcode: Opcode::Mov, + byte_mode: false, + operands: vec![Operand::Absolute(0x3c030), Operand::Immediate(4)], + }, + Instruction { + opcode: Opcode::Ret, + byte_mode: false, + operands: Vec::new(), + }, + ], + }; + + let result = program + .execute(Invocation { + input: &[1, 2, 3], + regs: [0; 7], + global_data: &[0; 64], + file_offset: 0x1_0000_0002, + exec_count: 9, + }) + .unwrap(); + + assert_eq!(result.output, [1, 2, 3]); + assert_eq!(result.globals.len(), 68); + assert_eq!(&result.globals[64..], b"stat"); +} + +#[test] +fn parse_rejects_huge_static_data_size_without_preallocating() { + let err = Program::parse(&[0xff, 0xff, 0xff, 0xff, 0, 0]).unwrap_err(); + assert_eq!(err, Error::InvalidData("RARVM static data is too large")); +} + +#[test] +fn parse_rejects_static_data_larger_than_vm_memory() { + let mut bits = BitWriter::new(); + bits.write_bits(1, 1); + write_vm_number(&mut bits, MAX_STATIC_DATA as u32); + + let err = Program::parse(&with_xor(bits.finish())).unwrap_err(); + assert_eq!(err, Error::InvalidData("RARVM static data is too large")); +} + +fn instr(opcode: Opcode, byte_mode: bool, operands: Vec) -> Instruction { + Instruction { + opcode, + byte_mode, + operands, + } +} + +fn execute_instructions(instructions: Vec) -> ExecutionResult { + Program { + static_data: Vec::new(), + instructions, + } + .execute(Invocation { + input: &[0], + regs: [0; 7], + global_data: &[], + file_offset: 0, + exec_count: 0, + }) + .unwrap() +} + +struct BitWriter { + output: Vec, + bit_pos: usize, +} + +impl BitWriter { + fn new() -> Self { + Self { + output: Vec::new(), + bit_pos: 0, + } + } + + fn write_bits(&mut self, value: u32, count: usize) { + for i in (0..count).rev() { + if self.bit_pos.is_multiple_of(8) { + self.output.push(0); + } + if (value >> i) & 1 != 0 { + let idx = self.output.len() - 1; + self.output[idx] |= 1 << (7 - (self.bit_pos % 8)); + } + self.bit_pos += 1; + } + } + + fn finish(self) -> Vec { + self.output + } +} + +fn with_xor(mut payload: Vec) -> Vec { + let checksum = payload.iter().fold(0u8, |acc, &byte| acc ^ byte); + payload.insert(0, checksum); + payload +} + +fn write_opcode(bits: &mut BitWriter, opcode: Opcode) { + let value = opcode as u8; + if value <= 7 { + bits.write_bits(0, 1); + bits.write_bits(u32::from(value), 3); + } else { + bits.write_bits(1, 1); + bits.write_bits(u32::from(value - 8), 5); + } +} + +fn write_reg(bits: &mut BitWriter, reg: u8) { + bits.write_bits(1, 1); + bits.write_bits(u32::from(reg), 3); +} + +fn write_number_immediate(bits: &mut BitWriter, value: u32) { + bits.write_bits(0, 2); + write_vm_number(bits, value); +} + +fn write_byte_immediate(bits: &mut BitWriter, value: u8) { + bits.write_bits(0, 2); + bits.write_bits(u32::from(value), 8); +} + +fn write_reg_indirect(bits: &mut BitWriter, reg: u8) { + bits.write_bits(0b010, 3); + bits.write_bits(u32::from(reg), 3); +} + +fn write_indexed(bits: &mut BitWriter, reg: u8, base: u32) { + bits.write_bits(0b0110, 4); + bits.write_bits(u32::from(reg), 3); + write_vm_number(bits, base); +} + +fn write_absolute(bits: &mut BitWriter, address: u32) { + bits.write_bits(0b0111, 4); + write_vm_number(bits, address); +} + +fn write_vm_number(bits: &mut BitWriter, value: u32) { + if value <= 15 { + bits.write_bits(0, 2); + bits.write_bits(value, 4); + } else if value <= 255 { + bits.write_bits(1, 2); + bits.write_bits(value, 8); + } else if value <= 0xffff { + bits.write_bits(2, 2); + bits.write_bits(value, 16); + } else { + bits.write_bits(3, 2); + bits.write_bits(value, 32); + } +} diff --git a/arca-rar/vendor/rars/src/codec/workspace.rs b/arca-rar/vendor/rars/src/codec/workspace.rs new file mode 100644 index 00000000..be297ecc --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/workspace.rs @@ -0,0 +1,869 @@ +//! Fallible ownership of codec allocation capacity. A local allowance never +//! reaches into a sibling worker's spare bytes. Internal admission tests connect +//! these owners to coordinator reservations; unlimited handles preserve behaviour. +use super::{Error, Result}; +pub(crate) mod coordinator; +mod ledger; +pub(crate) use ledger::{Charge, Limited, Reservation, RESERVATION_BYTES}; + +/// A zero-sized policy: its buffers have exactly Vec's layout and no ledger +/// branch in their hot operations. Limited buffers are a separate instantiation. +#[derive(Clone, Debug, Default)] +pub(crate) struct Allowance { + _unlimited: (), +} +impl Allowance { + pub(crate) fn limited(limit: u64) -> Limited { + Limited::new(limit) + } +} + +pub(crate) trait Budget: Clone + std::fmt::Debug { + type Charge: std::fmt::Debug; + type Failure: Into; + fn grow( + values: &mut Vec, + charge: &mut Self::Charge, + additional: usize, + ) -> std::result::Result<(), Self::Failure>; + const LIMITED: bool; + fn charge(&self) -> Self::Charge; + fn allowance(charge: &Self::Charge) -> Self; + fn resize(charge: &mut Self::Charge, bytes: u64) -> Result<()>; +} + +/// Admit a boxed value before constructing it. Its storage is freed before +/// the charge, including when construction fails or a decoder is replaced. +#[derive(Debug)] +pub(crate) struct Boxed { + value: Box, + _charge: B::Charge, +} +impl Boxed { + pub(crate) fn try_new>( + make: impl FnOnce() -> std::result::Result, + allowance: &B, + ) -> std::result::Result { + let mut charge = allowance.charge(); + B::resize(&mut charge, allocation_size::(1)?)?; + let value = make()?; + Ok(Self { + value: Box::new(value), + _charge: charge, + }) + } +} +impl std::ops::Deref for Boxed { + type Target = T; + fn deref(&self) -> &T { + &self.value + } +} +impl std::ops::DerefMut for Boxed { + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} +impl Budget for Allowance { + type Charge = (); + type Failure = std::convert::Infallible; + fn grow( + values: &mut Vec, + _: &mut (), + additional: usize, + ) -> std::result::Result<(), Self::Failure> { + values.reserve(additional); + Ok(()) + } + const LIMITED: bool = false; + fn charge(&self) {} + fn allowance(_: &()) -> Self { + Self::default() + } + fn resize(_: &mut (), _: u64) -> Result<()> { + Ok(()) + } +} + +impl Budget for Limited { + type Charge = Charge; + type Failure = Error; + fn grow(values: &mut Vec, charge: &mut Charge, additional: usize) -> Result<()> { + let end = values + .len() + .checked_add(additional) + .ok_or(Error::InvalidData("codec capacity overflows"))?; + if end <= values.capacity() { + return Ok(()); + } + let capacity = end.max(values.capacity().saturating_mul(2)).max(4); + let bytes = allocation_size::(capacity)?; + let peak = bytes + .checked_add(charge.bytes) + .ok_or(Error::InvalidData("codec capacity overflows"))?; + Self::resize(charge, peak)?; + let mut replacement = Vec::with_capacity(capacity); + replacement.append(values); + *values = replacement; + Self::resize(charge, bytes)?; + Ok(()) + } + const LIMITED: bool = true; + fn charge(&self) -> Charge { + Charge { + allowance: self.clone(), + bytes: 0, + } + } + fn allowance(charge: &Charge) -> Self { + charge.allowance.clone() + } + fn resize(charge: &mut Charge, bytes: u64) -> Result<()> { + charge.resize(bytes) + } +} + +/// Test policy that uses the production ledger but can refuse one chosen +/// growth operation. Keeping the policy beside [`Budget`] lets coverage count +/// its monomorphizations as tests of production generic code rather than as +/// test-module functions. +#[cfg(test)] +#[derive(Clone, Debug)] +pub(crate) struct RefusingBudget { + inner: Limited, + attempts: std::sync::Arc, + fail_at: usize, +} + +#[cfg(test)] +#[derive(Debug)] +pub(crate) struct RefusingCharge { + inner: Charge, + attempts: std::sync::Arc, + fail_at: usize, +} + +#[cfg(test)] +impl RefusingBudget { + pub(crate) fn new(fail_at: usize) -> Self { + Self { + inner: Allowance::limited(64 * 1024 * 1024), + attempts: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), + fail_at, + } + } + + pub(crate) fn attempts(&self) -> usize { + self.attempts.load(std::sync::atomic::Ordering::Relaxed) + } + + pub(crate) fn used(&self) -> u64 { + self.inner.used() + } +} + +#[cfg(test)] +fn refuse_once(attempts: &std::sync::atomic::AtomicUsize, fail_at: usize) -> Result<()> { + if attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == fail_at { + Err(Error::Cancelled) + } else { + Ok(()) + } +} + +#[cfg(test)] +impl Budget for RefusingBudget { + type Charge = RefusingCharge; + type Failure = Error; + const LIMITED: bool = true; + + fn grow(values: &mut Vec, charge: &mut Self::Charge, additional: usize) -> Result<()> { + refuse_once(&charge.attempts, charge.fail_at)?; + ::grow(values, &mut charge.inner, additional) + } + + fn charge(&self) -> Self::Charge { + RefusingCharge { + inner: ::charge(&self.inner), + attempts: self.attempts.clone(), + fail_at: self.fail_at, + } + } + + fn allowance(charge: &Self::Charge) -> Self { + Self { + inner: ::allowance(&charge.inner), + attempts: charge.attempts.clone(), + fail_at: charge.fail_at, + } + } + + fn resize(charge: &mut Self::Charge, bytes: u64) -> Result<()> { + if bytes > charge.inner.bytes() { + refuse_once(&charge.attempts, charge.fail_at)?; + } + ::resize(&mut charge.inner, bytes) + } +} + +/// Storage is dropped before its charge. Slice access cannot grow an allocation +/// behind the ledger's back; growth and ownership transfer are explicit. +#[derive(Debug)] +pub(crate) struct Buffer { + values: Vec, + charge: B::Charge, +} +impl Buffer { + pub(crate) fn new(allowance: &B) -> Self { + Self { + values: Vec::new(), + charge: allowance.charge(), + } + } + pub(crate) fn with_capacity(capacity: usize, allowance: &B) -> Result { + let bytes = allocation_size::(capacity)?; + let mut out = Self::new(allowance); + B::resize(&mut out.charge, bytes)?; + out.values = Vec::with_capacity(capacity); + Ok(out) + } + pub(crate) fn filled(len: usize, value: T, allowance: &B) -> Result + where + T: Clone, + { + // Keep vec![0; n]'s zeroed allocation fast path for finder links. + let bytes = allocation_size::(len)?; + let mut out = Self::new(allowance); + B::resize(&mut out.charge, bytes)?; + out.values = vec![value; len]; + Ok(out) + } + /// Admit one exact allocation before copying slices that form a buffer. + /// The total is checked first, so none of the copies can grow past its + /// charged capacity. + #[cfg(feature = "write")] + pub(crate) fn from_slices(slices: &[&[T]], allowance: &B) -> Result + where + T: Copy, + { + let len = slices.iter().try_fold(0usize, |total, slice| { + total + .checked_add(slice.len()) + .ok_or(Error::InvalidData("codec capacity overflows")) + })?; + let mut out = Self::with_capacity(len, allowance)?; + for slice in slices { + out.values.extend_from_slice(slice); + } + Ok(out) + } + pub(crate) fn capacity(&self) -> usize { + self.values.capacity() + } + pub(crate) fn discard_prefix(&mut self, count: usize) { + self.values.drain(..count); + } + pub(crate) fn extend_from_within(&mut self, range: std::ops::Range) -> Result<()> + where + T: Copy, + { + if B::LIMITED { + self.reserve(range.len())?; + } + self.values.extend_from_within(range); + Ok(()) + } + pub(crate) fn allowance(&self) -> B { + B::allowance(&self.charge) + } + fn reserve(&mut self, additional: usize) -> Result<()> { + B::grow(&mut self.values, &mut self.charge, additional).map_err(Into::into) + } + #[cfg(feature = "write")] + pub(crate) fn reserve_total_capacity(&mut self, total: usize) -> Result<()> { + if total > self.values.capacity() { + self.reserve(total.saturating_sub(self.values.len()))?; + } + debug_assert!(self.values.capacity() >= total); + Ok(()) + } + #[inline] + pub(crate) fn push(&mut self, value: T) -> std::result::Result<(), B::Failure> { + if B::LIMITED && self.values.len() == self.values.capacity() { + B::grow(&mut self.values, &mut self.charge, 1)?; + } + self.values.push(value); + Ok(()) + } + /// Append after the caller has admitted enough capacity for the complete + /// collection. This cannot grow the allocation or change its charge. + pub(crate) fn push_admitted(&mut self, value: T) { + debug_assert!(self.values.len() < self.values.capacity()); + self.values.push(value); + } + pub(crate) fn try_push(&mut self, value: T) -> Result<()> { + self.push(value).map_err(Into::into) + } + pub(crate) fn collect(values: impl IntoIterator, allowance: &B) -> Result { + let values = values.into_iter(); + let mut out = Self::with_capacity(values.size_hint().0, allowance)?; + for value in values { + out.try_push(value)?; + } + Ok(out) + } + pub(crate) fn retain(&mut self, keep: impl FnMut(&T) -> bool) { + self.values.retain(keep); + } + #[cfg(any(test, feature = "write", feature = "encryption"))] + pub(crate) fn truncate(&mut self, len: usize) { + self.values.truncate(len); + } + pub(crate) fn resize(&mut self, len: usize, value: T) -> Result<()> + where + T: Clone, + { + if B::LIMITED && len > self.values.capacity() { + self.reserve(len - self.values.len())?; + } + self.values.resize(len, value); + Ok(()) + } + #[cfg(all(test, feature = "write"))] + pub(crate) fn resize_with(&mut self, len: usize, make: impl FnMut() -> T) -> Result<()> { + if B::LIMITED && len > self.values.capacity() { + self.reserve(len - self.values.len())?; + } + self.values.resize_with(len, make); + Ok(()) + } + pub(crate) fn clear(&mut self) { + self.values.clear(); + } + pub(crate) fn insert(&mut self, index: usize, value: T) -> Result<()> { + if B::LIMITED { + self.reserve(1)?; + } + self.values.insert(index, value); + Ok(()) + } + pub(crate) fn remove(&mut self, index: usize) -> T { + self.values.remove(index) + } + pub(crate) fn try_collect( + values: impl ExactSizeIterator>, + allowance: &B, + ) -> Result { + let mut out = Self::with_capacity(values.len(), allowance)?; + for value in values { + out.push_admitted(value?); + } + Ok(out) + } + pub(crate) fn pop(&mut self) -> Option { + self.values.pop() + } + pub(crate) fn copied(values: &[T], allowance: &B) -> Result + where + T: Copy, + { + let mut out = Self::with_capacity(values.len(), allowance)?; + out.extend_from_slice(values).map_err(Into::into)?; + Ok(out) + } + /// Admit the final window before modifying it. A refusal keeps the old + /// history intact, and input larger than the window is never copied in full. + #[cfg(any(test, feature = "write"))] + pub(crate) fn remember(&mut self, input: &[T], limit: usize) -> Result<()> + where + T: Copy, + { + let input = &input[input.len().saturating_sub(limit)..]; + let keep = self.len().min(limit - input.len()); + self.reserve((keep + input.len()).saturating_sub(self.len()))?; + let start = self.len() - keep; + self.values.copy_within(start.., 0); + self.values.truncate(keep); + self.extend_from_slice(input).map_err(Into::into) + } + pub(crate) fn extend_from_slice(&mut self, values: &[T]) -> std::result::Result<(), B::Failure> + where + T: Copy, + { + if B::LIMITED && values.len() > self.values.capacity() - self.values.len() { + B::grow(&mut self.values, &mut self.charge, values.len())?; + } + self.values.extend_from_slice(values); + Ok(()) + } + #[cfg(any(test, feature = "write"))] + pub(crate) fn prepend(&mut self, prefix: impl ExactSizeIterator) -> Result<()> { + self.reserve(prefix.len())?; + let old_len = self.values.len(); + for value in prefix { + self.push(value).map_err(Into::into)?; + } + let inserted = self.values.len() - old_len; + self.values.rotate_right(inserted); + Ok(()) + } +} +/// A charged ring keeps its allocation ownership when converted to or from a +/// contiguous history. Neither conversion allocates new storage. +#[derive(Debug)] +pub(crate) struct Deque { + values: std::collections::VecDeque, + charge: B::Charge, +} +impl Deque { + pub(crate) fn from_buffer(buffer: Buffer) -> Self { + Self { + values: buffer.values.into(), + charge: buffer.charge, + } + } + pub(crate) fn into_buffer(self) -> Buffer { + Buffer { + values: self.values.into(), + charge: self.charge, + } + } + pub(crate) fn with_capacity(capacity: usize, allowance: &B) -> Result { + Ok(Self::from_buffer(Buffer::with_capacity( + capacity, allowance, + )?)) + } + pub(crate) fn allowance(&self) -> B { + B::allowance(&self.charge) + } + pub(crate) fn capacity(&self) -> usize { + self.values.capacity() + } + pub(crate) fn discard_prefix(&mut self, count: usize) { + self.values.drain(..count); + } + pub(crate) fn extend_admitted(&mut self, values: impl ExactSizeIterator) { + assert!(values.len() <= self.values.capacity() - self.values.len()); + self.values.extend(values); + } +} +impl std::ops::Deref for Deque { + type Target = std::collections::VecDeque; + fn deref(&self) -> &Self::Target { + &self.values + } +} +pub(crate) struct BufferIter { + values: std::vec::IntoIter, + _charge: B::Charge, +} +impl<'a, T, B: Budget> IntoIterator for &'a Buffer { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} +impl<'a, T, B: Budget> IntoIterator for &'a mut Buffer { + type Item = &'a mut T; + type IntoIter = std::slice::IterMut<'a, T>; + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} +impl IntoIterator for Buffer { + type Item = T; + type IntoIter = BufferIter; + fn into_iter(self) -> Self::IntoIter { + BufferIter { + values: self.values.into_iter(), + _charge: self.charge, + } + } +} +impl Iterator for BufferIter { + type Item = T; + fn next(&mut self) -> Option { + self.values.next() + } + fn size_hint(&self) -> (usize, Option) { + self.values.size_hint() + } +} +impl ExactSizeIterator for BufferIter {} +#[cfg(test)] +impl From> for Buffer { + fn from(values: Vec) -> Self { + Self::from_vec(values) + } +} +impl Buffer { + #[cfg(any(test, feature = "write"))] + pub(crate) fn from_vec(values: Vec) -> Self { + Self { values, charge: () } + } + /// Only unlimited buffers can cross an existing unaccounted Vec boundary. + pub(crate) fn into_vec(self) -> Vec { + self.values + } +} +impl Buffer { + pub(crate) fn read_to_end(&mut self, input: &mut impl std::io::Read) -> Result<()> { + let mut chunk = [0u8; 32 * 1024]; + loop { + let len = match input.read(&mut chunk) { + Ok(0) => return Ok(()), + Ok(len) => len, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(Error::from(error)), + }; + self.extend_from_slice(&chunk[..len]).map_err(Into::into)?; + } + } + #[cfg(feature = "write")] + pub(crate) fn write_msb_bits( + &mut self, + bit_pos: &mut usize, + value: u64, + count: usize, + ) -> std::result::Result<(), B::Failure> { + let used = *bit_pos % 8; + let additional = (used + count).div_ceil(8) - usize::from(used != 0); + if B::LIMITED && additional > self.values.capacity() - self.values.len() { + B::grow(&mut self.values, &mut self.charge, additional)?; + } + super::fast::write_msb_bits(&mut self.values, bit_pos, value, count); + Ok(()) + } + #[cfg(feature = "write")] + pub(crate) fn write_msb_bits_admitted( + &mut self, + bit_pos: &mut usize, + value: u64, + count: usize, + ) { + let used = *bit_pos % 8; + let additional = (used + count).div_ceil(8) - usize::from(used != 0); + debug_assert!(additional <= self.values.capacity() - self.values.len()); + super::fast::write_msb_bits(&mut self.values, bit_pos, value, count); + } +} +fn allocation_size(capacity: usize) -> Result { + capacity + .checked_mul(std::mem::size_of::()) + .filter(|bytes| *bytes <= isize::MAX as usize) + .map(|bytes| bytes as u64) + .ok_or(Error::InvalidData("codec capacity overflows")) +} +impl std::io::Write for Buffer { + fn write(&mut self, values: &[u8]) -> std::io::Result { + self.extend_from_slice(values) + .map_err(|error| std::io::Error::other(crate::Error::from(error.into())))?; + Ok(values.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +impl std::ops::Deref for Buffer { + type Target = [T]; + #[inline] + fn deref(&self) -> &[T] { + &self.values + } +} +impl std::ops::DerefMut for Buffer { + #[inline] + fn deref_mut(&mut self) -> &mut [T] { + &mut self.values + } +} +impl PartialEq> for Buffer { + fn eq(&self, other: &Buffer) -> bool { + **self == **other + } +} +impl PartialEq> for Buffer { + fn eq(&self, other: &Vec) -> bool { + &**self == other.as_slice() + } +} + +#[cfg(test)] +mod tests { + #[test] + fn limited_insert_preserves_reordered_storage_and_refuses_growth_atomically() { + let ledger = Allowance::limited(4); + let mut bytes = Buffer::copied(b"ABCD", &ledger).unwrap(); + let storage = bytes.as_ptr(); + let value = bytes.remove(2); + bytes.insert(0, value).unwrap(); + assert_eq!(&*bytes, b"CABD"); + assert_eq!(bytes.as_ptr(), storage); + assert_eq!(ledger.used(), 4); + assert!(matches!( + bytes.insert(0, b'X'), + Err(Error::WorkspaceLimitExceeded(_)) + )); + assert_eq!(&*bytes, b"CABD"); + assert_eq!(bytes.as_ptr(), storage); + assert_eq!(ledger.used(), 4); + drop(bytes); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn interrupted_reads_retry_and_buffer_flush_preserves_charge() { + use std::io::{self, Cursor, Read, Write}; + struct InterruptedOnce { + interrupted: bool, + data: Cursor<&'static [u8]>, + } + impl Read for InterruptedOnce { + fn read(&mut self, out: &mut [u8]) -> io::Result { + if !self.interrupted { + self.interrupted = true; + return Err(io::ErrorKind::Interrupted.into()); + } + self.data.read(out) + } + } + let ledger = Allowance::limited(16); + let mut input = InterruptedOnce { + interrupted: false, + data: Cursor::new(b"payload".as_slice()), + }; + let mut bytes = Buffer::new(&ledger); + bytes.read_to_end(&mut input).unwrap(); + assert!(input.interrupted); + assert_eq!(&*bytes, b"payload"); + let charged = ledger.used(); + assert!(charged >= 7); + bytes.flush().unwrap(); + assert_eq!(&*bytes, b"payload"); + assert_eq!(ledger.used(), charged); + drop(bytes); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn boxed_workspace_is_admitted_before_construction_and_released_on_failure() { + let small = Allowance::limited(31); + let mut constructed = false; + let denied = Boxed::try_new( + || { + constructed = true; + Ok::<_, Error>([0u8; 32]) + }, + &small, + ); + assert!(matches!(denied, Err(Error::WorkspaceLimitExceeded(_)))); + assert!(!constructed); + assert_eq!(small.used(), 0); + let budget = Allowance::limited(32); + let failed = + Boxed::<[u8; 32], _>::try_new(|| Err::<[u8; 32], _>(Error::Cancelled), &budget); + assert!(matches!(failed, Err(Error::Cancelled))); + assert_eq!(budget.used(), 0); + let value = Boxed::try_new(|| Ok::<_, Error>([7u8; 32]), &budget).unwrap(); + assert_eq!(budget.used(), 32); + assert_eq!(*value, [7; 32]); + drop(value); + assert_eq!(budget.used(), 0); + } + + use super::*; + + #[test] + fn reader_buffer_writer_preserves_typed_workspace_failure() { + use std::io::Write; + let ledger = Allowance::limited(0); + let mut buffer = Buffer::new(&ledger); + let error = buffer.write_all(b"cannot fit").unwrap_err(); + assert!(matches!( + Error::from(error), + Error::WorkspaceLimitExceeded(_) + )); + assert!(buffer.is_empty()); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn reader_ring_preserves_capacity_charge_through_wrapped_conversion() { + let ledger = Allowance::limited(8); + let buffer = Buffer::copied(b"ABCDEFGH", &ledger).unwrap(); + let storage = buffer.as_ptr(); + let mut ring = Deque::from_buffer(buffer); + ring.discard_prefix(5); + ring.extend_admitted(b"12345".iter().copied()); + assert!(!ring.as_slices().1.is_empty()); + assert_eq!(ledger.used(), 8); + let buffer = ring.into_buffer(); + assert_eq!(&*buffer, b"FGH12345"); + assert_eq!(buffer.as_ptr(), storage); + assert_eq!(buffer.capacity(), 8); + assert_eq!(ledger.used(), 8); + drop(buffer); + assert_eq!(ledger.used(), 0); + assert_eq!( + std::mem::size_of::>(), + std::mem::size_of::>() + ); + } + + #[test] + fn history_admits_before_mutation_and_only_copies_the_retained_tail() { + for limit in [10, 11] { + let allowance = Allowance::limited(limit); + let mut history = Buffer::copied(b"abc", &allowance).unwrap(); + let result = history.remember(b"0123456789", 8); + if limit == 10 { + assert!(matches!(result, Err(Error::WorkspaceLimitExceeded(_)))); + assert_eq!(&*history, b"abc"); + assert_eq!(allowance.used(), 3); + } else { + result.unwrap(); + assert_eq!(&*history, b"23456789"); + assert_eq!(allowance.used(), 8); + } + } + let input = vec![42; 1024 * 1024]; + let allowance = Allowance::limited(8); + let mut history = Buffer::new(&allowance); + history.remember(&input, 8).unwrap(); + history.remember(b"abc", 8).unwrap(); + assert_eq!(&*history, &[42, 42, 42, 42, 42, b'a', b'b', b'c']); + assert_eq!(allowance.used(), 8); + history.remember(b"discard", 0).unwrap(); + assert!(history.is_empty()); + assert_eq!(allowance.used(), 8, "spare capacity remains owned"); + drop(history); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn consuming_a_container_keeps_its_allocation_and_extracted_children_charged() { + let allowance = Allowance::limited(4096); + let mut owners = Buffer::with_capacity(2, &allowance).unwrap(); + owners + .push(Buffer::filled(8, 1u8, &allowance).unwrap()) + .unwrap(); + owners + .push(Buffer::filled(16, 2u8, &allowance).unwrap()) + .unwrap(); + let total = allowance.used(); + let mut iter = owners.into_iter(); + let first = iter.next().unwrap(); + assert_eq!(iter.len(), 1); + assert_eq!(allowance.used(), total); + drop(iter); + assert_eq!(allowance.used(), 8); + assert_eq!(&*first, &[1; 8]); + drop(first); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn unlimited_buffers_and_success_results_keep_the_existing_layout() { + assert_eq!( + std::mem::size_of::>(), + std::mem::size_of::>() + ); + assert!( + std::mem::size_of::>() <= std::mem::size_of::<(&str, usize)>(), + "large diagnostics must not widen hot codec results" + ); + } + + #[test] + fn growth_reserves_replacement_peak_and_refusal_preserves_storage() { + for limit in [23, 24] { + let allowance = Allowance::limited(limit); + let mut buffer = Buffer::filled(1, 7u64, &allowance).unwrap(); + // Bounded growth requests at least four elements: 8 old + 32 new. + assert!(matches!( + buffer.push(9), + Err(Error::WorkspaceLimitExceeded(details)) if details.required == 40 && details.used == 8 + )); + assert_eq!(&*buffer, &[7]); + assert_eq!(allowance.used(), 8); + drop(buffer); + assert_eq!(allowance.used(), 0); + } + let allowance = Allowance::limited(40); + let mut buffer = Buffer::filled(1, 7u64, &allowance).unwrap(); + buffer.push(9).unwrap(); + assert_eq!(&*buffer, &[7, 9]); + assert_eq!(allowance.used(), 32); + buffer.clear(); + assert_eq!(allowance.used(), 32, "spare capacity is still owned"); + drop(buffer); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn prepend_keeps_order_and_refuses_before_mutating_a_full_buffer() { + let allowance = Allowance::limited(8); + let mut buffer = Buffer::filled(4, 7u8, &allowance).unwrap(); + assert!(buffer.prepend([1, 2].into_iter()).is_err()); + assert_eq!(&*buffer, &[7, 7, 7, 7]); + assert_eq!(allowance.used(), 4); + drop(buffer); + let allowance = Allowance::limited(12); + let mut buffer = Buffer::filled(4, 7u8, &allowance).unwrap(); + buffer.prepend([1, 2].into_iter()).unwrap(); + assert_eq!(&*buffer, &[1, 2, 7, 7, 7, 7]); + assert_eq!(allowance.used(), 8); + } + + #[test] + fn ownership_keeps_charges_through_failure_and_unwind() { + struct Probe(Limited); + impl Drop for Probe { + fn drop(&mut self) { + assert!(self.0.used() > 0); + } + } + let allowance = Allowance::limited(4096); + let result = std::panic::catch_unwind(|| { + let mut values = Buffer::with_capacity(1, &allowance).unwrap(); + values.push(Probe(allowance.clone())).unwrap(); + assert!(Buffer::::with_capacity(4096, &allowance).is_err()); + panic!("unwind the worker"); + }); + assert!(result.is_err()); + assert_eq!(allowance.used(), 0); + drop(Buffer::::with_capacity(4096, &allowance).unwrap()); + assert_eq!(allowance.used(), 0); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn retained_buffers_move_between_threads_without_releasing_capacity() { + let allowance = Allowance::limited(32); + let retained = Buffer::filled(32, 1u8, &allowance).unwrap(); + let clone = allowance.clone(); + std::thread::spawn(move || { + assert_eq!(clone.used(), 32); + assert!(Buffer::::with_capacity(1, &clone).is_err()); + drop(retained); + assert_eq!(clone.used(), 0); + }) + .join() + .unwrap(); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn independent_allowances_do_not_race_for_each_others_spare_bytes() { + let first = Allowance::limited(16); + let second = Allowance::limited(16); + let kept = Buffer::::with_capacity(16, &first).unwrap(); + assert!(Buffer::::with_capacity(1, &first).is_err()); + drop(Buffer::::with_capacity(16, &second).unwrap()); + assert_eq!(first.used(), 16); + assert_eq!(second.used(), 0); + drop(kept); + assert_eq!(first.used(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/codec/workspace/coordinator.rs b/arca-rar/vendor/rars/src/codec/workspace/coordinator.rs new file mode 100644 index 00000000..502cdec7 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/workspace/coordinator.rs @@ -0,0 +1,217 @@ +//! Batch admission happens before dispatch. Workers cannot borrow a sibling's +//! unused allowance, and publication starts only after every callback joins. +use super::{Buffer, Limited, Reservation, RESERVATION_BYTES}; +use crate::{Error, Result}; +use std::sync::atomic::{AtomicBool, Ordering}; + +struct Slot { + item: T, + hint: u64, + reservation: Option, + result: Option>, +} + +pub(crate) fn extract_windowed( + items: I, + ledger: &Limited, + hint: H, + map: M, + mut publish: P, +) -> Result<()> +where + T: Copy + Send, + O: Send, + I: Iterator, + H: Fn(T) -> u64, + M: Fn(T, &Limited) -> Result + Sync + Send, + P: FnMut(O) -> Result<()>, +{ + let mut items = items.peekable(); + while let Some(&first) = items.peek() { + let overhead = std::mem::size_of::>() as u64; + let first_cost = hint(first) + .saturating_add(overhead) + .saturating_add(RESERVATION_BYTES); + ledger.check_capacity(first_cost)?; + let capacity = crate::parallel::default_window() + .max(1) + .min((ledger.available() / first_cost.max(1)).min(usize::MAX as u64) as usize); + let mut slots = Buffer::with_capacity(capacity, ledger)?; + let mut required = 0u64; + while slots.len() < capacity { + let Some(&item) = items.peek() else { break }; + let hint = hint(item); + let next = required + .saturating_add(hint) + .saturating_add(RESERVATION_BYTES); + if next > ledger.available() { + break; + } + items.next(); + slots.push_admitted(Slot { + item, + hint, + reservation: None, + result: None, + }); + required = next; + } + // Admission hints choose the batch width; they are not decoder upper bounds. + // The first-cost check includes its slot allocation, so at least one + // member fits. Only the coordinator holds the global allowance here. + debug_assert!(!slots.is_empty()); + let spare = ledger.available() - required; + let extra = spare / slots.len() as u64; + for slot in &mut slots { + slot.reservation = Some(ledger.reserve(slot.hint + extra)?); + } + let stopped = AtomicBool::new(false); + crate::parallel::for_each_mut(&mut slots, |_, slot| { + let mut reservation = slot.reservation.take().expect("admitted worker"); + if !stopped.load(Ordering::Acquire) { + reservation.start(); + let result = map(slot.item, &reservation.allowance()); + if result.is_err() { + stopped.store(true, Ordering::Release); + } + slot.result = Some(result); + } + reservation.retire(); + }); + // No successful sibling is published on a batch failure. Dropping slots + // frees their results, including results whose reservations have retired. + for slot in &mut slots { + if slot.result.as_ref().is_some_and(Result::is_err) { + return Err(slot.result.take().expect("failed worker").err().unwrap()); + } + } + for slot in &mut slots { + let result = slot.result.take().ok_or(Error::Cancelled)??; + publish(result)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::codec::workspace::Allowance; + use std::sync::atomic::AtomicUsize; + + #[test] + fn a_large_later_hint_waits_for_the_next_batch_in_publication_order() { + let overhead = std::mem::size_of::>>() as u64; + let limit = 4 * (overhead + RESERVATION_BYTES + 16); + let ledger = Allowance::limited(limit); + let published = AtomicUsize::new(0); + extract_windowed( + 0..3usize, + &ledger, + |index| { + if index == 1 { + limit - overhead - RESERVATION_BYTES + } else { + 16 + } + }, + |index, local| { + if index == 1 { + assert_eq!(published.load(Ordering::SeqCst), 1); + } + Ok(Buffer::filled(16, index as u8, local)?) + }, + |data| { + let expected = published.fetch_add(1, Ordering::SeqCst) as u8; + assert_eq!(&*data, &[expected; 16]); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(published.load(Ordering::SeqCst), 3); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn admitted_results_remain_charged_during_publication() { + let ledger = Allowance::limited(8192); + let published = AtomicUsize::new(0); + extract_windowed( + 0..4, + &ledger, + |_| 256, + |index, local| Ok(Buffer::filled(256, index as u8, local)?), + |data| { + assert!(ledger.used() >= data.len() as u64 + RESERVATION_BYTES); + assert!(ledger.used() <= ledger.limit()); + published.fetch_add(1, Ordering::Relaxed); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(published.load(Ordering::Relaxed), 4); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn admission_refusal_runs_no_worker() { + let ledger = Allowance::limited(1); + let error = extract_windowed( + 0..1, + &ledger, + |_| 1, + |_, _| -> Result<()> { panic!("unadmitted worker ran") }, + |_| panic!("unadmitted result published"), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn underestimated_hint_refuses_runtime_growth_without_publishing() { + let ledger = Allowance::limited(8192); + let error = extract_windowed( + 0..2, + &ledger, + |_| 1, + |_, local| Ok(Buffer::filled(16384, 0u8, local)?), + |_| panic!("refused result published"), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn worker_and_sink_failures_release_sibling_results_and_stop_batches() { + for sink_failure in [false, true] { + let ledger = Allowance::limited(8192); + let launched = AtomicUsize::new(0); + let published = AtomicUsize::new(0); + let error = extract_windowed( + 0..100, + &ledger, + |_| 256, + |_, local| { + launched.fetch_add(1, Ordering::Relaxed); + let data = Buffer::filled(256, 0u8, local)?; + if sink_failure { + Ok(data) + } else { + Err(Error::Cancelled) + } + }, + |_| { + published.fetch_add(1, Ordering::Relaxed); + Err(Error::Cancelled) + }, + ) + .unwrap_err(); + assert_eq!(error, Error::Cancelled); + assert!(launched.load(Ordering::Relaxed) <= crate::parallel::default_window()); + assert_eq!(published.load(Ordering::Relaxed), usize::from(sink_failure)); + assert_eq!(ledger.used(), 0); + } + } +} diff --git a/arca-rar/vendor/rars/src/codec/workspace/ledger.rs b/arca-rar/vendor/rars/src/codec/workspace/ledger.rs new file mode 100644 index 00000000..af24fea0 --- /dev/null +++ b/arca-rar/vendor/rars/src/codec/workspace/ledger.rs @@ -0,0 +1,286 @@ +//! Internal admission ledger. Reservations are fixed before dispatch; retiring +//! one returns only unused space, while escaped owners retain their charges. +use super::{Error, Result}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug)] +struct Ledger { + limit: u64, + used: Mutex, +} +#[derive(Debug)] +struct Usage { + ledger: Arc, + capacity: u64, + used: u64, + started: bool, + retired: bool, +} +// Count the per-reservation control payload until its last handle drops. +// Allocator/reference-count headers follow the existing overhead exclusion. +pub(crate) const RESERVATION_BYTES: u64 = std::mem::size_of::>() as u64; +impl Drop for Usage { + fn drop(&mut self) { + debug_assert!(self.retired && self.used == 0); + *self.ledger.used.lock().unwrap() -= RESERVATION_BYTES; + } +} +#[derive(Clone, Debug)] +pub(crate) struct Limited { + ledger: Arc, + scope: Option>>, +} +#[derive(Debug)] +pub(crate) struct Charge { + pub(super) allowance: Limited, + pub(super) bytes: u64, +} +#[derive(Debug)] +pub(crate) struct Reservation { + allowance: Limited, +} +fn refusal(limit: u64, required: u64, used: u64) -> Error { + Error::WorkspaceLimitExceeded(Box::new(crate::codec::WorkspaceLimitError { + limit, + required, + used, + })) +} +impl Limited { + pub(super) fn new(limit: u64) -> Self { + Self { + ledger: Arc::new(Ledger { + limit, + used: Mutex::new(0), + }), + scope: None, + } + } + #[cfg(any(test, feature = "write"))] + pub(crate) fn limit(&self) -> u64 { + self.ledger.limit + } + pub(crate) fn used(&self) -> u64 { + match &self.scope { + Some(scope) => scope.lock().unwrap().used, + None => *self.ledger.used.lock().unwrap(), + } + } + pub(crate) fn available(&self) -> u64 { + match &self.scope { + Some(scope) => { + let usage = scope.lock().unwrap(); + usage.capacity - usage.used + } + None => self.ledger.limit - self.used(), + } + } + pub(crate) fn check_capacity(&self, required: u64) -> Result<()> { + if let Some(scope) = &self.scope { + let usage = scope.lock().unwrap(); + if required > usage.capacity - usage.used { + return Err(refusal(usage.capacity, required, usage.used)); + } + } else { + let used = *self.ledger.used.lock().unwrap(); + if required > self.ledger.limit - used { + return Err(refusal(self.ledger.limit, required, used)); + } + } + Ok(()) + } + fn reserve_global(&self, bytes: u64, required: u64) -> Result<()> { + let mut used = self.ledger.used.lock().unwrap(); + if bytes > self.ledger.limit - *used { + return Err(refusal(self.ledger.limit, required, *used)); + } + *used += bytes; + Ok(()) + } + fn release_global(&self, bytes: u64) { + *self.ledger.used.lock().unwrap() -= bytes; + } + pub(crate) fn reserve(&self, capacity: u64) -> Result { + if self.scope.is_some() { + return Err(Error::InvalidData( + "workers cannot reserve global workspace", + )); + } + let committed = capacity + .checked_add(RESERVATION_BYTES) + .ok_or(Error::InvalidData("workspace reservation overflows"))?; + self.reserve_global(committed, committed)?; + Ok(Reservation { + allowance: Self { + ledger: self.ledger.clone(), + scope: Some(Arc::new(Mutex::new(Usage { + ledger: self.ledger.clone(), + capacity, + used: 0, + started: false, + retired: false, + }))), + }, + }) + } +} +impl Charge { + #[cfg(any(test, feature = "write"))] + pub(crate) fn bytes(&self) -> u64 { + self.bytes + } + pub(super) fn resize(&mut self, bytes: u64) -> Result<()> { + if let Some(scope) = &self.allowance.scope { + let mut usage = scope.lock().unwrap(); + if bytes > self.bytes { + let extra = bytes - self.bytes; + if extra > usage.capacity - usage.used { + return Err(refusal(usage.capacity, bytes, usage.used)); + } + usage.used += extra; + } else { + let released = self.bytes - bytes; + usage.used -= released; + if usage.retired { + usage.capacity -= released; + self.allowance.release_global(released); + } + } + } else if bytes > self.bytes { + self.allowance.reserve_global(bytes - self.bytes, bytes)?; + } else { + self.allowance.release_global(self.bytes - bytes); + } + self.bytes = bytes; + Ok(()) + } +} +impl Drop for Charge { + fn drop(&mut self) { + self.resize(0).expect("releasing capacity cannot fail"); + } +} +impl Reservation { + pub(crate) fn allowance(&self) -> Limited { + self.allowance.clone() + } + /// Extensions are a coordinator action before dispatch, never a worker's + /// attempt to take spare global capacity after an underestimate. + #[cfg(test)] + pub(crate) fn extend(&mut self, additional: u64) -> Result<()> { + let mut usage = self.allowance.scope.as_ref().unwrap().lock().unwrap(); + if usage.started || usage.retired { + return Err(Error::InvalidData("workspace extension after dispatch")); + } + let capacity = usage + .capacity + .checked_add(additional) + .ok_or(Error::InvalidData("workspace reservation overflows"))?; + self.allowance.reserve_global(additional, capacity)?; + usage.capacity = capacity; + Ok(()) + } + pub(crate) fn start(&mut self) { + let mut usage = self.allowance.scope.as_ref().unwrap().lock().unwrap(); + assert!(!usage.retired); + usage.started = true; + } + pub(crate) fn retire(self) { + drop(self); + } +} +impl Drop for Reservation { + fn drop(&mut self) { + let mut usage = self.allowance.scope.as_ref().unwrap().lock().unwrap(); + let unused = usage.capacity - usage.used; + usage.capacity = usage.used; + usage.retired = true; + self.allowance.release_global(unused); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::codec::workspace::Buffer; + + #[test] + fn scoped_capacity_never_borrows_global_spare_space_and_stays_retired() { + let ledger = Limited::new(128 + RESERVATION_BYTES); + let reservation = ledger.reserve(16).unwrap(); + let local = reservation.allowance(); + assert_eq!(local.available(), 16); + local.check_capacity(16).unwrap(); + assert!( + matches!(local.check_capacity(17), Err(Error::WorkspaceLimitExceeded(e)) if (e.limit, e.required, e.used) == (16, 17, 0)) + ); + let bytes = Buffer::filled(8, 1u8, &local).unwrap(); + assert_eq!(local.available(), 8); + assert!(ledger.available() > local.available()); + local.check_capacity(8).unwrap(); + assert!( + matches!(local.check_capacity(9), Err(Error::WorkspaceLimitExceeded(e)) if (e.limit, e.required, e.used) == (16, 9, 8)) + ); + reservation.retire(); + assert_eq!(local.available(), 0); + local.check_capacity(0).unwrap(); + assert!( + matches!(local.check_capacity(1), Err(Error::WorkspaceLimitExceeded(e)) if (e.limit, e.required, e.used) == (8, 1, 8)) + ); + drop(bytes); + assert_eq!(local.available(), 0); + assert_eq!(ledger.used(), RESERVATION_BYTES); + drop(local); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn retirement_releases_only_unused_reservations_and_keeps_escaped_outputs() { + let ledger = Limited::new(128 + 2 * RESERVATION_BYTES); + let mut first = ledger.reserve(64).unwrap(); + let mut second = ledger.reserve(32).unwrap(); + first.extend(16).unwrap(); + assert_eq!(ledger.used(), 112 + 2 * RESERVATION_BYTES); + assert!(ledger.reserve(17).is_err()); + first.start(); + second.start(); + assert!(first.extend(1).is_err()); + let a = first.allowance(); + let b = second.allowance(); + assert!(a.reserve(1).is_err()); + let left = Buffer::filled(48, 7u8, &a).unwrap(); + let mut right = Buffer::filled(16, 9u8, &b).unwrap(); + assert!(Buffer::filled(33, 0u8, &a).is_err()); + assert_eq!(ledger.used(), 112 + 2 * RESERVATION_BYTES); + first.retire(); + assert_eq!(ledger.used(), 80 + 2 * RESERVATION_BYTES); + second.retire(); + assert_eq!(ledger.used(), 64 + 2 * RESERVATION_BYTES); + assert!(right.resize(17, 0).is_err()); + assert_eq!(&*right, &[9u8; 16]); + drop(left); + assert_eq!(ledger.used(), 16 + 2 * RESERVATION_BYTES); + drop(right); + assert_eq!(ledger.used(), 2 * RESERVATION_BYTES); + assert!(Buffer::filled(1, 0u8, &a).is_err()); + drop((a, b)); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn failed_extension_and_unwind_release_the_admitted_scope() { + let ledger = Limited::new(64 + RESERVATION_BYTES); + let result = std::panic::catch_unwind(|| { + let mut reservation = ledger.reserve(48).unwrap(); + assert!(reservation.extend(17).is_err()); + assert_eq!(ledger.used(), 48 + RESERVATION_BYTES); + let _bytes = Buffer::filled(32, 0u8, &reservation.allowance()).unwrap(); + reservation.start(); + panic!("injected unwind"); + }); + assert!(result.is_err()); + assert_eq!(ledger.used(), 0); + ledger.reserve(64).unwrap().retire(); + assert_eq!(ledger.used(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/crc32/mod.rs b/arca-rar/vendor/rars/src/crc32/mod.rs new file mode 100644 index 00000000..891e1e71 --- /dev/null +++ b/arca-rar/vendor/rars/src/crc32/mod.rs @@ -0,0 +1,224 @@ +//! Shared RAR CRC-32 primitives. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Crc32 { + value: u32, +} + +impl Crc32 { + pub const fn new() -> Self { + Self { value: 0xffff_ffff } + } + + pub fn update(&mut self, input: &[u8]) { + self.value = update_raw(self.value, input); + } + + pub fn update_zeroes(&mut self, len: u64) { + let mut matrix = zero_byte_matrix(); + let mut count = len; + while count != 0 { + if count & 1 != 0 { + self.value = gf2_matrix_times(&matrix, self.value); + } + count >>= 1; + if count != 0 { + matrix = gf2_matrix_square(&matrix); + } + } + } + + pub const fn finish(self) -> u32 { + !self.value + } +} + +impl Default for Crc32 { + fn default() -> Self { + Self::new() + } +} + +pub fn crc32(input: &[u8]) -> u32 { + let mut crc = Crc32::new(); + crc.update(input); + crc.finish() +} + +pub fn crc32_raw(input: &[u8]) -> u32 { + update_raw(0xffff_ffff, input) +} + +pub fn table_entry(index: u8) -> u32 { + TABLE[index as usize] +} + +fn update_raw_byte(crc: u32, byte: u8) -> u32 { + (crc >> 8) ^ table_entry((crc as u8) ^ byte) +} + +fn update_raw(mut crc: u32, mut input: &[u8]) -> u32 { + while input.len() >= 8 { + let word = u32::from_le_bytes([input[0], input[1], input[2], input[3]]); + crc ^= word; + crc = TABLES[7][(crc & 0xff) as usize] + ^ TABLES[6][((crc >> 8) & 0xff) as usize] + ^ TABLES[5][((crc >> 16) & 0xff) as usize] + ^ TABLES[4][((crc >> 24) & 0xff) as usize] + ^ TABLES[3][input[4] as usize] + ^ TABLES[2][input[5] as usize] + ^ TABLES[1][input[6] as usize] + ^ TABLES[0][input[7] as usize]; + input = &input[8..]; + } + for &byte in input { + crc = update_raw_byte(crc, byte); + } + crc +} + +fn zero_byte_matrix() -> [u32; 32] { + let mut matrix = [0; 32]; + for (bit, slot) in matrix.iter_mut().enumerate() { + let mut value = 1u32 << bit; + let index = value as u8; + value = (value >> 8) ^ table_entry(index); + *slot = value; + } + matrix +} + +fn gf2_matrix_times(matrix: &[u32; 32], mut vector: u32) -> u32 { + let mut sum = 0; + let mut index = 0; + while vector != 0 { + if vector & 1 != 0 { + sum ^= matrix[index]; + } + vector >>= 1; + index += 1; + } + sum +} + +fn gf2_matrix_square(matrix: &[u32; 32]) -> [u32; 32] { + let mut square = [0; 32]; + for (index, slot) in square.iter_mut().enumerate() { + *slot = gf2_matrix_times(matrix, matrix[index]); + } + square +} + +const TABLES: [[u32; 256]; 8] = crc32_tables(); +const TABLE: [u32; 256] = TABLES[0]; + +const fn crc32_tables() -> [[u32; 256]; 8] { + let mut tables = [[0; 256]; 8]; + let mut i = 0; + while i < 256 { + let mut value = i as u32; + let mut bit = 0; + while bit < 8 { + let mask = 0u32.wrapping_sub(value & 1); + value = (value >> 1) ^ (0xedb8_8320 & mask); + bit += 1; + } + tables[0][i] = value; + i += 1; + } + + let mut table = 1; + while table < 8 { + let mut i = 0; + while i < 256 { + let previous = tables[table - 1][i]; + tables[table][i] = (previous >> 8) ^ tables[0][(previous & 0xff) as usize]; + i += 1; + } + table += 1; + } + tables +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crc32_matches_standard_check_value() { + assert_eq!(crc32(b"123456789"), 0xcbf4_3926); + assert_eq!(crc32(b""), 0); + } + + #[test] + fn default_accumulator_uses_the_standard_seed_and_finalization() { + assert_eq!(Crc32::default().finish(), 0); + let mut crc = Crc32::default(); + crc.update(b"1234"); + crc.update(b""); + crc.update(b"56789"); + assert_eq!(crc.finish(), 0xcbf4_3926); + } + + #[test] + fn raw_crc_matches_unfinalized_seeded_rar15_value() { + assert_eq!(crc32_raw(b"password"), 0xca3d_b92a); + } + + #[test] + fn update_zeroes_matches_byte_update() { + let mut skipped = Crc32::new(); + skipped.update(b"prefix"); + skipped.update_zeroes(1024); + skipped.update(b"suffix"); + + let mut bytewise = Crc32::new(); + bytewise.update(b"prefix"); + bytewise.update(&[0; 1024]); + bytewise.update(b"suffix"); + + assert_eq!(skipped.finish(), bytewise.finish()); + } + + fn reference_crc32(input: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + for &byte in input { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = 0u32.wrapping_sub(crc & 1); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc + } + + fn deterministic_bytes(len: usize) -> Vec { + let mut state = 0x1234_5678u32; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + out.push((state >> 24) as u8); + } + out + } + + #[test] + fn crc32_matches_bitwise_reference_across_chunk_boundaries() { + for len in 0..=257 { + let input = deterministic_bytes(len); + assert_eq!(crc32(&input), reference_crc32(&input), "len {len}"); + } + + for len in [1024, 4095, 4096, 4097, 65_536] { + let input = deterministic_bytes(len); + assert_eq!(crc32(&input), reference_crc32(&input), "len {len}"); + } + } + + #[test] + fn table_entry_matches_bitwise_generation() { + assert_eq!(table_entry(0), 0); + assert_eq!(table_entry(1), 0x7707_3096); + assert_eq!(table_entry(0xff), 0x2d02_ef8d); + } +} diff --git a/arca-rar/vendor/rars/src/crypto/mod.rs b/arca-rar/vendor/rars/src/crypto/mod.rs new file mode 100644 index 00000000..16d7f95e --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/mod.rs @@ -0,0 +1,90 @@ +//! RAR legacy and modern archive encryption primitives used by `rars`. + +pub mod rar13; +#[cfg(feature = "encryption")] +pub mod rar15; +pub mod rar20; +pub mod rar30; +pub mod rar50; + +#[cfg(not(feature = "encryption"))] +pub(crate) mod unavailable; + +/// The longest password any RAR key derivation sees. +/// +/// Reference builds copy the password into a 128-element `wchar_t` buffer and +/// keep 127 entries plus a terminator, so nothing past character 127 reaches +/// the KDF. Measured against RAR 7.12: a 133-character password and its first +/// 127 characters open the same archive, and the first 126 do not. +pub const MAX_PASSWORD_CHARS: usize = 127; + +/// Cuts a password down to the [`MAX_PASSWORD_CHARS`] the reference tools keep. +/// +/// Without this rars derives a key no official tool can reproduce, in both +/// directions: it cannot open a WinRAR archive when handed the same long +/// password, and the archives it writes from character 128 on cannot be opened +/// by WinRAR with that password or with any prefix of it. The boundary is +/// sharp, so a password of exactly 127 characters is untouched. +/// +/// A character here is a Unicode scalar, which matches the Linux reference +/// where `wchar_t` is 4 bytes wide. The Windows build has a 2-byte `wchar_t` +/// and so keeps 127 UTF-16 code units instead. The two disagree only for a +/// password carrying astral characters past position 127, where they derive +/// different keys from each other, so no single rule matches both. Scalars win +/// here because truncating UTF-16 can split a surrogate pair, and a Rust `str` +/// cannot hold the lone surrogate that would produce. +pub fn clamp_password(password: &[u8]) -> &[u8] { + match std::str::from_utf8(password) { + Ok(text) => match text.char_indices().nth(MAX_PASSWORD_CHARS) { + Some((byte_offset, _)) => &password[..byte_offset], + None => password, + }, + // RAR 1.3 through 2.0 take single-byte OEM/ANSI passwords, so one byte + // is one character. + Err(_) => &password[..password.len().min(MAX_PASSWORD_CHARS)], + } +} + +/// Refuses encrypted operations when the implementation is omitted. +pub(crate) fn require_encryption() -> crate::Result<()> { + if cfg!(feature = "encryption") { + Ok(()) + } else { + Err(crate::Error::FeatureDisabled { + feature: "encryption", + }) + } +} + +#[cfg(test)] +mod tests { + use super::{clamp_password, MAX_PASSWORD_CHARS}; + + #[test] + fn clamp_password_keeps_127_characters() { + let short = b"hunter2"; + assert_eq!(clamp_password(short), short); + + let exact = "A".repeat(MAX_PASSWORD_CHARS); + assert_eq!(clamp_password(exact.as_bytes()), exact.as_bytes()); + + let long = "A".repeat(130) + "ZZZ"; + assert_eq!(clamp_password(long.as_bytes()), exact.as_bytes()); + } + + #[test] + fn clamp_password_counts_characters_not_bytes() { + // 140 astral scalars: 560 UTF-8 bytes, 280 UTF-16 code units. + let emoji = "\u{1F600}".repeat(140); + let clamped = clamp_password(emoji.as_bytes()); + let text = std::str::from_utf8(clamped).unwrap(); + assert_eq!(text.chars().count(), MAX_PASSWORD_CHARS); + assert_eq!(clamped.len(), MAX_PASSWORD_CHARS * 4); + } + + #[test] + fn clamp_password_falls_back_to_bytes_when_not_utf8() { + let latin1 = vec![0xe9u8; 200]; + assert_eq!(clamp_password(&latin1).len(), MAX_PASSWORD_CHARS); + } +} diff --git a/arca-rar/vendor/rars/src/crypto/rar13.rs b/arca-rar/vendor/rars/src/crypto/rar13.rs new file mode 100644 index 00000000..043528cb --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar13.rs @@ -0,0 +1,107 @@ +#[cfg(feature = "encryption")] +use zeroize::ZeroizeOnDrop; + +#[cfg_attr(feature = "encryption", derive(ZeroizeOnDrop))] +pub struct Rar13Cipher { + key: [u8; 3], +} + +pub struct Rar13DecryptReader { + inner: R, + cipher: Rar13Cipher, +} + +impl Rar13Cipher { + #[cfg(feature = "encryption")] + pub fn new(password: &[u8]) -> Self { + let password = crate::crypto::clamp_password(password); + let mut key = [0u8; 3]; + for &byte in password { + key[0] = key[0].wrapping_add(byte); + key[1] ^= byte; + key[2] = key[2].wrapping_add(byte).rotate_left(1); + } + Self { key } + } + + pub(crate) fn for_password(password: &[u8]) -> crate::Result { + #[cfg(feature = "encryption")] + { + Ok(Self::new(password)) + } + #[cfg(not(feature = "encryption"))] + { + let _ = password; + Err(crate::Error::FeatureDisabled { + feature: "encryption", + }) + } + } + + pub fn new_comment() -> Self { + Self { key: [0, 7, 77] } + } + + #[cfg(any(test, feature = "write"))] + pub fn encrypt_in_place(mut self, data: &mut [u8]) { + for byte in data { + *byte = self.encrypt_byte(*byte); + } + } + + pub fn decrypt_in_place(mut self, data: &mut [u8]) { + for byte in data { + *byte = self.decrypt_byte(*byte); + } + } + + #[cfg(any(test, feature = "write"))] + pub fn encrypt_byte(&mut self, byte: u8) -> u8 { + self.advance(); + byte.wrapping_add(self.key[0]) + } + + pub fn decrypt_byte(&mut self, byte: u8) -> u8 { + self.advance(); + byte.wrapping_sub(self.key[0]) + } + + fn advance(&mut self) { + self.key[1] = self.key[1].wrapping_add(self.key[2]); + self.key[0] = self.key[0].wrapping_add(self.key[1]); + } +} + +impl Rar13DecryptReader { + pub fn new(inner: R, cipher: Rar13Cipher) -> Self { + Self { inner, cipher } + } +} + +impl std::io::Read for Rar13DecryptReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let read = self.inner.read(out)?; + for byte in &mut out[..read] { + *byte = self.cipher.decrypt_byte(*byte); + } + Ok(read) + } +} + +#[cfg(all(test, feature = "encryption"))] +mod tests { + use super::Rar13Cipher; + + #[test] + fn rar13_cipher_matches_pinned_stream_vector() { + let mut data = *b"hello world"; + Rar13Cipher::new(b"password").encrypt_in_place(&mut data); + assert_eq!( + data, + [0x37, 0xcd, 0xaa, 0xbd, 0x10, 0x4e, 0x6f, 0x6e, 0xb5, 0x30, 0xe6] + ); + + Rar13Cipher::new(b"password").decrypt_in_place(&mut data); + assert_eq!(&data, b"hello world"); + } +} diff --git a/arca-rar/vendor/rars/src/crypto/rar15.rs b/arca-rar/vendor/rars/src/crypto/rar15.rs new file mode 100644 index 00000000..d4e13a73 --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar15.rs @@ -0,0 +1,76 @@ +use crate::crc32::{crc32_raw, table_entry as crc32_table_entry}; +use zeroize::ZeroizeOnDrop; + +#[derive(ZeroizeOnDrop)] +pub struct Rar15Cipher { + key: [u16; 4], +} + +impl Rar15Cipher { + pub fn new(password: &[u8]) -> Self { + let password = crate::crypto::clamp_password(password); + let password = password.split(|&byte| byte == 0).next().unwrap_or(password); + let password_crc = crc32_raw(password); + let mut key = [ + (password_crc & 0xffff) as u16, + (password_crc >> 16) as u16, + 0, + 0, + ]; + + for &byte in password { + let crc = crc32_table_entry(byte); + key[2] ^= u16::from(byte) ^ crc as u16; + key[3] = key[3].wrapping_add(u16::from(byte).wrapping_add((crc >> 16) as u16)); + } + + Self { key } + } + + pub fn crypt_in_place(&mut self, data: &mut [u8]) { + for byte in data { + *byte ^= self.next_mask(); + } + } + + fn next_mask(&mut self) -> u8 { + self.key[0] = self.key[0].wrapping_add(0x1234); + let crc = crc32_table_entry(((self.key[0] & 0x01fe) >> 1) as u8); + self.key[1] ^= crc as u16; + self.key[2] = self.key[2].wrapping_sub((crc >> 16) as u16); + self.key[0] ^= self.key[2]; + self.key[3] = self.key[3].rotate_right(1) ^ self.key[1]; + self.key[3] = self.key[3].rotate_right(1); + self.key[0] ^= self.key[3]; + (self.key[0] >> 8) as u8 + } +} + +#[cfg(test)] +mod tests { + use super::Rar15Cipher; + + #[test] + fn crypts_known_rar15_stream_vector() { + let mut data = *b"hello world"; + Rar15Cipher::new(b"password").crypt_in_place(&mut data); + assert_eq!( + data, + [0x2b, 0xb9, 0xf3, 0x9c, 0x41, 0xa6, 0xaa, 0xe7, 0x1a, 0x7a, 0x7b] + ); + + Rar15Cipher::new(b"password").crypt_in_place(&mut data); + assert_eq!(&data, b"hello world"); + } + + #[test] + fn password_is_legacy_c_string() { + let mut truncated = *b"hello world"; + let mut plain = *b"hello world"; + + Rar15Cipher::new(b"password\0ignored").crypt_in_place(&mut truncated); + Rar15Cipher::new(b"password").crypt_in_place(&mut plain); + + assert_eq!(truncated, plain); + } +} diff --git a/arca-rar/vendor/rars/src/crypto/rar20.rs b/arca-rar/vendor/rars/src/crypto/rar20.rs new file mode 100644 index 00000000..aa42b8fb --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar20.rs @@ -0,0 +1,22 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + UnalignedInput, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnalignedInput => f.write_str("RAR 2.0 cipher input is not block aligned"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +#[cfg(feature = "encryption")] +mod implementation; +#[cfg(feature = "encryption")] +pub use implementation::*; diff --git a/arca-rar/vendor/rars/src/crypto/rar20/implementation.rs b/arca-rar/vendor/rars/src/crypto/rar20/implementation.rs new file mode 100644 index 00000000..3604a9aa --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar20/implementation.rs @@ -0,0 +1,233 @@ +use super::{Error, Result}; + +use crate::crc32::table_entry as crc32_table_entry; +use zeroize::{ZeroizeOnDrop, Zeroizing}; + +const INIT_SUBST_TABLE: [u8; 256] = [ + 215, 19, 149, 35, 73, 197, 192, 205, 249, 28, 16, 119, 48, 221, 2, 42, 232, 1, 177, 233, 14, + 88, 219, 25, 223, 195, 244, 90, 87, 239, 153, 137, 255, 199, 147, 70, 92, 66, 246, 13, 216, 40, + 62, 29, 217, 230, 86, 6, 71, 24, 171, 196, 101, 113, 218, 123, 93, 91, 163, 178, 202, 67, 44, + 235, 107, 250, 75, 234, 49, 167, 125, 211, 83, 114, 157, 144, 32, 193, 143, 36, 158, 124, 247, + 187, 89, 214, 141, 47, 121, 228, 61, 130, 213, 194, 174, 251, 97, 110, 54, 229, 115, 57, 152, + 94, 105, 243, 212, 55, 209, 245, 63, 11, 164, 200, 31, 156, 81, 176, 227, 21, 76, 99, 139, 188, + 127, 17, 248, 51, 207, 120, 189, 210, 8, 226, 41, 72, 183, 203, 135, 165, 166, 60, 98, 7, 122, + 38, 155, 170, 69, 172, 252, 238, 39, 134, 59, 128, 236, 27, 240, 80, 131, 3, 85, 206, 145, 79, + 154, 142, 159, 220, 201, 133, 74, 64, 20, 129, 224, 185, 138, 103, 173, 182, 43, 34, 254, 82, + 198, 151, 231, 180, 58, 10, 118, 26, 102, 12, 50, 132, 22, 191, 136, 111, 162, 179, 45, 4, 148, + 108, 161, 56, 78, 126, 242, 222, 15, 175, 146, 23, 33, 241, 181, 190, 77, 225, 0, 46, 169, 186, + 68, 95, 237, 65, 53, 208, 253, 168, 9, 18, 100, 52, 116, 184, 160, 96, 109, 37, 30, 106, 140, + 104, 150, 5, 204, 117, 112, 84, +]; + +const INIT_KEY: [u32; 4] = [0xd3a3_b879, 0x3f6d_12f7, 0x7515_a235, 0xa4e7_f123]; + +#[derive(ZeroizeOnDrop)] +pub struct Rar20Cipher { + key: [u32; 4], + subst: [u8; 256], +} + +impl Rar20Cipher { + pub fn new(password: &[u8]) -> Self { + let mut cipher = Self { + key: INIT_KEY, + subst: INIT_SUBST_TABLE, + }; + cipher.set_key(password); + cipher + } + + pub fn decrypt_in_place(&mut self, data: &mut [u8]) -> Result<()> { + if !data.len().is_multiple_of(16) { + return Err(Error::UnalignedInput); + } + for block in data.as_chunks_mut::<16>().0.iter_mut() { + self.decrypt_block(block); + } + Ok(()) + } + + #[cfg(any(test, feature = "write"))] + pub fn encrypt_in_place(&mut self, data: &mut [u8]) -> Result<()> { + if !data.len().is_multiple_of(16) { + return Err(Error::UnalignedInput); + } + for block in data.as_chunks_mut::<16>().0.iter_mut() { + self.encrypt_block(block); + } + Ok(()) + } + + fn set_key(&mut self, password: &[u8]) { + let password = crate::crypto::clamp_password(password); + for j in 0..=255u32 { + for i in (0..password.len()).step_by(2) { + let n1_index = password[i].wrapping_sub(j as u8); + let next = password.get(i + 1).copied().unwrap_or(0); + let n2_index = next.wrapping_add(j as u8); + let mut n1 = crc32_table_entry(n1_index) as u8; + let n2 = crc32_table_entry(n2_index) as u8; + let mut k = 1usize; + while n1 != n2 { + self.subst.swap(n1 as usize, (n1 as usize + i + k) & 0xff); + n1 = n1.wrapping_add(1); + k += 1; + } + } + } + + // clamp_password retains at most 127 Unicode scalars (four UTF-8 + // bytes each). Round that fixed upper bound up to a cipher block. + const PADDED_PASSWORD_BYTES: usize = + (crate::crypto::MAX_PASSWORD_CHARS * 4).div_ceil(16) * 16; + let mut padded = Zeroizing::new([0u8; PADDED_PASSWORD_BYTES]); + padded[..password.len()].copy_from_slice(password); + let padded_len = password.len().div_ceil(16) * 16; + for block in padded[..padded_len].as_chunks_mut::<16>().0.iter_mut() { + self.encrypt_block(block); + } + } + + fn encrypt_block(&mut self, block: &mut [u8]) { + let (mut a, mut b, mut c, mut d) = self.load_block(block); + for i in 0..32 { + (a, b, c, d) = self.round(i, a, b, c, d); + } + write_block( + block, + c ^ self.key[0], + d ^ self.key[1], + a ^ self.key[2], + b ^ self.key[3], + ); + self.update_keys(block); + } + + pub(crate) fn decrypt_block(&mut self, block: &mut [u8; 16]) { + let saved = *block; + let (mut a, mut b, mut c, mut d) = self.load_block(block); + for i in (0..32).rev() { + (a, b, c, d) = self.round(i, a, b, c, d); + } + write_block( + block, + c ^ self.key[0], + d ^ self.key[1], + a ^ self.key[2], + b ^ self.key[3], + ); + self.update_keys(&saved); + } + + fn load_block(&self, block: &[u8]) -> (u32, u32, u32, u32) { + ( + read_u32_le(&block[0..4]) ^ self.key[0], + read_u32_le(&block[4..8]) ^ self.key[1], + read_u32_le(&block[8..12]) ^ self.key[2], + read_u32_le(&block[12..16]) ^ self.key[3], + ) + } + + fn round(&self, i: usize, a: u32, b: u32, c: u32, d: u32) -> (u32, u32, u32, u32) { + let t = c.wrapping_add(d.rotate_left(11)) ^ self.key[i & 3]; + let ta = a ^ self.subst_long(t); + let t = (d ^ c.rotate_left(17)).wrapping_add(self.key[i & 3]); + let tb = b ^ self.subst_long(t); + (c, d, ta, tb) + } + + fn subst_long(&self, value: u32) -> u32 { + u32::from(self.subst[(value & 0xff) as usize]) + | (u32::from(self.subst[((value >> 8) & 0xff) as usize]) << 8) + | (u32::from(self.subst[((value >> 16) & 0xff) as usize]) << 16) + | (u32::from(self.subst[((value >> 24) & 0xff) as usize]) << 24) + } + + fn update_keys(&mut self, block: &[u8]) { + for chunk in block.as_chunks::<4>().0.iter() { + self.key[0] ^= crc32_table_entry(chunk[0]); + self.key[1] ^= crc32_table_entry(chunk[1]); + self.key[2] ^= crc32_table_entry(chunk[2]); + self.key[3] ^= crc32_table_entry(chunk[3]); + } + } +} + +fn read_u32_le(bytes: &[u8]) -> u32 { + u32::from_le_bytes(bytes.try_into().expect("u32 input size")) +} + +fn write_block(block: &mut [u8], a: u32, b: u32, c: u32, d: u32) { + block[0..4].copy_from_slice(&a.to_le_bytes()); + block[4..8].copy_from_slice(&b.to_le_bytes()); + block[8..12].copy_from_slice(&c.to_le_bytes()); + block[12..16].copy_from_slice(&d.to_le_bytes()); +} + +#[cfg(test)] +mod tests { + use super::{Error, Rar20Cipher}; + + #[test] + fn rar20_encrypt_decrypt_round_trips_blocks() { + let mut encrypted = *b"0123456789abcdefRAR2.0 block pad"; + let original = encrypted; + + Rar20Cipher::new(b"password") + .encrypt_in_place(&mut encrypted) + .unwrap(); + assert_eq!( + encrypted, + [ + 0xb7, 0x14, 0x54, 0x5a, 0x55, 0x8b, 0xca, 0xf7, 0xbc, 0x18, 0x38, 0x17, 0x1d, 0x9e, + 0x31, 0xab, 0x81, 0x40, 0x72, 0xfe, 0x02, 0x76, 0x76, 0x65, 0x4a, 0xa5, 0x3f, 0x4b, + 0xb3, 0x0c, 0xad, 0x07, + ] + ); + + Rar20Cipher::new(b"password") + .decrypt_in_place(&mut encrypted) + .unwrap(); + assert_eq!(encrypted, original); + } + + #[test] + fn rar20_maximum_unicode_password_preserves_the_pinned_key_schedule() { + let password = "😀".repeat(crate::crypto::MAX_PASSWORD_CHARS); + let mut encrypted = *b"0123456789abcdef"; + Rar20Cipher::new(password.as_bytes()) + .encrypt_in_place(&mut encrypted) + .unwrap(); + // Captured from the former Vec-based key schedule before replacing it. + assert_eq!( + encrypted, + [162, 163, 252, 197, 198, 249, 120, 35, 101, 82, 12, 143, 172, 74, 33, 216] + ); + let mut longer = *b"0123456789abcdef"; + Rar20Cipher::new((password.clone() + "ignored").as_bytes()) + .encrypt_in_place(&mut longer) + .unwrap(); + assert_eq!(longer, encrypted); + Rar20Cipher::new(password.as_bytes()) + .decrypt_in_place(&mut encrypted) + .unwrap(); + assert_eq!(&encrypted, b"0123456789abcdef"); + } + + #[test] + fn rar20_cipher_rejects_partial_tail() { + let mut data = *b"0123456789abcdef!"; + let original = data; + + assert_eq!( + Rar20Cipher::new(b"password").encrypt_in_place(&mut data), + Err(Error::UnalignedInput) + ); + assert_eq!(data, original); + assert_eq!( + Rar20Cipher::new(b"password").decrypt_in_place(&mut data), + Err(Error::UnalignedInput) + ); + assert_eq!(data, original); + } +} diff --git a/arca-rar/vendor/rars/src/crypto/rar30.rs b/arca-rar/vendor/rars/src/crypto/rar30.rs new file mode 100644 index 00000000..54dbde9f --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar30.rs @@ -0,0 +1,24 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + NonUtf8Password, + UnalignedInput, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NonUtf8Password => f.write_str("RAR 3.x password is not UTF-8"), + Self::UnalignedInput => f.write_str("RAR 3.x AES input is not block aligned"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +#[cfg(feature = "encryption")] +mod implementation; +#[cfg(feature = "encryption")] +pub use implementation::*; diff --git a/arca-rar/vendor/rars/src/crypto/rar30/implementation.rs b/arca-rar/vendor/rars/src/crypto/rar30/implementation.rs new file mode 100644 index 00000000..91d6f023 --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar30/implementation.rs @@ -0,0 +1,312 @@ +use super::{Error, Result}; + +#[cfg(any(test, feature = "write"))] +use aes::cipher::BlockCipherEncrypt; +use aes::cipher::{BlockCipherDecrypt, KeyInit}; +use aes::Aes128; +use sha1::{Digest, Sha1 as FastSha1}; +use std::str; +use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; + +const HASH_ROUNDS: u32 = 0x40000; + +#[derive(Clone, ZeroizeOnDrop)] +pub struct Rar30Cipher { + cipher: Aes128, + iv: [u8; 16], +} + +impl Rar30Cipher { + pub fn new(password: &[u8], salt: Option<[u8; 8]>) -> Result { + let (mut key, iv) = derive_key_iv(password, salt)?; + let cipher = Aes128::new(&key.into()); + key.zeroize(); + Ok(Self { cipher, iv }) + } + + pub fn decrypt_in_place(&mut self, data: &mut [u8]) -> Result<()> { + if !data.len().is_multiple_of(16) { + return Err(Error::UnalignedInput); + } + for block in data.as_chunks_mut::<16>().0.iter_mut() { + self.decrypt_block(block); + } + Ok(()) + } + + #[cfg(any(test, feature = "write"))] + pub fn encrypt_in_place(&mut self, data: &mut [u8]) -> Result<()> { + if !data.len().is_multiple_of(16) { + return Err(Error::UnalignedInput); + } + for block in data.as_chunks_mut::<16>().0.iter_mut() { + self.encrypt_block(block); + } + Ok(()) + } + + #[cfg(any(test, feature = "write"))] + fn encrypt_block(&mut self, block: &mut [u8]) { + for (byte, iv_byte) in block.iter_mut().zip(self.iv) { + *byte ^= iv_byte; + } + let block: &mut [u8; 16] = block.try_into().expect("AES block size"); + self.cipher.encrypt_block(block.into()); + self.iv.copy_from_slice(block); + } + + pub(crate) fn decrypt_block(&mut self, block: &mut [u8; 16]) { + let ciphertext = *block; + self.cipher.decrypt_block(block.into()); + for (byte, iv_byte) in block.iter_mut().zip(self.iv) { + *byte ^= iv_byte; + } + self.iv = ciphertext; + } +} + +fn derive_key_iv(password: &[u8], salt: Option<[u8; 8]>) -> Result<([u8; 16], [u8; 16])> { + let password = crate::crypto::clamp_password(password); + // The reference password ceiling is in Unicode scalar values. Each can + // contribute at most two UTF-16 units, followed by an optional eight-byte salt. + let mut storage = Zeroizing::new([0u8; crate::crypto::MAX_PASSWORD_CHARS * 4 + 8]); + let mut used = 0; + let password = str::from_utf8(password).map_err(|_| Error::NonUtf8Password)?; + for code_unit in password.encode_utf16() { + storage[used..used + 2].copy_from_slice(&code_unit.to_le_bytes()); + used += 2; + } + if let Some(salt) = salt { + storage[used..used + salt.len()].copy_from_slice(&salt); + used += salt.len(); + } + + let raw = &mut storage[..used]; + + // RAR 3.x mutates password/salt bytes only when the repeated KDF input + // crosses complete SHA-1 blocks. The stock SHA-1 path is equivalent while + // the password+salt material never fills a 64-byte block. + if raw.len() < 64 { + return Ok(derive_key_iv_fast(raw)); + } + + Ok(derive_key_iv_slow(raw)) +} + +fn derive_key_iv_slow(raw: &mut [u8]) -> ([u8; 16], [u8; 16]) { + let raw_size = raw.len(); + let mut storage = Zeroizing::new([0u8; crate::crypto::MAX_PASSWORD_CHARS * 4 + 8 + 64]); + storage[..raw_size].copy_from_slice(raw); + let raw = &mut storage[..raw_size + 64]; + let mut sha1 = FastSha1::new(); + let mut iv = [0; 16]; + let mut pos = 0u32; + for i in 0..HASH_ROUNDS { + sha1.update(&raw[..raw_size]); + let end_pos = (pos + raw_size as u32) & !(64 - 1); + if end_pos > pos + 64 { + let mut cur_pos = (pos & !(64 - 1)) + 64; + while cur_pos != end_pos { + let offset = (cur_pos - pos) as usize; + update_password_data_sha1(&mut raw[offset..offset + 64]); + cur_pos += 64; + } + } + pos = pos.wrapping_add(raw_size as u32); + + sha1.update([ + (i & 0xff) as u8, + ((i >> 8) & 0xff) as u8, + ((i >> 16) & 0xff) as u8, + ]); + pos = pos.wrapping_add(3); + if i.is_multiple_of(HASH_ROUNDS / 16) { + let digest = sha1.clone().finalize(); + iv[(i / (HASH_ROUNDS / 16)) as usize] = digest[19]; + } + } + + let digest = sha1.finalize(); + let mut key = [0; 16]; + for (word_index, chunk) in digest[..16].as_chunks::<4>().0.iter().enumerate() { + key[word_index * 4..word_index * 4 + 4] + .copy_from_slice(&[chunk[3], chunk[2], chunk[1], chunk[0]]); + } + (key, iv) +} + +fn update_password_data_sha1(data: &mut [u8]) { + let mut w = [0u32; 80]; + for (i, chunk) in data.as_chunks::<4>().0.iter().take(16).enumerate() { + w[i] = u32::from_be_bytes(*chunk); + } + for i in 16..80 { + w[i] = (w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]).rotate_left(1); + } + for (i, word) in w[64..80].iter().enumerate() { + data[i * 4..i * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } +} + +fn derive_key_iv_fast(raw: &[u8]) -> ([u8; 16], [u8; 16]) { + let mut sha1 = FastSha1::new(); + let mut iv = [0; 16]; + for i in 0..HASH_ROUNDS { + sha1.update(raw); + sha1.update([ + (i & 0xff) as u8, + ((i >> 8) & 0xff) as u8, + ((i >> 16) & 0xff) as u8, + ]); + if i.is_multiple_of(HASH_ROUNDS / 16) { + let digest = sha1.clone().finalize(); + iv[(i / (HASH_ROUNDS / 16)) as usize] = digest[19]; + } + } + + let digest = sha1.finalize(); + let mut key = [0; 16]; + for (word_index, chunk) in digest[..16].as_chunks::<4>().0.iter().enumerate() { + key[word_index * 4..word_index * 4 + 4] + .copy_from_slice(&[chunk[3], chunk[2], chunk[1], chunk[0]]); + } + (key, iv) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maximum_supplementary_unicode_password_fits_bounded_kdf_scratch() { + let password = "🙂".repeat(crate::crypto::MAX_PASSWORD_CHARS); + let longer = format!("{password}🙂"); + let mut exact = *b"0123456789abcdef"; + let mut clamped = exact; + let salt = Some(*b"longsalt"); + Rar30Cipher::new(password.as_bytes(), salt) + .unwrap() + .encrypt_in_place(&mut exact) + .unwrap(); + Rar30Cipher::new(longer.as_bytes(), salt) + .unwrap() + .encrypt_in_place(&mut clamped) + .unwrap(); + assert_eq!(exact, clamped); + } + + fn raw_kdf_material(password: &[u8], salt: Option<[u8; 8]>) -> Vec { + let mut raw = Vec::with_capacity(password.len() * 2 + 8); + let password = str::from_utf8(password).unwrap(); + for code_unit in password.encode_utf16() { + raw.extend_from_slice(&code_unit.to_le_bytes()); + } + if let Some(salt) = salt { + raw.extend_from_slice(&salt); + } + raw + } + + #[test] + fn rar30_aes_encrypt_decrypt_round_trips_blocks() { + let salt = Some([1, 2, 3, 4, 5, 6, 7, 8]); + let mut data = *b"0123456789abcdefRAR AES CBC data"; + let plain = data; + + Rar30Cipher::new(b"password", salt) + .unwrap() + .encrypt_in_place(&mut data) + .unwrap(); + assert_eq!( + data, + [ + 0x5e, 0x59, 0xce, 0xa1, 0x16, 0xca, 0xa2, 0x1d, 0x4d, 0xc5, 0x05, 0xeb, 0xa9, 0x3f, + 0x7b, 0xcd, 0x0d, 0x04, 0xff, 0xea, 0x60, 0x67, 0x3d, 0xaf, 0x6a, 0x8f, 0x02, 0xb2, + 0x03, 0xc8, 0x7d, 0xde, + ] + ); + + Rar30Cipher::new(b"password", salt) + .unwrap() + .decrypt_in_place(&mut data) + .unwrap(); + assert_eq!(data, plain); + } + + #[test] + fn rar30_aes_round_trips_with_long_password_slow_path() { + // Password long enough that utf-16(password) + 8-byte salt >= 64, + // forcing derive_key_iv to use the RAR3 password-buffer mutation path + // instead of derive_key_iv_fast. + let password = b"this-password-is-deliberately-long-enough-to-exceed-64-bytes-utf16"; + let salt = Some(*b"longsalt"); + let mut data = *b"0123456789abcdefRAR AES CBC data"; + let plain = data; + + Rar30Cipher::new(password, salt) + .unwrap() + .encrypt_in_place(&mut data) + .unwrap(); + assert_eq!( + data, + [ + 0xb9, 0xa7, 0xac, 0x4b, 0x81, 0x0a, 0x5c, 0xf1, 0x6e, 0xd4, 0x5a, 0x4c, 0xbc, 0x1e, + 0x2e, 0xef, 0x53, 0x7b, 0x89, 0x63, 0x7a, 0xc5, 0x7a, 0x1e, 0xfc, 0x43, 0x3c, 0x18, + 0xea, 0xfd, 0x54, 0xed, + ] + ); + + Rar30Cipher::new(password, salt) + .unwrap() + .decrypt_in_place(&mut data) + .unwrap(); + assert_eq!(data, plain); + } + + #[test] + fn rar30_aes_rejects_partial_tail() { + let mut data = *b"partial block!!"; + + assert_eq!( + Rar30Cipher::new(b"password", None) + .unwrap() + .encrypt_in_place(&mut data), + Err(Error::UnalignedInput) + ); + assert_eq!( + Rar30Cipher::new(b"password", None) + .unwrap() + .decrypt_in_place(&mut data), + Err(Error::UnalignedInput) + ); + } + + #[test] + fn rejects_non_utf8_passwords() { + assert!(matches!( + Rar30Cipher::new(b"\xffpassword", None), + Err(Error::NonUtf8Password) + )); + } + + #[test] + fn rar30_fast_kdf_matches_reference_path_for_short_material() { + for (password, salt) in [ + (b"".as_slice(), None), + (b"password".as_slice(), Some(*b"rarsalt!")), + ("páss".as_bytes(), Some([1, 2, 3, 4, 5, 6, 7, 8])), + ] { + let raw = raw_kdf_material(password, salt); + assert!( + raw.len() < 64, + "case should exercise the fast-path precondition" + ); + + let fast = derive_key_iv_fast(&raw); + let mut reference_raw = raw.clone(); + let reference = derive_key_iv_slow(&mut reference_raw); + + assert_eq!(fast, reference); + } + } +} diff --git a/arca-rar/vendor/rars/src/crypto/rar50.rs b/arca-rar/vendor/rars/src/crypto/rar50.rs new file mode 100644 index 00000000..931b73f1 --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar50.rs @@ -0,0 +1,122 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + KdfCountTooLarge, + BadPassword, + UnalignedInput, + FeatureDisabled, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::KdfCountTooLarge => f.write_str("RAR 5 KDF count is too large"), + Self::BadPassword => f.write_str("wrong password or corrupt encrypted data"), + Self::FeatureDisabled => f.write_str("Cargo feature encryption is disabled"), + Self::UnalignedInput => f.write_str("RAR 5 AES input is not block aligned"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// The PBKDF2 iteration exponent this writer stores in new archives, giving +/// `2^15` = 32768 iterations. +/// +/// WinRAR writes 15 and a reader honours whatever the archive declares, so the +/// only thing a smaller exponent buys is a faster offline password guess +/// against the archives we produce. It costs about 15 ms per encrypted member +/// on the write side, which is what WinRAR pays too. +pub const WRITE_KDF_COUNT_LOG: u8 = 15; + +#[derive(Clone)] +#[cfg_attr(feature = "encryption", derive(zeroize::ZeroizeOnDrop))] +#[non_exhaustive] +pub struct Rar50Keys { + pub key: [u8; 32], + pub hash_key: [u8; 32], + pub password_check: [u8; 8], +} + +impl std::fmt::Debug for Rar50Keys { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Rar50Keys").finish_non_exhaustive() + } +} + +impl PartialEq for Rar50Keys { + fn eq(&self, other: &Self) -> bool { + let key_eq = constant_time_eq(&self.key, &other.key); + let hash_eq = constant_time_eq(&self.hash_key, &other.hash_key); + let check_eq = constant_time_eq(&self.password_check, &other.password_check); + key_eq & hash_eq & check_eq + } +} + +impl Eq for Rar50Keys {} + +fn constant_time_eq(left: &[u8; N], right: &[u8; N]) -> bool { + let mut diff = 0u8; + for (&left, &right) in left.iter().zip(right) { + diff |= left ^ right; + } + diff == 0 +} + +#[cfg(feature = "encryption")] +mod implementation; +#[cfg(feature = "encryption")] +pub use implementation::*; + +impl Rar50Keys { + pub(crate) fn checked_crc_mac(&self, value: u32) -> crate::Result { + #[cfg(feature = "encryption")] + { + Ok(self.mac_crc32(value)) + } + #[cfg(not(feature = "encryption"))] + { + let _ = value; + Err(crate::Error::FeatureDisabled { + feature: "encryption", + }) + } + } + pub(crate) fn checked_hash_mac(&self, value: [u8; 32]) -> crate::Result<[u8; 32]> { + #[cfg(feature = "encryption")] + { + Ok(self.mac_hash32(value)) + } + #[cfg(not(feature = "encryption"))] + { + let _ = value; + Err(crate::Error::FeatureDisabled { + feature: "encryption", + }) + } + } +} + +#[cfg(not(feature = "encryption"))] +impl Rar50Keys { + pub fn derive(_password: &[u8], _salt: [u8; 16], _kdf_count_log: u8) -> Result { + Err(Error::FeatureDisabled) + } +} +#[cfg(feature = "write")] +impl Rar50Keys { + pub(crate) fn checked_password_record(&self) -> crate::Result<[u8; 12]> { + #[cfg(feature = "encryption")] + { + Ok(self.password_check_record()) + } + #[cfg(not(feature = "encryption"))] + { + Err(crate::Error::FeatureDisabled { + feature: "encryption", + }) + } + } +} diff --git a/arca-rar/vendor/rars/src/crypto/rar50/implementation.rs b/arca-rar/vendor/rars/src/crypto/rar50/implementation.rs new file mode 100644 index 00000000..591f25f7 --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/rar50/implementation.rs @@ -0,0 +1,280 @@ +use super::{constant_time_eq, Error, Rar50Keys, Result}; + +#[cfg(any(test, feature = "write", feature = "recovery"))] +use aes::cipher::BlockCipherEncrypt; +use aes::cipher::{BlockCipherDecrypt, KeyInit}; +use aes::Aes256; +use hmac::{Hmac, Mac}; +use sha2::{Digest, Sha256}; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +const MAX_KDF_COUNT_LOG: u8 = 24; + +type HmacSha256 = Hmac; + +impl Rar50Keys { + pub fn derive(password: &[u8], salt: [u8; 16], kdf_count_log: u8) -> Result { + if kdf_count_log > MAX_KDF_COUNT_LOG { + return Err(Error::KdfCountTooLarge); + } + let password = crate::crypto::clamp_password(password); + + let mut first_input = [0u8; 20]; + first_input[..16].copy_from_slice(&salt); + first_input[16..].copy_from_slice(&1u32.to_be_bytes()); + + let mut u = hmac_sha256(password, &first_input); + let mut accumulator = u; + let mut taps = [[0u8; 32]; 3]; + let mut iterations = (1u32 << kdf_count_log) - 1; + + for tap in &mut taps { + for _ in 0..iterations { + u = hmac_sha256(password, &u); + for (acc, byte) in accumulator.iter_mut().zip(u) { + *acc ^= byte; + } + } + *tap = accumulator; + iterations = 16; + } + + let mut password_check = [0u8; 8]; + for (i, byte) in password_check.iter_mut().enumerate() { + *byte = taps[2][i] ^ taps[2][i + 8] ^ taps[2][i + 16] ^ taps[2][i + 24]; + } + + let result = Self { + key: taps[0], + hash_key: taps[1], + password_check, + }; + u.zeroize(); + accumulator.zeroize(); + taps.zeroize(); + Ok(result) + } + + /// Verifies the password against a stored check record, unless the record + /// is the all-zero one WinRAR 5.21 and earlier wrote. + /// + /// Those versions set the check-present flag and then left the eight bytes + /// zero, so verifying rejects the correct password and the archive cannot + /// be opened at all. Treat all-zero as "no check available" and let the + /// data checksum decide instead. + /// + /// The reference readers do the same. Zeroing the field in a WinRAR 7.12 + /// archive leaves the right password working, and changes what a wrong one + /// reports from "Incorrect password" to a checksum error, which is the + /// explicit check dropping out and detection falling through. Zeroing the + /// record's own trailing checksum or leaving it stale makes no difference + /// to that, so the all-zero test comes first here too. + pub fn check_password(&self, stored: &[u8; 12]) -> Result<()> { + if stored[..8] == [0u8; 8] { + return Ok(()); + } + let checksum = sha256(&stored[..8]); + let checksum_matches = constant_time_eq( + checksum.first_chunk::<4>().unwrap(), + stored.last_chunk::<4>().unwrap(), + ); + let password_matches = + constant_time_eq(&self.password_check, stored.first_chunk::<8>().unwrap()); + if !(checksum_matches & password_matches) { + return Err(Error::BadPassword); + } + Ok(()) + } + + pub fn password_check_record(&self) -> [u8; 12] { + let mut record = [0u8; 12]; + record[..8].copy_from_slice(&self.password_check); + record[8..].copy_from_slice(&sha256(&self.password_check)[..4]); + record + } + + pub fn mac_crc32(&self, crc: u32) -> u32 { + let digest = hmac_sha256(&self.hash_key, &crc.to_le_bytes()); + digest + .as_chunks::<4>() + .0 + .iter() + .fold(0, |acc, chunk| acc ^ u32::from_le_bytes(*chunk)) + } + + pub fn mac_hash32(&self, hash: [u8; 32]) -> [u8; 32] { + hmac_sha256(&self.hash_key, &hash) + } +} + +#[derive(ZeroizeOnDrop)] +pub struct Rar50Cipher { + cipher: Aes256, + iv: [u8; 16], +} + +impl Rar50Cipher { + pub fn new(key: [u8; 32], iv: [u8; 16]) -> Self { + Self { + cipher: Aes256::new(&key.into()), + iv, + } + } + + pub fn decrypt_in_place(&mut self, data: &mut [u8]) -> Result<()> { + if !data.len().is_multiple_of(16) { + return Err(Error::UnalignedInput); + } + for block in data.as_chunks_mut::<16>().0.iter_mut() { + self.decrypt_block(block); + } + Ok(()) + } + + #[cfg(any(test, feature = "write", feature = "recovery"))] + pub fn encrypt_in_place(&mut self, data: &mut [u8]) -> Result<()> { + if !data.len().is_multiple_of(16) { + return Err(Error::UnalignedInput); + } + for block in data.as_chunks_mut::<16>().0.iter_mut() { + self.encrypt_block(block); + } + Ok(()) + } + + #[cfg(any(test, feature = "write", feature = "recovery"))] + fn encrypt_block(&mut self, block: &mut [u8]) { + for (byte, iv_byte) in block.iter_mut().zip(self.iv) { + *byte ^= iv_byte; + } + let block: &mut [u8; 16] = block.try_into().expect("AES block size"); + self.cipher.encrypt_block(block.into()); + self.iv.copy_from_slice(block); + } + + pub(crate) fn decrypt_block(&mut self, block: &mut [u8; 16]) { + let ciphertext = *block; + self.cipher.decrypt_block(block.into()); + for (byte, iv_byte) in block.iter_mut().zip(self.iv) { + *byte ^= iv_byte; + } + self.iv = ciphertext; + } +} + +fn hmac_sha256(key: &[u8], data: &[u8]) -> [u8; 32] { + let mut hmac = + ::new_from_slice(key).expect("HMAC accepts keys of any size"); + hmac.update(data); + hmac.finalize().into_bytes().into() +} + +fn sha256(data: &[u8]) -> [u8; 32] { + Sha256::digest(data).into() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sha256_matches_standard_vectors() { + assert_eq!( + hex(&sha256(b"")), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + hex(&sha256(b"abc")), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn hmac_sha256_matches_standard_vector() { + assert_eq!( + hex(&hmac_sha256(&[0x0b; 20], b"Hi There")), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7" + ); + } + + #[test] + fn password_check_uses_the_check_value_and_its_checksum() { + let keys = Rar50Keys::derive(b"secret", [7; 16], 4).unwrap(); + let mut record = keys.password_check_record(); + + assert_eq!(keys.check_password(&record), Ok(())); + + record[0] ^= 0x01; + assert_eq!(keys.check_password(&record), Err(Error::BadPassword)); + + let mut record = keys.password_check_record(); + record[11] ^= 0x01; + assert_eq!(keys.check_password(&record), Err(Error::BadPassword)); + } + + #[test] + fn rar50_aes_encrypt_decrypt_round_trips_blocks() { + let key = [9u8; 32]; + let iv = [5u8; 16]; + let mut data = *b"0123456789abcdefRAR5 block two!!"; + let plain = data; + + Rar50Cipher::new(key, iv) + .encrypt_in_place(&mut data) + .unwrap(); + assert_ne!(data, plain); + + Rar50Cipher::new(key, iv) + .decrypt_in_place(&mut data) + .unwrap(); + assert_eq!(data, plain); + } + + #[test] + fn rar50_aes_rejects_partial_tail() { + let key = [9u8; 32]; + let iv = [5u8; 16]; + let mut data = *b"partial block!!"; + + assert_eq!( + Rar50Cipher::new(key, iv).encrypt_in_place(&mut data), + Err(Error::UnalignedInput) + ); + assert_eq!( + Rar50Cipher::new(key, iv).decrypt_in_place(&mut data), + Err(Error::UnalignedInput) + ); + } + + #[test] + fn rar50_kdf_matches_pinned_vector() { + let keys = Rar50Keys::derive( + b"password", + [ + 0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13, 0x20, 0x21, 0x22, 0x23, 0x30, 0x31, + 0x32, 0x33, + ], + 4, + ) + .unwrap(); + + assert_eq!( + hex(&keys.key), + "cae43ebc57fcbdfc97ddc6f4a2d09687fd06010b51f651bec8f911f20caf008f" + ); + assert_eq!( + hex(&keys.hash_key), + "e65c566ff17139eaabdf60986e64058aac7e8dd82d6c5b027dd2e6d761a44d3c" + ); + assert_eq!(hex(&keys.password_check), "118929fdcad8a74f"); + assert_eq!( + hex(&keys.password_check_record()), + "118929fdcad8a74f5379ff2d" + ); + assert_eq!(keys.mac_crc32(0x1234_5678), 0xd742_398d); + } + + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } +} diff --git a/arca-rar/vendor/rars/src/crypto/unavailable.rs b/arca-rar/vendor/rars/src/crypto/unavailable.rs new file mode 100644 index 00000000..28123779 --- /dev/null +++ b/arca-rar/vendor/rars/src/crypto/unavailable.rs @@ -0,0 +1,64 @@ +//! Uninhabited reader storage preserves shared generic extraction types when +//! encrypted implementations are excluded. Constructors always report availability. +use crate::{Error, Result}; +use std::convert::Infallible; +use std::marker::PhantomData; +pub(crate) struct Reader { + unavailable: Infallible, + _parameters: PhantomData<(R, B)>, +} +impl Reader { + pub(crate) fn with_allowance( + _inner: R, + _version: u8, + _password: &[u8], + _salt: Option<[u8; 8]>, + _allowance: &B, + ) -> Result { + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } +} +impl Reader { + pub(crate) fn with_keys(_inner: R, _key: [u8; 32], _iv: [u8; 16]) -> Result { + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } +} +impl std::io::Read for Reader { + fn read(&mut self, _out: &mut [u8]) -> std::io::Result { + match self.unavailable {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn constructors_refuse_before_input_or_workspace_access() { + struct Unread; + impl std::io::Read for Unread { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + panic!("disabled crypto must not read input") + } + } + let allowance = crate::codec::workspace::Allowance::limited(0); + let result = Reader::with_allowance(Unread, 29, b"password", None, &allowance); + assert!(matches!( + result, + Err(Error::FeatureDisabled { + feature: "encryption" + }) + )); + assert_eq!(allowance.used(), 0); + let result = Reader::with_keys(Unread, [0; 32], [0; 16]); + assert!(matches!( + result, + Err(Error::FeatureDisabled { + feature: "encryption" + }) + )); + } +} diff --git a/arca-rar/vendor/rars/src/detect.rs b/arca-rar/vendor/rars/src/detect.rs new file mode 100644 index 00000000..23febc89 --- /dev/null +++ b/arca-rar/vendor/rars/src/detect.rs @@ -0,0 +1,192 @@ +use crate::version::ArchiveFamily; +use aho_corasick::AhoCorasick; +use std::sync::OnceLock; + +pub const RAR13_SIGNATURE: &[u8; 4] = b"RE~^"; +pub const RAR15_SIGNATURE: &[u8; 7] = b"Rar!\x1a\x07\x00"; +pub const RAR50_SIGNATURE: &[u8; 8] = b"Rar!\x1a\x07\x01\x00"; + +/// Default upper bound for scanning past an SFX stub when looking for the RAR +/// signature. Most installers in the wild place the archive within a few +/// hundred KiB, but large SFX modules (notably WinRAR's own installer plus a +/// bundled runtime) can push the offset past 1 MiB. +pub const SFX_SCAN_LIMIT: usize = 8 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct ArchiveSignature { + pub family: ArchiveFamily, + pub offset: usize, + pub length: usize, +} + +pub fn detect_archive_family(input: &[u8]) -> Option { + detect_at(input, 0) +} + +const SIGNATURES: &[&[u8]] = &[RAR50_SIGNATURE, RAR15_SIGNATURE, RAR13_SIGNATURE]; +const MAX_SIGNATURE_LEN: usize = { + let mut max_len = 0; + let mut i = 0; + while i < SIGNATURES.len() { + if SIGNATURES[i].len() > max_len { + max_len = SIGNATURES[i].len(); + } + i += 1; + } + max_len +}; + +fn signatures_searcher() -> &'static AhoCorasick { + static INSTANCE: OnceLock = OnceLock::new(); + INSTANCE.get_or_init(|| AhoCorasick::new(SIGNATURES).unwrap()) +} + +fn signatures_searcher_only_15_plus() -> &'static AhoCorasick { + static INSTANCE: OnceLock = OnceLock::new(); + INSTANCE.get_or_init(|| AhoCorasick::new(&SIGNATURES[..2]).unwrap()) +} + +pub fn find_archive_start(input: &[u8], max_scan: usize) -> Option { + // The limit is the maximum offset to find the signature, so we need to look + // up to the longest signature after that point. + let limit = input.len().min(max_scan.saturating_add(MAX_SIGNATURE_LEN)); + let input = &input[..limit]; + let mut first_rar13 = None; + let mut it = signatures_searcher().find_iter(input); + while let Some(m) = it.next() { + let family = match m.pattern().as_u32() { + 0 => ArchiveFamily::Rar50Plus, + 1 => ArchiveFamily::Rar15To40, + 2 => { + // Only find the first rar13 signature + debug_assert!(first_rar13.is_none()); + first_rar13 = Some(ArchiveSignature { + family: ArchiveFamily::Rar13, + offset: m.start(), + length: m.len(), + }); + // Switch to a searcher that doesn't have the rar13 signature anymore + it = signatures_searcher_only_15_plus() + .find_iter(aho_corasick::Input::new(input).range(m.end()..)); + continue; + } + _ => unreachable!(), + }; + if m.start() > max_scan { + break; + } + return Some(ArchiveSignature { + family, + offset: m.start(), + length: m.len(), + }); + } + first_rar13.filter(|sig| sig.offset <= max_scan) +} + +fn detect_at(input: &[u8], offset: usize) -> Option { + let tail = input.get(offset..)?; + + if tail.starts_with(RAR50_SIGNATURE) { + Some(ArchiveSignature { + family: ArchiveFamily::Rar50Plus, + offset, + length: RAR50_SIGNATURE.len(), + }) + } else if tail.starts_with(RAR15_SIGNATURE) { + Some(ArchiveSignature { + family: ArchiveFamily::Rar15To40, + offset, + length: RAR15_SIGNATURE.len(), + }) + } else if tail.starts_with(RAR13_SIGNATURE) { + Some(ArchiveSignature { + family: ArchiveFamily::Rar13, + offset, + length: RAR13_SIGNATURE.len(), + }) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_all_known_signatures() { + assert_eq!( + detect_archive_family(b"RE~^").unwrap().family, + ArchiveFamily::Rar13 + ); + assert_eq!( + detect_archive_family(b"Rar!\x1a\x07\x00").unwrap().family, + ArchiveFamily::Rar15To40 + ); + assert_eq!( + detect_archive_family(b"Rar!\x1a\x07\x01\x00") + .unwrap() + .family, + ArchiveFamily::Rar50Plus + ); + } + + #[test] + fn finds_sfx_prefixed_archive() { + let sig = find_archive_start(b"stub bytes RE~^payload", 128).unwrap(); + assert_eq!(sig.family, ArchiveFamily::Rar13); + assert_eq!(sig.offset, 11); + } + + #[test] + fn sfx_scan_prefers_stronger_rar15_signature_over_earlier_rar13_bytes() { + let sig = find_archive_start(b"stub RE~^ bytes Rar!\x1a\x07\x00payload", 128).unwrap(); + assert_eq!(sig.family, ArchiveFamily::Rar15To40); + assert_eq!(sig.offset, 16); + } + + #[test] + fn rejects_unknown_and_truncated_signatures() { + assert_eq!(detect_archive_family(b""), None); + assert_eq!(detect_archive_family(b"RAR!"), None); + assert_eq!(detect_archive_family(b"Rar!\x1a\x07"), None); + assert_eq!(find_archive_start(b"not an archive", 128), None); + } + + #[test] + fn scan_limit_bounds_sfx_detection() { + let input = b"stub bytes RE~^payload"; + + assert_eq!(find_archive_start(input, 10), None); + + let sig = find_archive_start(input, 11).unwrap(); + assert_eq!(sig.family, ArchiveFamily::Rar13); + assert_eq!(sig.offset, 11); + assert_eq!(sig.length, RAR13_SIGNATURE.len()); + } + + #[test] + fn shorter_signatures_in_scan_lookahead_do_not_escape_the_offset_limit() { + for signature in [RAR15_SIGNATURE.as_slice(), RAR13_SIGNATURE.as_slice()] { + let mut input = vec![0]; + input.extend_from_slice(signature); + assert_eq!(find_archive_start(&input, 0), None); + let found = find_archive_start(&input, 1).unwrap(); + assert_eq!(found.offset, 1); + assert_eq!(found.length, signature.len()); + } + } + + #[test] + fn sfx_scan_limit_finds_signature_past_128kib_stub() { + // Real SFX installers routinely place the RAR payload past 128 KiB + // (modern WinRAR-built SFXes, Nero, anti-virus installers, etc.). + let mut stub = vec![0u8; 300 * 1024]; + stub.extend_from_slice(RAR15_SIGNATURE); + let sig = find_archive_start(&stub, SFX_SCAN_LIMIT).unwrap(); + assert_eq!(sig.family, ArchiveFamily::Rar15To40); + assert_eq!(sig.offset, 300 * 1024); + } +} diff --git a/arca-rar/vendor/rars/src/entropy.rs b/arca-rar/vendor/rars/src/entropy.rs new file mode 100644 index 00000000..fbbd315a --- /dev/null +++ b/arca-rar/vendor/rars/src/entropy.rs @@ -0,0 +1,65 @@ +//! Fresh salts and IVs for writing archives or framing repaired headers. + +use crate::{Error, Result}; + +fn entropy_error(error: getrandom::Error, context: &'static str) -> Error { + std::io::Error::other(format!("{context}: {error}")).into() +} + +/// Fill salt or IV bytes using the operating system's entropy source. +pub(crate) fn fill_entropy(bytes: &mut [u8], context: &'static str) -> Result<()> { + fill_entropy_with(bytes, context, getrandom::fill) +} + +fn fill_entropy_with( + bytes: &mut [u8], + context: &'static str, + fill: fn(&mut [u8]) -> std::result::Result<(), getrandom::Error>, +) -> Result<()> { + fill(bytes).map_err(|error| entropy_error(error, context)) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn entropy_failure_is_io_and_keeps_the_backend_diagnostic() { + let error = super::entropy_error(getrandom::Error::UNSUPPORTED, "encryption salt"); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert!(error.to_string().contains("encryption salt")); + assert!(error + .to_string() + .contains(&getrandom::Error::UNSUPPORTED.to_string())); + } + + #[test] + fn entropy_fill_delegates_the_whole_buffer_and_refuses_backend_failure() { + fn success(bytes: &mut [u8]) -> std::result::Result<(), getrandom::Error> { + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = index as u8; + } + Ok(()) + } + fn failure(bytes: &mut [u8]) -> std::result::Result<(), getrandom::Error> { + bytes[0] = 42; + Err(getrandom::Error::UNSUPPORTED) + } + for context in [ + "RAR 3.x writer could not generate encryption salt", + "RAR 5 writer could not generate encryption salt", + "RAR 5 writer could not generate encryption IV", + ] { + let mut bytes = [0xff; 16]; + fill_entropy_with(&mut bytes, context, success).unwrap(); + assert_eq!(bytes, std::array::from_fn(|index| index as u8)); + let error = fill_entropy_with(&mut bytes, context, failure).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.to_string(), + format!("I/O error: {context}: {}", getrandom::Error::UNSUPPORTED) + ); + assert_eq!(bytes[0], 42); + assert_eq!(&bytes[1..], &(1u8..16).collect::>()); + } + } +} diff --git a/arca-rar/vendor/rars/src/error.rs b/arca-rar/vendor/rars/src/error.rs new file mode 100644 index 00000000..7af9f3c9 --- /dev/null +++ b/arca-rar/vendor/rars/src/error.rs @@ -0,0 +1,1098 @@ +use crate::version::{ArchiveFamily, ArchiveVersion}; +use std::sync::Arc; + +pub type Result = std::result::Result; + +/// Stable error categories for callers and language bindings. Context wrappers +/// do not change the category; display text remains a diagnostic, not a protocol. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ErrorKind { + InvalidArchive, + InvalidArgument, + UnsupportedFormat, + UnsupportedFeature, + PasswordRequired, + BadPassword, + ChecksumMismatch, + EntryNotFound, + DuplicateEntry, + UnsafePath, + Io, + ResourceLimit, + Cancelled, + SourceChanged, + /// Archive construction failed because the writer reached an invalid state. + WriterFailure, +} + +impl ErrorKind { + /// Stable machine-readable code, independent of diagnostic wording. + pub fn code(self) -> &'static str { + match self { + Self::InvalidArchive => "INVALID_ARCHIVE", + Self::InvalidArgument => "INVALID_OPTION", + Self::UnsupportedFormat => "UNSUPPORTED_FORMAT", + Self::UnsupportedFeature => "UNSUPPORTED_FEATURE", + Self::PasswordRequired => "PASSWORD_REQUIRED", + Self::BadPassword => "BAD_PASSWORD", + Self::ChecksumMismatch => "CHECKSUM_MISMATCH", + Self::EntryNotFound => "ENTRY_NOT_FOUND", + Self::DuplicateEntry => "DUPLICATE_ENTRY", + Self::UnsafePath => "UNSAFE_ENTRY_NAME", + Self::Io => "IO", + Self::ResourceLimit => "RESOURCE_LIMIT", + Self::Cancelled => "CANCELLED", + Self::SourceChanged => "SOURCE_CHANGED", + Self::WriterFailure => "WRITE_FAILED", + } + } +} + +#[derive(Debug, Clone)] +pub struct IoError { + pub kind: std::io::ErrorKind, + pub message: String, + source: Arc, +} + +impl IoError { + pub fn source(&self) -> &(dyn std::error::Error + 'static) { + self.source.as_ref() + } +} + +impl PartialEq for IoError { + fn eq(&self, other: &Self) -> bool { + self.kind == other.kind && self.message == other.message + } +} + +impl Eq for IoError {} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + TooShort, + UnsupportedSignature, + InvalidHeader(&'static str), + InvalidArgument(&'static str), + EntryNotFound, + DuplicateEntry, + InputSymlink, + UnsafePath(&'static str), + SourceChanged(&'static str), + WriterFailure(&'static str), + AtArchiveOffset { + offset: usize, + source: Box, + }, + AtEntry { + name: Vec, + operation: &'static str, + source: Box, + }, + /// A fault traced to one volume of a set, numbered from one. + InVolume { + number: usize, + source: Box, + }, + UnsupportedVersion(ArchiveVersion), + /// The requested implementation was omitted at compile time. + FeatureDisabled { + feature: &'static str, + }, + /// Skipping payload decoding would leave solid history unavailable. + CannotSkipSolidMember, + UnsupportedFeature { + version: ArchiveVersion, + feature: &'static str, + }, + /// A writer option that would change the output, for a format that cannot + /// honour it. + UnsupportedWriterOption { + target: ArchiveVersion, + option: crate::WriterOption, + /// Why, when the format alone is not the reason: "in a compressed + /// archive", "levels run from 0 to 5". Rendered after the format. + because: Option<&'static str>, + }, + /// `required` is the declared size or the minimum attempted output size. + MemberOutputLimitExceeded { + limit: u64, + required: u64, + }, + /// Cumulative top-level headers required; diagnostic sums saturate at u64::MAX. + HeaderCountLimitExceeded { + limit: u64, + required: u64, + }, + /// Cumulative plaintext header bytes required, saturated at u64::MAX. + HeaderBytesLimitExceeded { + limit: u64, + required: u64, + }, + /// Total logical output refusal. `used` is accepted output before refusal; + /// `required` is used plus the declared size or offered chunk, saturated at + /// u64::MAX (a lower bound if the attempted sum cannot be represented). + TotalOutputLimitExceeded { + limit: u64, + required: u64, + used: u64, + }, + Rar50DictionaryLimitExceeded { + limit: u64, + required: u64, + }, + Rar50BufferedDecodeLimitExceeded { + limit: u64, + required: u64, + }, + Rar50ScratchLimitExceeded { + limit: u64, + required: u64, + }, + /// Retained rewrite payload bytes, excluding decoder and writer workspace. + RewriteStagingLimitExceeded { + limit: u64, + required: u64, + }, + Rar50FilterMemoryLimitExceeded { + limit: u64, + required: u64, + }, + /// Aggregate logical spool storage, including in-flight growth reservations. + /// Diagnostic `required` saturates at u64::MAX on accounting overflow. + WriterSpoolLimitExceeded { + limit: u64, + required: u64, + used: u64, + }, + /// Shared capacity of in-memory spool payloads and indexes, including reservations. + WriterSpoolMemoryLimitExceeded { + limit: u64, + required: u64, + used: u64, + }, + /// Retained RAR5/7 prepared header image capacity, including reservations. + WriterPreparedHeaderLimitExceeded { + limit: u64, + required: u64, + used: u64, + }, + WriterPreparationLimitExceeded { + limit: u64, + required: u64, + used: u64, + }, + MemoryLimitExceeded { + limit: u64, + required: u64, + dictionary_size: u64, + }, + UnsupportedFamilyFeature { + family: ArchiveFamily, + feature: &'static str, + }, + UnsupportedCompression { + family: &'static str, + unpack_version: u8, + method: u8, + }, + UnsupportedEncryption { + family: &'static str, + unpack_version: u8, + }, + Io(IoError), + NeedPassword, + WrongPasswordOrCorruptData, + /// Archive writing was cancelled by a token or progress callback. + Cancelled, + CrcMismatch { + expected: u16, + actual: u16, + }, + Crc32Mismatch { + expected: u32, + actual: u32, + }, + HashMismatch { + hash_type: u64, + }, + Codec(crate::codec::Error), + Rar3Recovery(crate::recovery::rar3::Error), + Rar5Recovery(crate::recovery::rar5::Error), + Rar20Crypto(crate::crypto::rar20::Error), + Rar30Crypto(crate::crypto::rar30::Error), + Rar50Crypto(crate::crypto::rar50::Error), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooShort => write!(f, "input is too short"), + Self::UnsupportedSignature => write!(f, "unsupported archive signature"), + Self::InvalidHeader(msg) => write!(f, "invalid header: {msg}"), + Self::InvalidArgument(msg) => write!(f, "invalid argument: {msg}"), + Self::EntryNotFound => f.write_str("no such archive entry"), + Self::DuplicateEntry => f.write_str("duplicate archive entry name"), + Self::InputSymlink => f.write_str("input is a symlink; refusing to follow it"), + Self::UnsafePath(msg) | Self::SourceChanged(msg) | Self::WriterFailure(msg) => f.write_str(msg), + Self::AtArchiveOffset { offset, source } => { + write!(f, "at archive offset {offset:#x}: {source}") + } + Self::InVolume { number, source } => write!(f, "in volume {number}: {source}"), + Self::AtEntry { + name, + operation, + source, + } => { + write!( + f, + "while {operation} entry '{}': {source}", + String::from_utf8_lossy(name) + ) + } + Self::UnsupportedVersion(version) => write!(f, "unsupported version: {version}"), + Self::FeatureDisabled { feature } => write!(f, "Cargo feature {feature} is disabled"), + Self::CannotSkipSolidMember => write!(f, "cannot skip file data in a solid archive"), + Self::UnsupportedFeature { version, feature } => { + write!(f, "feature {feature} is not supported by {version}") + } + Self::UnsupportedWriterOption { + target, + option, + because, + } => match because { + Some(reason) => { + write!(f, "{} is not supported by {target} ({reason})", option.name()) + } + None => write!(f, "{} is not supported by {target}", option.name()), + }, + Self::MemberOutputLimitExceeded { limit, required } => write!(f, + "member requires {required} output bytes, above the configured limit of {limit} bytes"), + Self::HeaderCountLimitExceeded { limit, required } => write!(f, + "parsing requires at least {required} headers, above the configured limit of {limit} headers"), + Self::HeaderBytesLimitExceeded { limit, required } => write!(f, + "parsing requires at least {required} header bytes, above the configured limit of {limit} bytes"), + Self::TotalOutputLimitExceeded { limit, required, used } => write!(f, + "extraction requires at least {required} total output bytes, above the configured limit of {limit} bytes ({used} bytes already accepted)"), + Self::Rar50DictionaryLimitExceeded { limit, required } => write!( + f, + "RAR 5 member declares a {required}-byte dictionary, above the configured limit of {limit} bytes" + ), + Self::Rar50BufferedDecodeLimitExceeded { limit, required } => write!( + f, + "RAR 5 filtered member requires buffered decoding of {required} bytes, above the configured limit of {limit} bytes" + ), + Self::Rar50ScratchLimitExceeded { limit, required } => write!(f, "RAR 5 scratch limit {limit} bytes exceeded (requires {required})"), + Self::RewriteStagingLimitExceeded { limit, required } => write!(f, "rewrite staging limit {limit} bytes exceeded (requires {required})"), + Self::Rar50FilterMemoryLimitExceeded { limit, required } => write!(f, "RAR 5 filter workspace limit {limit} bytes exceeded (requires {required})"), + Self::WriterSpoolLimitExceeded { limit, required, used } => write!(f, + "writer spool limit {limit} bytes exceeded (requires {required}; {used} bytes already retained or reserved)"), + Self::WriterPreparationLimitExceeded { limit, required, used } => write!(f, + "writer preparation limit {limit} bytes exceeded (requires {required}; {used} bytes already retained or reserved)"), + Self::WriterPreparedHeaderLimitExceeded { limit, required, used } => write!(f, + "writer prepared header limit {limit} bytes exceeded (requires {required}; {used} bytes already retained or reserved)"), + Self::WriterSpoolMemoryLimitExceeded { limit, required, used } => write!(f, + "writer spool memory limit {limit} bytes exceeded (requires {required}; {used} bytes already retained or reserved)"), + Self::MemoryLimitExceeded { + limit, + required, + dictionary_size, + } => write!( + f, + "compression needs at least {required} bytes of working memory for a {dictionary_size}-byte dictionary, above the configured limit of {limit} bytes" + ), + Self::UnsupportedFamilyFeature { family, feature } => { + write!(f, "feature {feature} is not supported by {family:?}") + } + Self::UnsupportedCompression { + family, + unpack_version, + method, + } => write!( + f, + "{family} compression is not supported: unpack version {unpack_version}, method {method:#04x}" + ), + Self::UnsupportedEncryption { + family, + unpack_version, + } => write!( + f, + "{family} encryption is not supported: unpack version {unpack_version}" + ), + Self::Io(error) => write!(f, "I/O error: {}", error.message), + Self::NeedPassword => write!(f, "a password is required"), + Self::WrongPasswordOrCorruptData => { + write!(f, "wrong password or corrupt encrypted data") + } + Self::Cancelled => f.write_str("archive writing was cancelled"), + Self::CrcMismatch { expected, actual } => { + write!( + f, + "checksum mismatch: expected {expected:#06x}, got {actual:#06x}" + ) + } + Self::Crc32Mismatch { expected, actual } => { + write!( + f, + "checksum mismatch: expected {expected:#010x}, got {actual:#010x}" + ) + } + Self::HashMismatch { hash_type } => { + write!(f, "hash mismatch for hash type {hash_type}") + } + Self::Codec(error) => write!(f, "{error}"), + Self::Rar3Recovery(error) => write!(f, "{error}"), + Self::Rar5Recovery(error) => write!(f, "{error}"), + Self::Rar20Crypto(error) => write!(f, "{error}"), + Self::Rar30Crypto(error) => write!(f, "{error}"), + Self::Rar50Crypto(error) => write!(f, "{error}"), + } + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + // A reader buried inside a decoder has only `io::Error` to report + // through, so it wraps the real error. Unwrap it again here, otherwise + // a checksum mismatch surfaces as an I/O failure. + let error = match error.downcast::() { + Ok(inner) => return inner, + Err(error) => error, + }; + Self::Io(IoError { + kind: error.kind(), + message: error.to_string(), + source: Arc::new(error), + }) + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::AtArchiveOffset { source, .. } + | Self::AtEntry { source, .. } + | Self::InVolume { source, .. } => Some(source), + Self::Codec(source) => Some(source), + Self::Rar3Recovery(source) => Some(source), + Self::Rar5Recovery(source) => Some(source), + Self::Rar20Crypto(source) => Some(source), + Self::Rar30Crypto(source) => Some(source), + Self::Rar50Crypto(source) => Some(source), + Self::Io(source) => Some(source.source()), + _ => None, + } + } +} + +impl Error { + /// The underlying library error, with archive/entry/volume context removed. + /// The original error still owns and displays every context wrapper. + pub fn root_cause(&self) -> &Self { + let mut error = self; + while let Self::AtArchiveOffset { source, .. } + | Self::AtEntry { source, .. } + | Self::InVolume { source, .. } = error + { + error = source; + } + error + } + + /// The outermost member context, if this error identifies a member. + pub fn entry_context(&self) -> Option<(&[u8], &'static str)> { + let mut error = self; + loop { + match error { + Self::AtEntry { + name, operation, .. + } => return Some((name, operation)), + Self::AtArchiveOffset { source, .. } | Self::InVolume { source, .. } => { + error = source + } + _ => return None, + } + } + } + + /// Classify the cause without discarding its diagnostic context. + pub fn kind(&self) -> ErrorKind { + match self.root_cause() { + Self::AtArchiveOffset { .. } | Self::AtEntry { .. } | Self::InVolume { .. } => { + unreachable!("root cause has no context wrapper") + } + Self::UnsupportedSignature | Self::UnsupportedVersion(_) => { + ErrorKind::UnsupportedFormat + } + Self::FeatureDisabled { .. } + | Self::Rar50Crypto(crate::crypto::rar50::Error::FeatureDisabled) + | Self::CannotSkipSolidMember + | Self::UnsupportedFeature { .. } + | Self::UnsupportedFamilyFeature { .. } + | Self::UnsupportedCompression { .. } + | Self::UnsupportedEncryption { .. } + | Self::UnsupportedWriterOption { .. } => ErrorKind::UnsupportedFeature, + Self::NeedPassword => ErrorKind::PasswordRequired, + Self::WrongPasswordOrCorruptData + | Self::Rar50Crypto(crate::crypto::rar50::Error::BadPassword) => ErrorKind::BadPassword, + Self::CrcMismatch { .. } | Self::Crc32Mismatch { .. } | Self::HashMismatch { .. } => { + ErrorKind::ChecksumMismatch + } + Self::Rar5Recovery(crate::recovery::rar5::Error::Cancelled) => ErrorKind::Cancelled, + Self::Io(_) | Self::Rar5Recovery(crate::recovery::rar5::Error::Io(_)) => ErrorKind::Io, + Self::Rar50ScratchLimitExceeded { .. } + | Self::RewriteStagingLimitExceeded { .. } + | Self::Rar50FilterMemoryLimitExceeded { .. } + | Self::MemoryLimitExceeded { .. } + | Self::WriterSpoolLimitExceeded { .. } + | Self::WriterSpoolMemoryLimitExceeded { .. } + | Self::WriterPreparedHeaderLimitExceeded { .. } + | Self::WriterPreparationLimitExceeded { .. } + | Self::MemberOutputLimitExceeded { .. } + | Self::HeaderCountLimitExceeded { .. } + | Self::HeaderBytesLimitExceeded { .. } + | Self::TotalOutputLimitExceeded { .. } + | Self::Rar50DictionaryLimitExceeded { .. } + | Self::Rar50BufferedDecodeLimitExceeded { .. } + | Self::Rar5Recovery(crate::recovery::rar5::Error::RebuildTooLarge) + | Self::Rar5Recovery(crate::recovery::rar5::Error::RepairWorkspaceLimitExceeded { + .. + }) => ErrorKind::ResourceLimit, + Self::Codec(crate::codec::Error::Io(error)) => error.kind(), + Self::Codec(crate::codec::Error::WorkspaceLimitExceeded(_)) => ErrorKind::ResourceLimit, + Self::Cancelled | Self::Codec(crate::codec::Error::Cancelled) => ErrorKind::Cancelled, + Self::EntryNotFound => ErrorKind::EntryNotFound, + Self::DuplicateEntry => ErrorKind::DuplicateEntry, + Self::InputSymlink | Self::InvalidArgument(_) => ErrorKind::InvalidArgument, + Self::UnsafePath(_) => ErrorKind::UnsafePath, + Self::SourceChanged(_) => ErrorKind::SourceChanged, + Self::WriterFailure(_) => ErrorKind::WriterFailure, + Self::TooShort + | Self::InvalidHeader(_) + | Self::Codec(_) + | Self::Rar3Recovery(_) + | Self::Rar5Recovery(_) + | Self::Rar20Crypto(_) + | Self::Rar30Crypto(_) + | Self::Rar50Crypto(_) => ErrorKind::InvalidArchive, + } + } + + pub fn at_archive_offset(self, offset: usize) -> Self { + Self::AtArchiveOffset { + offset, + source: Box::new(self), + } + } + + pub fn at_entry(self, name: Vec, operation: &'static str) -> Self { + Self::AtEntry { + name, + operation, + source: Box::new(self), + } + } +} + +impl From for Error { + fn from(error: crate::codec::Error) -> Self { + match error { + crate::codec::Error::Cancelled => Self::Cancelled, + crate::codec::Error::Io(error) => *error, + error => Self::Codec(error), + } + } +} + +impl From for Error { + fn from(error: crate::recovery::rar5::Error) -> Self { + match error { + crate::recovery::rar5::Error::Cancelled => Self::Cancelled, + crate::recovery::rar5::Error::TypedIo(error) => *error, + error => Self::Rar5Recovery(error), + } + } +} + +impl From for Error { + fn from(error: crate::recovery::rar3::Error) -> Self { + Self::Rar3Recovery(error) + } +} + +impl From for Error { + fn from(error: crate::crypto::rar20::Error) -> Self { + Self::Rar20Crypto(error) + } +} + +impl From for Error { + fn from(error: crate::crypto::rar30::Error) -> Self { + Self::Rar30Crypto(error) + } +} + +impl From for Error { + fn from(error: crate::crypto::rar50::Error) -> Self { + if error == crate::crypto::rar50::Error::FeatureDisabled { + return Self::FeatureDisabled { + feature: "encryption", + }; + } + Self::Rar50Crypto(error) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn machine_readable_codes_remain_stable_and_distinct() { + let cases = [ + (ErrorKind::InvalidArchive, "INVALID_ARCHIVE"), + (ErrorKind::InvalidArgument, "INVALID_OPTION"), + (ErrorKind::UnsupportedFormat, "UNSUPPORTED_FORMAT"), + (ErrorKind::UnsupportedFeature, "UNSUPPORTED_FEATURE"), + (ErrorKind::PasswordRequired, "PASSWORD_REQUIRED"), + (ErrorKind::BadPassword, "BAD_PASSWORD"), + (ErrorKind::ChecksumMismatch, "CHECKSUM_MISMATCH"), + (ErrorKind::EntryNotFound, "ENTRY_NOT_FOUND"), + (ErrorKind::DuplicateEntry, "DUPLICATE_ENTRY"), + (ErrorKind::UnsafePath, "UNSAFE_ENTRY_NAME"), + (ErrorKind::Io, "IO"), + (ErrorKind::ResourceLimit, "RESOURCE_LIMIT"), + (ErrorKind::Cancelled, "CANCELLED"), + (ErrorKind::SourceChanged, "SOURCE_CHANGED"), + (ErrorKind::WriterFailure, "WRITE_FAILED"), + ]; + let mut codes = std::collections::HashSet::new(); + for (kind, expected) in cases { + assert_eq!(kind.code(), expected); + assert!(codes.insert(kind.code()), "duplicate error code {expected}"); + } + } + + #[test] + fn resource_diagnostics_retain_required_used_and_limit_values() { + let cases = [ + ( + Error::Rar50ScratchLimitExceeded { limit: 11, required: 23 }, + "RAR 5 scratch limit 11 bytes exceeded (requires 23)", + ), + ( + Error::Rar50FilterMemoryLimitExceeded { limit: 11, required: 23 }, + "RAR 5 filter workspace limit 11 bytes exceeded (requires 23)", + ), + ( + Error::WriterSpoolLimitExceeded { limit: 11, required: 23, used: 7 }, + "writer spool limit 11 bytes exceeded (requires 23; 7 bytes already retained or reserved)", + ), + ( + Error::WriterPreparationLimitExceeded { limit: 11, required: 23, used: 7 }, + "writer preparation limit 11 bytes exceeded (requires 23; 7 bytes already retained or reserved)", + ), + ( + Error::WriterPreparedHeaderLimitExceeded { limit: 11, required: 23, used: 7 }, + "writer prepared header limit 11 bytes exceeded (requires 23; 7 bytes already retained or reserved)", + ), + ( + Error::WriterSpoolMemoryLimitExceeded { limit: 11, required: 23, used: 7 }, + "writer spool memory limit 11 bytes exceeded (requires 23; 7 bytes already retained or reserved)", + ), + ]; + for (error, expected) in cases { + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!(error.to_string(), expected); + assert!(std::error::Error::source(&error).is_none()); + } + assert_eq!( + Error::CannotSkipSolidMember.to_string(), + "cannot skip file data in a solid archive" + ); + assert_eq!( + Error::CannotSkipSolidMember.kind(), + ErrorKind::UnsupportedFeature + ); + assert!(Error::CannotSkipSolidMember.entry_context().is_none()); + } + + #[test] + fn codec_io_transport_retains_library_error_types() { + for cause in [ + Error::Cancelled, + Error::NeedPassword, + Error::MemberOutputLimitExceeded { + limit: 1, + required: 2, + }, + ] { + let transported = crate::codec::Error::from(std::io::Error::other(cause.clone())); + assert_eq!(Error::from(transported.clone()), cause); + assert_eq!(Error::Codec(transported).kind(), cause.kind()); + } + } + + #[test] + fn codec_workspace_refusal_keeps_its_resource_classification_and_details() { + let cause = crate::codec::Error::WorkspaceLimitExceeded(Box::new( + crate::codec::WorkspaceLimitError { + limit: 12, + required: 16, + used: 8, + }, + )); + let error = Error::from(cause.clone()).at_archive_offset(42); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert!(matches!(error.root_cause(), Error::Codec(actual) if actual == &cause)); + assert!(error.to_string().contains("codec workspace limit 12")); + } + + use super::*; + + #[test] + fn classification_survives_every_context_order_without_losing_details() { + let cases = [ + (Error::NeedPassword, ErrorKind::PasswordRequired), + ( + Error::Rar50Crypto(crate::crypto::rar50::Error::BadPassword), + ErrorKind::BadPassword, + ), + ( + Error::UnsupportedCompression { + family: "RAR", + unpack_version: 99, + method: 9, + }, + ErrorKind::UnsupportedFeature, + ), + ( + Error::UnsupportedVersion(ArchiveVersion::Rar50), + ErrorKind::UnsupportedFormat, + ), + ( + Error::from(std::io::Error::from(std::io::ErrorKind::PermissionDenied)), + ErrorKind::Io, + ), + ( + Error::Rar5Recovery(crate::recovery::rar5::Error::Io( + std::io::ErrorKind::WriteZero, + )), + ErrorKind::Io, + ), + ( + Error::MemoryLimitExceeded { + limit: 1, + required: 2, + dictionary_size: 1, + }, + ErrorKind::ResourceLimit, + ), + ( + Error::TotalOutputLimitExceeded { + limit: 100, + required: 110, + used: 60, + }, + ErrorKind::ResourceLimit, + ), + ( + Error::Codec(crate::codec::Error::Cancelled), + ErrorKind::Cancelled, + ), + ( + Error::HeaderCountLimitExceeded { + limit: 1, + required: 2, + }, + ErrorKind::ResourceLimit, + ), + ( + Error::HeaderBytesLimitExceeded { + limit: 1, + required: 2, + }, + ErrorKind::ResourceLimit, + ), + (Error::Cancelled, ErrorKind::Cancelled), + ( + Error::WriterFailure("layout failure"), + ErrorKind::WriterFailure, + ), + (Error::EntryNotFound, ErrorKind::EntryNotFound), + (Error::DuplicateEntry, ErrorKind::DuplicateEntry), + (Error::InputSymlink, ErrorKind::InvalidArgument), + (Error::UnsafePath("path refusal"), ErrorKind::UnsafePath), + ( + Error::SourceChanged("changed source"), + ErrorKind::SourceChanged, + ), + ( + Error::InvalidHeader("password is required; unsafe; I/O error"), + ErrorKind::InvalidArchive, + ), + ( + Error::HashMismatch { hash_type: 0 }, + ErrorKind::ChecksumMismatch, + ), + ]; + for (cause, expected) in cases { + assert_eq!(cause.kind(), expected); + for order in [ + [0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0], + ] { + let mut wrapped = cause.clone(); + for wrapper in order { + wrapped = match wrapper { + 0 => wrapped.at_archive_offset(42), + 1 => wrapped.at_entry(b"member".to_vec(), "reading"), + _ => Error::InVolume { + number: 3, + source: Box::new(wrapped), + }, + }; + } + assert_eq!(wrapped.kind(), expected); + assert_eq!(wrapped.root_cause(), &cause); + assert_eq!( + wrapped.entry_context(), + Some((b"member".as_slice(), "reading")) + ); + assert!(wrapped.to_string().contains("in volume 3")); + assert!(wrapped.to_string().contains("member")); + } + } + } + + #[test] + fn archive_offset_context_exposes_source_error() { + let error = Error::InvalidHeader("bad block").at_archive_offset(0x1234); + + assert_eq!( + error.to_string(), + "at archive offset 0x1234: invalid header: bad block" + ); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("invalid header: bad block".to_string()) + ); + } + + #[test] + fn entry_context_exposes_source_error() { + let error = Error::Crc32Mismatch { + expected: 1, + actual: 2, + } + .at_entry(b"hello.txt".to_vec(), "verifying"); + + assert_eq!( + error.to_string(), + "while verifying entry 'hello.txt': checksum mismatch: expected 0x00000001, got 0x00000002" + ); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("checksum mismatch: expected 0x00000001, got 0x00000002".to_string()) + ); + } + + #[test] + fn io_error_preserves_error_kind() { + let error = Error::from(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "locked", + )); + + assert!(matches!( + error, + Error::Io(ref source) if source.kind == std::io::ErrorKind::PermissionDenied + )); + assert_eq!(error.to_string(), "I/O error: locked"); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("locked".to_string()) + ); + } + + #[test] + fn io_error_partial_eq_compares_kind_and_message_ignoring_source_identity() { + let permission = Error::from(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "locked", + )); + let permission_again = Error::from(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "locked", + )); + let different_kind = + Error::from(std::io::Error::new(std::io::ErrorKind::NotFound, "locked")); + let different_message = Error::from(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "elsewhere", + )); + + // Same kind + same message → equal even though Arc differs. + assert_eq!(permission, permission_again); + // Differing kind or message → not equal. + assert_ne!(permission, different_kind); + assert_ne!(permission, different_message); + } + + #[test] + fn unsupported_codec_errors_are_not_reported_as_invalid_headers() { + assert_eq!( + Error::UnsupportedCompression { + family: "RAR 1.5-4.x", + unpack_version: 14, + method: 0x33, + } + .to_string(), + "RAR 1.5-4.x compression is not supported: unpack version 14, method 0x33" + ); + assert_eq!( + Error::UnsupportedEncryption { + family: "RAR 1.5-4.x", + unpack_version: 14, + } + .to_string(), + "RAR 1.5-4.x encryption is not supported: unpack version 14" + ); + } + + #[test] + fn codec_errors_remain_inspectable_without_changing_display_text() { + let error = Error::from(crate::codec::Error::NeedMoreInput); + + assert!(matches!( + error, + Error::Codec(crate::codec::Error::NeedMoreInput) + )); + assert_eq!(error.to_string(), "codec input is truncated"); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("codec input is truncated".to_string()) + ); + } + + #[test] + fn recovery_errors_remain_inspectable_without_changing_display_text() { + let error = Error::from(crate::recovery::rar5::Error::TooManyDamagedShards); + + assert!(matches!( + error, + Error::Rar5Recovery(crate::recovery::rar5::Error::TooManyDamagedShards) + )); + assert_eq!( + error.to_string(), + "RAR 5 recovery data cannot repair this many damaged shards" + ); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("RAR 5 recovery data cannot repair this many damaged shards".to_string()) + ); + } + + #[test] + fn rar3_recovery_errors_remain_in_source_chain() { + let error = Error::from(crate::recovery::rar3::Error::DecodeFailed); + + assert_eq!(error.to_string(), "RAR 3 recovery decode failed"); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("RAR 3 recovery decode failed".to_string()) + ); + } + + #[test] + fn rar30_crypto_errors_remain_in_source_chain() { + let error = Error::from(crate::crypto::rar30::Error::UnalignedInput); + + assert!(matches!( + error, + Error::Rar30Crypto(crate::crypto::rar30::Error::UnalignedInput) + )); + assert_eq!(error.to_string(), "RAR 3.x AES input is not block aligned"); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("RAR 3.x AES input is not block aligned".to_string()) + ); + } + + #[test] + fn rar20_crypto_errors_remain_in_source_chain() { + let error = Error::from(crate::crypto::rar20::Error::UnalignedInput); + + assert!(matches!( + error, + Error::Rar20Crypto(crate::crypto::rar20::Error::UnalignedInput) + )); + assert_eq!( + error.to_string(), + "RAR 2.0 cipher input is not block aligned" + ); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("RAR 2.0 cipher input is not block aligned".to_string()) + ); + } + + #[test] + fn rar50_crypto_errors_remain_in_source_chain() { + let error = Error::from(crate::crypto::rar50::Error::UnalignedInput); + + assert_eq!(error.to_string(), "RAR 5 AES input is not block aligned"); + assert_eq!( + std::error::Error::source(&error).map(ToString::to_string), + Some("RAR 5 AES input is not block aligned".to_string()) + ); + } + + #[test] + fn simple_display_variants_render_their_messages() { + assert_eq!(Error::TooShort.to_string(), "input is too short"); + assert_eq!( + Error::UnsupportedSignature.to_string(), + "unsupported archive signature" + ); + assert_eq!(Error::NeedPassword.to_string(), "a password is required"); + assert_eq!( + Error::WrongPasswordOrCorruptData.to_string(), + "wrong password or corrupt encrypted data" + ); + assert_eq!( + Error::HashMismatch { hash_type: 7 }.to_string(), + "hash mismatch for hash type 7" + ); + assert_eq!( + Error::CrcMismatch { + expected: 0xabcd, + actual: 0x1234 + } + .to_string(), + "checksum mismatch: expected 0xabcd, got 0x1234" + ); + assert_eq!( + Error::UnsupportedVersion(ArchiveVersion::Rar50).to_string(), + "unsupported version: rar50" + ); + assert_eq!( + Error::UnsupportedFeature { + version: ArchiveVersion::Rar50, + feature: "quantum compression", + } + .to_string(), + "feature quantum compression is not supported by rar50" + ); + assert_eq!( + Error::Rar50BufferedDecodeLimitExceeded { + limit: 536_870_912, + required: 734_003_200, + } + .to_string(), + "RAR 5 filtered member requires buffered decoding of 734003200 bytes, above the configured limit of 536870912 bytes" + ); + } + + #[test] + fn rar3_recovery_errors_render_messages_through_from_conversion() { + assert_eq!( + Error::from(crate::recovery::rar3::Error::InvalidParitySize).to_string(), + "RAR 3 recovery parity size is invalid" + ); + assert_eq!( + Error::from(crate::recovery::rar3::Error::InvalidCodewordSize).to_string(), + "RAR 3 recovery codeword size is invalid" + ); + assert_eq!( + Error::from(crate::recovery::rar3::Error::TooManyErasures).to_string(), + "RAR 3 recovery data cannot repair this many erasures" + ); + assert_eq!( + Error::from(crate::recovery::rar3::Error::DecodeFailed).to_string(), + "RAR 3 recovery decode failed" + ); + } + + #[test] + fn rar5_recovery_errors_render_messages_for_every_named_variant() { + let cases = [ + ( + crate::recovery::rar5::Error::BadRecoveryChunk, + "RAR 5 recovery chunk is invalid", + ), + ( + crate::recovery::rar5::Error::OddShardSize, + "RAR 5 recovery shard size is odd", + ), + ( + crate::recovery::rar5::Error::PlanOverflow, + "RAR 5 recovery plan overflows", + ), + ( + crate::recovery::rar5::Error::PrefixExceedsPlan, + "RAR 5 recovery prefix exceeds planned shard capacity", + ), + ( + crate::recovery::rar5::Error::ShardSizeMismatch, + "RAR 5 recovery shard sizes differ", + ), + ( + crate::recovery::rar5::Error::TooManyShards, + "RAR 5 recovery shard count is invalid", + ), + ( + crate::recovery::rar5::Error::SingularElement, + "RAR 5 recovery matrix is singular", + ), + ]; + for (variant, expected) in cases { + assert_eq!(Error::from(variant).to_string(), expected); + } + } + + #[test] + fn rar50_crypto_errors_render_messages_through_from_conversion() { + assert_eq!( + Error::from(crate::crypto::rar50::Error::KdfCountTooLarge).to_string(), + "RAR 5 KDF count is too large" + ); + assert_eq!( + Error::from(crate::crypto::rar50::Error::BadPassword).to_string(), + "wrong password or corrupt encrypted data" + ); + } + + #[test] + fn codec_invalid_data_display_uses_inner_message() { + assert_eq!( + Error::from(crate::codec::Error::InvalidData("bad symbol")).to_string(), + "bad symbol" + ); + } + + #[test] + fn unsupported_family_feature_display_renders_family_and_feature() { + assert_eq!( + Error::UnsupportedFamilyFeature { + family: ArchiveFamily::Rar13, + feature: "recovery repair for RAR 1.3/1.4 archives", + } + .to_string(), + "feature recovery repair for RAR 1.3/1.4 archives is not supported by Rar13", + ); + } + + #[test] + fn rar50_crypto_unaligned_input_display_uses_named_message() { + assert_eq!( + Error::from(crate::crypto::rar50::Error::UnalignedInput).to_string(), + "RAR 5 AES input is not block aligned" + ); + } +} diff --git a/arca-rar/vendor/rars/src/extraction_control.rs b/arca-rar/vendor/rars/src/extraction_control.rs new file mode 100644 index 00000000..321a188f --- /dev/null +++ b/arca-rar/vendor/rars/src/extraction_control.rs @@ -0,0 +1,99 @@ +use crate::{ArchiveMember, ArchiveReadOptions, Error, Result}; +use std::io::Write; + +/// Decision made before a member's payload is admitted or decoded. +pub enum ExtractionDecision { + /// Decode and verify the member, writing its contents to this destination. + Extract(Box), + /// Do not read, decrypt, decode or verify this member's payload. + Skip, + /// Finish successfully without processing this member or any later member. + Stop, +} + +/// Whether a controlled extraction visited every member or stopped by request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExtractionOutcome { + /// Reached the end; skipped members and explicitly continued failures were not verified. + Complete, + /// The callback requested an early stop. + Stopped, +} + +/// Response to a failed independent member's extraction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExtractionErrorAction { + /// Return the original extraction error. + Abort, + /// Keep any partial output and attempt the next independent member. + Continue, +} + +pub(crate) type ErrorHandler<'a> = + dyn FnMut(&ArchiveMember, &Error) -> Result + 'a; + +/// Returns true only when an independent member failure was explicitly accepted. +pub(crate) fn finish_member( + handler: &mut Option<&mut ErrorHandler<'_>>, + options: ArchiveReadOptions<'_>, + member: impl FnOnce() -> ArchiveMember, + solid: bool, + result: Result<()>, +) -> Result { + let Err(error) = result else { + return Ok(false); + }; + if solid + || matches!( + error.kind(), + crate::ErrorKind::Cancelled | crate::ErrorKind::ResourceLimit + ) + { + return Err(error); + } + let Some(handler) = handler.as_mut() else { + return Err(error); + }; + options.check_cancelled()?; + let member = member(); + if member.meta.is_split_before || member.meta.is_split_after { + return Err(error); + } + let action = handler(&member, &error)?; + options.check_cancelled()?; + match action { + ExtractionErrorAction::Abort => Err(error), + ExtractionErrorAction::Continue => Ok(true), + } +} + +pub(crate) type Selector<'a> = dyn FnMut(&ArchiveMember) -> Result + 'a; + +pub(crate) fn select( + selector: &mut Option<&mut Selector<'_>>, + options: ArchiveReadOptions<'_>, + member: impl FnOnce() -> ArchiveMember, + solid: bool, +) -> Result> { + let Some(selector) = selector.as_mut() else { + return Ok(None); + }; + options.check_cancelled()?; + let member = member(); + let decision = selector(&member).map_err(|error| { + if error.entry_context().is_none() { + error.at_entry(member.meta.name.clone(), "selecting") + } else { + error + } + })?; + options.check_cancelled()?; + if matches!(decision, ExtractionDecision::Skip) + && solid + && !member.meta.is_directory + && !member.meta.is_redirection + { + return Err(Error::CannotSkipSolidMember.at_entry(member.meta.name, "selecting")); + } + Ok(Some(decision)) +} diff --git a/arca-rar/vendor/rars/src/features.rs b/arca-rar/vendor/rars/src/features.rs new file mode 100644 index 00000000..3146460f --- /dev/null +++ b/arca-rar/vendor/rars/src/features.rs @@ -0,0 +1,113 @@ +/// What a caller is asking a writer to do, beyond the data it hands over. +/// +/// Only choices that cannot be read off the data itself live here. Whether an +/// archive has a comment, a password, per-file comments or a recovery record is +/// decided by whether a comment, a password or a percentage was supplied, so +/// carrying separate flags for those only created a way for the two to +/// disagree, which is a bug the writer then had to check for. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct FeatureSet { + /// Compress members against each other rather than independently. + pub solid: bool, + /// Encrypt the headers as well as the payloads, so names are not readable + /// without the password. + pub header_encryption: bool, + /// Write an index so a reader can list the archive without walking every + /// header. + pub quick_open: bool, +} + +/// One writer feature, in the order a rejection prefers to report them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Feature { + Solid, + HeaderEncryption, + QuickOpen, +} + +impl Feature { + pub const ALL: [Self; 3] = [Self::Solid, Self::HeaderEncryption, Self::QuickOpen]; + + /// How this feature is named when a writer has to refuse it. + pub const fn name(self) -> &'static str { + match self { + Self::Solid => "solid compression", + Self::HeaderEncryption => "header encryption", + Self::QuickOpen => "a quick-open index", + } + } + + const fn get(self, set: FeatureSet) -> bool { + match self { + Self::Solid => set.solid, + Self::HeaderEncryption => set.header_encryption, + Self::QuickOpen => set.quick_open, + } + } +} + +impl FeatureSet { + pub const fn store_only() -> Self { + Self { + solid: false, + header_encryption: false, + quick_open: false, + } + } + + /// The first feature asked for here that `supported` does not offer. + pub fn first_unsupported(self, supported: Self) -> Option { + Feature::ALL + .into_iter() + .find(|feature| feature.get(self) && !feature.get(supported)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Destructuring exhaustively, so a new field is a compile error here + /// rather than a silent hole in `Feature::ALL`. + #[test] + fn every_feature_field_has_an_entry_in_the_table() { + let mut all = FeatureSet::store_only(); + let FeatureSet { + solid, + header_encryption, + quick_open, + } = &mut all; + *solid = true; + *header_encryption = true; + *quick_open = true; + + let reported: Vec<_> = Feature::ALL + .into_iter() + .filter(|feature| feature.get(all)) + .collect(); + assert_eq!(reported, Feature::ALL); + } + + #[test] + fn the_first_missing_feature_is_the_one_reported() { + let asked = FeatureSet { + solid: true, + header_encryption: true, + quick_open: true, + }; + assert_eq!( + asked.first_unsupported(FeatureSet::store_only()), + Some(Feature::Solid) + ); + assert_eq!( + asked.first_unsupported(FeatureSet { + solid: true, + ..FeatureSet::store_only() + }), + Some(Feature::HeaderEncryption) + ); + assert_eq!(asked.first_unsupported(asked), None); + } +} diff --git a/arca-rar/vendor/rars/src/file_times.rs b/arca-rar/vendor/rars/src/file_times.rs new file mode 100644 index 00000000..386a8623 --- /dev/null +++ b/arca-rar/vendor/rars/src/file_times.rs @@ -0,0 +1,526 @@ +//! Lossless RAR5 timestamps and validation/conversion of legacy extended times. + +use crate::{Error, Result}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileTimestamp { + Unix { seconds: u32, nanoseconds: u32 }, + WindowsFiletime(u64), +} + +impl FileTimestamp { + pub fn unix_nanoseconds(self) -> i128 { + match self { + Self::Unix { + seconds, + nanoseconds, + } => i128::from(seconds) * 1_000_000_000 + i128::from(nanoseconds), + Self::WindowsFiletime(ticks) => (i128::from(ticks) - 116_444_736_000_000_000) * 100, + } + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FileTimes { + pub modified: Option, + pub created: Option, + pub accessed: Option, +} + +#[cfg(any(test, feature = "write"))] +pub(crate) struct EncodedTimes { + bytes: [u8; 25], + len: usize, +} +#[cfg(any(test, feature = "write"))] +impl std::ops::Deref for EncodedTimes { + type Target = [u8]; + fn deref(&self) -> &[u8] { + &self.bytes[..self.len] + } +} + +impl FileTimes { + /// Build a record from exact Unix nanoseconds, using FILETIME when needed. + /// FILETIME requires 100-nanosecond precision for every present timestamp. + pub fn from_unix_nanoseconds( + modified: Option, + created: Option, + accessed: Option, + ) -> Result { + let values = [modified, created, accessed]; + let unix = values + .iter() + .flatten() + .all(|value| *value >= 0 && *value / 1_000_000_000 <= i128::from(u32::MAX)); + let mut result = [None; 3]; + for (slot, value) in result.iter_mut().zip(values) { + if let Some(value) = value { + *slot = Some(if unix { + FileTimestamp::Unix { + seconds: (value / 1_000_000_000) as u32, + nanoseconds: (value % 1_000_000_000) as u32, + } + } else { + if value % 100 != 0 { + return Err(Error::InvalidArgument( + "FILETIME range requires 100-nanosecond precision", + )); + } + let ticks = (value / 100) + .checked_add(116_444_736_000_000_000) + .and_then(|ticks| u64::try_from(ticks).ok()) + .ok_or(Error::InvalidArgument("timestamp exceeds FILETIME range"))?; + FileTimestamp::WindowsFiletime(ticks) + }); + } + } + Ok(Self { + modified: result[0], + created: result[1], + accessed: result[2], + }) + } + + #[cfg(any(test, feature = "write"))] + pub(crate) fn encode(self) -> Result { + let times = [self.modified, self.created, self.accessed]; + let first = times + .iter() + .flatten() + .next() + .ok_or(Error::InvalidArgument("file time record is empty"))?; + let unix = matches!(first, FileTimestamp::Unix { .. }); + let mut flags = u8::from(unix); + let mut seconds = [0; 24]; + let mut seconds_len = 0; + let mut fractions = [0; 12]; + let mut fractions_len = 0; + for (index, time) in times.into_iter().enumerate() { + if let Some(time) = time { + flags |= 2 << index; + match time { + FileTimestamp::Unix { + seconds: value, + nanoseconds, + } if unix && nanoseconds < 1_000_000_000 => { + flags |= 0x10; + seconds[seconds_len..seconds_len + 4].copy_from_slice(&value.to_le_bytes()); + seconds_len += 4; + fractions[fractions_len..fractions_len + 4] + .copy_from_slice(&nanoseconds.to_le_bytes()); + fractions_len += 4; + } + FileTimestamp::WindowsFiletime(ticks) if !unix => { + seconds[seconds_len..seconds_len + 8].copy_from_slice(&ticks.to_le_bytes()); + seconds_len += 8; + } + _ => { + return Err(Error::InvalidArgument( + "file times require one encoding and fractions below one second", + )) + } + } + } + } + let mut bytes = [0; 25]; + bytes[0] = flags; + bytes[1..1 + seconds_len].copy_from_slice(&seconds[..seconds_len]); + bytes[1 + seconds_len..1 + seconds_len + fractions_len] + .copy_from_slice(&fractions[..fractions_len]); + Ok(EncodedTimes { + bytes, + len: 1 + seconds_len + fractions_len, + }) + } + + pub(crate) fn parse(flags: u64, data: &[u8]) -> Option { + if flags & !0x1f != 0 || flags & 0x0e == 0 || flags & 0x11 == 0x10 { + return None; + } + let unix = flags & 1 != 0; + let count = (flags & 0x0e).count_ones() as usize; + let width = if unix { 4 } else { 8 }; + let fractions = unix && flags & 0x10 != 0; + if data.len() != count * (width + if fractions { 4 } else { 0 }) { + return None; + } + let mut times = [None; 3]; + let mut at = 0; + for (index, slot) in times.iter_mut().enumerate() { + if flags & (2 << index) == 0 { + continue; + } + *slot = Some(if unix { + let seconds = u32::from_le_bytes(data[at * width..at * width + 4].try_into().ok()?); + let nanoseconds = if fractions { + let offset = count * width + at * 4; + u32::from_le_bytes(data[offset..offset + 4].try_into().ok()?) + } else { + 0 + }; + if nanoseconds >= 1_000_000_000 { + return None; + } + FileTimestamp::Unix { + seconds, + nanoseconds, + } + } else { + FileTimestamp::WindowsFiletime(u64::from_le_bytes( + data[at * width..at * width + 8].try_into().ok()?, + )) + }); + at += 1; + } + Some(Self { + modified: times[0], + created: times[1], + accessed: times[2], + }) + } + + #[cfg(any(test, feature = "write"))] + pub(crate) fn legacy(raw: &[u8], mtime: Option) -> Result> { + if raw.is_empty() { + return Ok(None); + } + let invalid = + || Error::InvalidArgument("legacy extended timestamps are incomplete or invalid"); + let flags = u16::from_le_bytes(raw.get(..2).ok_or_else(invalid)?.try_into().unwrap()); + let mut at = 2; + let mut times = [None; 3]; + // Fourth legacy slot is archival time. It has no RAR5 counterpart. + if flags & 8 != 0 { + return Err(Error::InvalidArgument( + "legacy archival time has no supported RAR5 representation", + )); + } + for (index, slot) in times.iter_mut().enumerate() { + let mode = (flags >> (12 - index * 4)) & 15; + if mode & 8 == 0 { + continue; + } + let seconds = if index == 0 { + mtime.ok_or_else(invalid)? + } else { + let value = u32::from_le_bytes( + raw.get(at..at + 4).ok_or_else(invalid)?.try_into().unwrap(), + ); + at += 4; + value + }; + let mut ticks = 0u32; + for _ in 0..mode & 3 { + ticks = (u32::from(*raw.get(at).ok_or_else(invalid)?) << 16) | (ticks >> 8); + at += 1; + } + if ticks >= 10_000_000 { + return Err(invalid()); + } + let time = crate::timestamp::extracted_system_time( + crate::ArchiveFamily::Rar15To40, + Some(seconds), + Some(crate::TimeRefinement { + add_second: mode & 4 != 0, + nanoseconds: ticks * 100, + }), + ) + .ok_or_else(invalid)?; + let duration = time + .duration_since(std::time::UNIX_EPOCH) + // extracted_system_time adds nonnegative durations to UNIX_EPOCH. + .expect("extracted timestamps cannot precede the Unix epoch"); + *slot = Some(FileTimestamp::Unix { + seconds: u32::try_from(duration.as_secs()).map_err(|_| invalid())?, + nanoseconds: duration.subsec_nanos(), + }); + } + if at != raw.len() { + return Err(invalid()); + } + let times = Self { + modified: times[0], + created: times[1], + accessed: times[2], + }; + Ok((times != Self::default()).then_some(times)) + } +} + +/// Validate a native legacy record without interpreting DOS wall-clock times. +/// Includes the fourth (archival) timestamp, which RAR5 cannot represent. +#[cfg(any(test, feature = "write"))] +pub(crate) fn validate_legacy_extended_times(raw: &[u8]) -> Result<()> { + let invalid = || Error::InvalidArgument("legacy extended timestamps are incomplete or invalid"); + let flags = u16::from_le_bytes(raw.get(..2).ok_or_else(invalid)?.try_into().unwrap()); + let mut at = 2; + for index in 0..4 { + let mode = (flags >> (12 - index * 4)) & 15; + if mode & 8 == 0 { + if mode != 0 { + return Err(invalid()); + } + continue; + } + if index != 0 { + raw.get(at..at + 4).ok_or_else(invalid)?; + at += 4; + } + let mut ticks = 0u32; + for _ in 0..mode & 3 { + ticks = (u32::from(*raw.get(at).ok_or_else(invalid)?) << 16) | (ticks >> 8); + at += 1; + } + if ticks >= 10_000_000 { + return Err(invalid()); + } + } + if at != raw.len() { + return Err(invalid()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn all_rar5_time_combinations_retain_full_precision() { + for unix in [true, false] { + for mask in 1..8 { + let stamp = |index: u64| { + if unix { + FileTimestamp::Unix { + seconds: if index == 0 { 0 } else { u32::MAX }, + nanoseconds: (index as u32 + 1) * 123, + } + } else { + FileTimestamp::WindowsFiletime(if index == 0 { + 0 + } else { + u64::MAX - index + }) + } + }; + let times = FileTimes { + modified: (mask & 1 != 0).then(|| stamp(0)), + created: (mask & 2 != 0).then(|| stamp(1)), + accessed: (mask & 4 != 0).then(|| stamp(2)), + }; + let bytes = times.encode().unwrap(); + assert_eq!( + FileTimes::parse(u64::from(bytes[0]), &bytes[1..]), + Some(times) + ); + } + } + assert_eq!( + FileTimestamp::WindowsFiletime(0).unix_nanoseconds(), + -11_644_473_600_000_000_000 + ); + } + + #[test] + fn malformed_or_mixed_time_records_are_rejected() { + assert!(FileTimes::parse(1, &[]).is_none()); + assert!(FileTimes::parse(0x23, &[0; 4]).is_none()); + assert!(FileTimes::parse(0x13, &[0; 4]).is_none()); + // Fractional seconds are available only with Unix encoding. + assert!(FileTimes::parse(0x12, &[0; 8]).is_none()); + let mut data = vec![0; 4]; + data.extend(1_000_000_000u32.to_le_bytes()); + assert!(FileTimes::parse(0x13, &data).is_none()); + let mixed = FileTimes { + modified: Some(FileTimestamp::WindowsFiletime(0)), + created: Some(FileTimestamp::Unix { + seconds: 0, + nanoseconds: 0, + }), + accessed: None, + }; + assert!(mixed.encode().is_err()); + assert!(FileTimes::default().encode().is_err()); + assert!(FileTimes { + modified: Some(FileTimestamp::Unix { + seconds: 0, + nanoseconds: 1_000_000_000, + }), + ..FileTimes::default() + } + .encode() + .is_err()); + let mut reverse_mixed = mixed; + std::mem::swap(&mut reverse_mixed.modified, &mut reverse_mixed.created); + assert!(reverse_mixed.encode().is_err()); + assert_eq!( + FileTimes::parse(3, &123u32.to_le_bytes()).unwrap().modified, + Some(FileTimestamp::Unix { + seconds: 123, + nanoseconds: 0 + }) + ); + } + + #[test] + fn exact_nanoseconds_choose_one_lossless_encoding_at_range_boundaries() { + let unix_end = i128::from(u32::MAX) * 1_000_000_000 + 999_999_999; + for value in [0, 1, unix_end] { + let times = FileTimes::from_unix_nanoseconds(Some(value), None, None).unwrap(); + assert!(matches!(times.modified, Some(FileTimestamp::Unix { .. }))); + assert_eq!(times.modified.unwrap().unix_nanoseconds(), value); + let encoded = times.encode().unwrap(); + assert_eq!( + FileTimes::parse(u64::from(encoded[0]), &encoded[1..]), + Some(times) + ); + } + let filetime_min = -11_644_473_600_000_000_000i128; + let filetime_max = (i128::from(u64::MAX) - 116_444_736_000_000_000) * 100; + for value in [filetime_min, -100, unix_end + 1, filetime_max] { + let times = FileTimes::from_unix_nanoseconds(Some(value), Some(0), None).unwrap(); + assert!(matches!( + times.modified, + Some(FileTimestamp::WindowsFiletime(_)) + )); + assert!(matches!( + times.created, + Some(FileTimestamp::WindowsFiletime(_)) + )); + assert_eq!(times.modified.unwrap().unix_nanoseconds(), value); + assert_eq!(times.created.unwrap().unix_nanoseconds(), 0); + let encoded = times.encode().unwrap(); + assert_eq!( + FileTimes::parse(u64::from(encoded[0]), &encoded[1..]), + Some(times) + ); + } + for value in [ + -1, + unix_end + 2, + filetime_min - 100, + filetime_max + 100, + i128::MIN, + i128::MAX, + ] { + assert!( + FileTimes::from_unix_nanoseconds(Some(value), None, None).is_err(), + "{value}" + ); + } + assert_eq!( + FileTimes::from_unix_nanoseconds(None, None, None).unwrap(), + FileTimes::default() + ); + } + + #[test] + fn legacy_conversion_refuses_missing_invalid_and_unix_out_of_range_dates() { + for (raw, modified) in [ + (vec![0], None), + (0x8000u16.to_le_bytes().to_vec(), None), + (0x8000u16.to_le_bytes().to_vec(), Some(0)), + (0x0800u16.to_le_bytes().to_vec(), None), + (0x9000u16.to_le_bytes().to_vec(), Some(0x5022_1882)), + ] { + assert!(FileTimes::legacy(&raw, modified).is_err()); + } + let future_dos = (127 << 25) | (1 << 21) | (1 << 16); // 2107-01-01 + let instant = crate::timestamp::extracted_system_time( + crate::ArchiveFamily::Rar15To40, + Some(future_dos), + None, + ) + .unwrap(); + assert!( + instant + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + > u64::from(u32::MAX) + ); + assert!(FileTimes::legacy(&0x8000u16.to_le_bytes(), Some(future_dos)).is_err()); + assert_eq!(FileTimes::legacy(&[], None).unwrap(), None); + assert!(validate_legacy_extended_times(&[0]).is_err()); + assert!(validate_legacy_extended_times(&0x0800u16.to_le_bytes()).is_err()); + assert!(validate_legacy_extended_times(&0x9000u16.to_le_bytes()).is_err()); + } + + #[test] + fn archival_time_is_preserved_natively_but_cannot_be_converted_to_rar5() { + // Legacy extended-time flags place archival time in the low nibble. + let mut raw = 0x0008u16.to_le_bytes().to_vec(); + raw.extend(0x5022_1882u32.to_le_bytes()); + validate_legacy_extended_times(&raw).unwrap(); + assert_eq!( + FileTimes::legacy(&raw, None), + Err(Error::InvalidArgument( + "legacy archival time has no supported RAR5 representation" + )) + ); + } + + #[test] + fn legacy_fractions_end_before_one_second_and_records_have_no_trailing_bytes() { + let dos = 0x5022_1882; + for (ticks, valid) in [ + (9_999_999u32, true), + (10_000_000, false), + (0xff_ffff, false), + ] { + let mut raw = 0xb000u16.to_le_bytes().to_vec(); + raw.extend_from_slice(&ticks.to_le_bytes()[..3]); + assert_eq!(validate_legacy_extended_times(&raw).is_ok(), valid); + let converted = FileTimes::legacy(&raw, Some(dos)); + assert_eq!(converted.is_ok(), valid); + if valid { + let FileTimestamp::Unix { nanoseconds, .. } = + converted.unwrap().unwrap().modified.unwrap() + else { + panic!("legacy conversion must use Unix seconds"); + }; + assert_eq!(nanoseconds, ticks * 100); + } + } + assert_eq!(FileTimes::legacy(&[0, 0], None).unwrap(), None); + assert!(FileTimes::legacy(&[0, 0, 1], None).is_err()); + assert!(validate_legacy_extended_times(&[0, 0, 1]).is_err()); + } + + #[test] + fn legacy_creation_and_access_times_keep_odd_seconds_and_fractions() { + let dos = 0x5022_1882u32; + let mut raw = 0xfb90u16.to_le_bytes().to_vec(); + raw.extend([1, 2, 3]); + raw.extend(dos.to_le_bytes()); + raw.extend([4, 5, 6]); + raw.extend(dos.to_le_bytes()); + raw.push(7); + let times = FileTimes::legacy(&raw, Some(dos)).unwrap().unwrap(); + let base = crate::timestamp::extracted_system_time( + crate::ArchiveFamily::Rar15To40, + Some(dos), + None, + ) + .unwrap() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() as i128 + * 1_000_000_000; + assert_eq!( + times.modified.unwrap().unix_nanoseconds(), + base + 1_000_000_000 + 0x030201 * 100 + ); + assert_eq!( + times.created.unwrap().unix_nanoseconds(), + base + 0x060504 * 100 + ); + assert_eq!( + times.accessed.unwrap().unix_nanoseconds(), + base + 0x070000 * 100 + ); + raw.pop(); + assert!(FileTimes::legacy(&raw, Some(dos)).is_err()); + } +} diff --git a/arca-rar/vendor/rars/src/filename.rs b/arca-rar/vendor/rars/src/filename.rs new file mode 100644 index 00000000..71890604 --- /dev/null +++ b/arca-rar/vendor/rars/src/filename.rs @@ -0,0 +1,607 @@ +//! Conversion at the boundary between archive member names and native paths. +//! +//! Legacy byte names have no reliably inferable code page. On Unix they remain +//! bytes; Unicode legacy names have already been decoded by the format reader. +//! RAR5 archive names remain in their wire encoding in member metadata and lookup +//! APIs. Its Unix byte mapping is applied only when importing/exporting paths. + +use crate::{Error, Result}; +use std::borrow::Cow; +use std::ffi::{OsStr, OsString}; +use std::path::{Component, Path, PathBuf}; + +/// Native filename bytes on Unix, UTF-8 elsewhere. Never substitutes characters. +pub fn native_bytes(name: &OsStr) -> Result<&[u8]> { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + Ok(name.as_bytes()) + } + #[cfg(not(unix))] + { + name.to_str() + .map(str::as_bytes) + .ok_or(Error::InvalidArgument( + "native filename cannot be represented as Unicode", + )) + } +} + +/// Convert decoded filename bytes to a native name without replacement. +/// Non-Unix platforms require Unicode; an unspecified legacy code page is not +/// guessed. This function does not validate paths or decode RAR5 wire names. +pub fn native_string(name: &[u8]) -> Result { + Ok(native_str(name)?.to_os_string()) +} + +/// Borrows decoded filename bytes as a native name without replacement. +/// Like [`native_string`], this does not validate paths or decode RAR5 wire names. +pub fn native_str(name: &[u8]) -> Result<&OsStr> { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + Ok(OsStr::from_bytes(name)) + } + #[cfg(not(unix))] + { + std::str::from_utf8(name).map(OsStr::new).map_err(|_| { + Error::InvalidArgument( + "archive name requires an explicit legacy code page on this platform", + ) + }) + } +} + +/// Constructs a nonempty native relative path from already decoded name bytes. +/// +/// Callers choose whether backslashes separate components (legacy RAR) or remain +/// literal characters (RAR5 on Unix). Native platform path syntax still applies. +/// Rejects NUL, drive prefixes, roots and parent components; removes `.` and +/// repeated separators. Filename interpretation and identity preflight remain +/// separate caller policies. +pub fn relative_path(name: &[u8], backslash_separator: bool) -> Result { + if name.contains(&0) { + return Err(Error::UnsafePath("unsafe archive path contains NUL byte")); + } + let bytes = if backslash_separator && name.contains(&b'\\') { + Cow::Owned( + name.iter() + .map(|&byte| if byte == b'\\' { b'/' } else { byte }) + .collect::>(), + ) + } else { + Cow::Borrowed(name) + }; + let text = native_str(&bytes)?; + if bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' { + return Err(Error::UnsafePath("unsafe archive path")); + } + let mut out = PathBuf::new(); + for component in Path::new(text).components() { + match component { + Component::Normal(part) => out.push(part), + Component::CurDir => {} + _ => return Err(Error::UnsafePath("unsafe archive path")), + } + } + if out.as_os_str().is_empty() { + return Err(Error::InvalidHeader("empty archive path")); + } + Ok(out) +} + +/// Joins normal relative components to a caller-selected, trusted root. +/// Rejects existing symlinks in appended components; an empty relative path +/// returns the root. This does not create or open files. Symlink checks are a +/// snapshot; callers control subsequent filesystem access and publication. +pub fn checked_output_path(root: &Path, relative: &Path) -> Result { + let mut path = root.to_path_buf(); + for component in relative.components() { + let Component::Normal(part) = component else { + return Err(Error::UnsafePath("unsafe archive path")); + }; + path.push(part); + if std::fs::symlink_metadata(&path).is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(Error::UnsafePath("unsafe archive path crosses symlink")); + } + } + Ok(path) +} + +/// Encode native Unix filename bytes as a RAR5 UTF-8 name. +/// +/// Valid UTF-8 is unchanged unless it contains the reserved U+FFFE marker. +/// Otherwise high bytes are mapped to U+E080..U+E0FF with a U+FFFE marker. +/// Mapping all high bytes in this case also protects literal private-use +/// characters and literal markers from being confused with encoded bytes. +/// See . +pub fn encode_rar50(name: &[u8]) -> Cow<'_, [u8]> { + if std::str::from_utf8(name).is_ok_and(|s| !s.contains('\u{fffe}')) { + return Cow::Borrowed(name); + } + let mut out = String::from("\u{fffe}"); + for &byte in name { + out.push(if byte < 0x80 { + char::from(byte) + } else { + char::from_u32(0xe000 + u32::from(byte)).expect("mapped high byte") + }); + } + Cow::Owned(out.into_bytes()) +} + +/// Encode a renamed legacy Unicode entry without relying on a code page. +/// Full UTF-16 units make the Unicode stream independent of the ASCII fallback. +#[cfg(any(test, feature = "write"))] +pub(crate) fn encode_legacy_unicode(name: &[u8]) -> Result> { + validate_relative(name)?; + let text = std::str::from_utf8(name) + .map_err(|_| Error::InvalidArgument("renaming a Unicode entry requires a UTF-8 name"))?; + let mut raw: Vec = text + .chars() + .map(|ch| if ch.is_ascii() { ch as u8 } else { b'_' }) + .collect(); + raw.extend_from_slice(&[0, 0]); + let units: Vec = text.encode_utf16().collect(); + for group in units.chunks(4) { + raw.push(0xaa); // Four full UTF-16 commands; unused low bits are ignored. + for unit in group { + raw.extend_from_slice(&unit.to_le_bytes()); + } + } + if raw.len() > usize::from(u16::MAX) - 32 { + return Err(Error::InvalidArgument( + "legacy Unicode filename is too long", + )); + } + Ok(raw) +} + +/// Restore a RAR5 Unix mapped name to filename bytes. Call only for Unix-host +/// names when writing to a Unix filesystem. Unmarked names, including malformed +/// UTF-8 from tolerant archive readers, retain their exact bytes. +pub fn decode_rar50(name: &[u8]) -> Cow<'_, [u8]> { + let Ok(text) = std::str::from_utf8(name) else { + return Cow::Borrowed(name); + }; + if !text.contains('\u{fffe}') { + return Cow::Borrowed(name); + } + let mut out = Vec::with_capacity(name.len()); + for ch in text.chars() { + match ch { + '\u{fffe}' => {} + '\u{e080}'..='\u{e0ff}' => out.push((ch as u32 - 0xe000) as u8), + _ => out.extend_from_slice(ch.encode_utf8(&mut [0; 4]).as_bytes()), + } + } + Cow::Owned(out) +} + +/// Validate relative member identity using ASCII path syntax, independently of +/// filename encoding and of the destination filesystem. +pub(crate) fn validate_relative(name: &[u8]) -> Result<()> { + if name.contains(&0) { + return Err(Error::UnsafePath("unsafe archive path contains NUL byte")); + } + if name.first().is_some_and(|b| matches!(b, b'/' | b'\\')) + || (name.len() >= 2 && name[0].is_ascii_alphabetic() && name[1] == b':') + || name + .split(|b| matches!(b, b'/' | b'\\')) + .any(|p| p == b"..") + { + return Err(Error::UnsafePath("unsafe archive path")); + } + if name + .split(|b| matches!(b, b'/' | b'\\')) + .all(|p| p.is_empty() || p == b".") + { + return Err(Error::InvalidArgument("empty archive path")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn legacy_unicode_names_respect_the_header_length_limit() { + // ASCII fallback plus full UTF-16 units and one command per four + // units: 20154 characters produce the largest fitting field. + let name = vec![b'a'; 20_154]; + let encoded = encode_legacy_unicode(&name).unwrap(); + assert_eq!(encoded.len(), 65_503); + crate::rar15_40::validate_unicode_name(&encoded, &name).unwrap(); + let too_long = vec![b'a'; 20_155]; + assert_eq!( + encode_legacy_unicode(&too_long), + Err(Error::InvalidArgument( + "legacy Unicode filename is too long" + )), + ); + } + + #[test] + fn legacy_unicode_commands_preserve_supplementary_characters_and_group_tails() { + for name in ["a", "abcd", "abcde", "notes/🦀/é.txt"] { + let encoded = encode_legacy_unicode(name.as_bytes()).unwrap(); + crate::rar15_40::validate_unicode_name(&encoded, name.as_bytes()).unwrap(); + } + } + + #[test] + fn utf8_legacy_decoding_is_strict_for_both_valid_and_invalid_input() { + for name in ["", "é", "🦀", "a/b"] { + assert_eq!( + LegacyNameEncoding::Utf8.decode(name.as_bytes()).unwrap(), + name + ); + } + assert_eq!( + LegacyNameEncoding::Utf8.decode(&[0xff]), + Err(Error::InvalidArgument("legacy name is not valid UTF-8")), + ); + } + + #[test] + fn native_names_preserve_utf8_and_unix_legacy_bytes_without_replacement() { + let bytes = "café.txt".as_bytes(); + let borrowed = native_str(bytes).unwrap(); + assert!(std::ptr::eq( + native_bytes(borrowed).unwrap().as_ptr(), + bytes.as_ptr() + )); + let name = native_string(bytes).unwrap(); + assert_eq!(native_bytes(&name).unwrap(), bytes); + #[cfg(unix)] + { + let bytes = b"caf\xff.txt"; + let name = native_string(bytes).unwrap(); + assert_eq!(native_bytes(&name).unwrap(), bytes); + } + #[cfg(not(unix))] + assert!(native_string(b"caf\xff.txt").is_err()); + } + + #[test] + fn destination_paths_preserve_selected_separator_policy_and_error_categories() { + for separator in [false, true] { + for name in [b"a\0b".as_slice(), b"/root", b"a/../b", b"C:root"] { + assert!(matches!( + relative_path(name, separator), + Err(Error::UnsafePath(_)) + )); + } + for name in [b"".as_slice(), b".", b"././"] { + assert_eq!( + relative_path(name, separator), + Err(Error::InvalidHeader("empty archive path")) + ); + } + assert_eq!( + relative_path(b"./a//./b", separator).unwrap(), + PathBuf::from("a").join("b") + ); + } + assert_eq!( + relative_path(b"a\\b", true).unwrap(), + PathBuf::from("a").join("b") + ); + assert!(relative_path(b"a\\..\\b", true).is_err()); + #[cfg(unix)] + { + assert_eq!( + relative_path(b"..\\literal", false).unwrap(), + PathBuf::from("..\\literal") + ); + assert_eq!( + native_bytes(relative_path(b"caf\xff", false).unwrap().as_os_str()).unwrap(), + b"caf\xff" + ); + } + #[cfg(not(unix))] + assert!(relative_path(b"caf\xff", false).is_err()); + } + + #[test] + #[cfg(unix)] + fn destination_paths_refuse_link_components_and_keep_root_selection_separate() { + let root = crate::scratch::case("shared-destination-policy"); + let target = root.join("target"); + std::fs::create_dir(&target).unwrap(); + std::os::unix::fs::symlink(&target, root.join("link")).unwrap(); + assert_eq!( + checked_output_path(&root, Path::new("link/file")), + Err(Error::UnsafePath("unsafe archive path crosses symlink")) + ); + assert_eq!( + checked_output_path(&root, Path::new("link")), + Err(Error::UnsafePath("unsafe archive path crosses symlink")) + ); + for path in ["../file", "/file", "./file"] { + assert_eq!( + checked_output_path(&root, Path::new(path)), + Err(Error::UnsafePath("unsafe archive path")) + ); + } + assert_eq!( + checked_output_path(&root, Path::new("")), + Ok(root.to_path_buf()) + ); + assert_eq!( + checked_output_path(&root.join("link"), Path::new("file")), + Ok(root.join("link/file")) + ); + assert_eq!( + checked_output_path(&root, Path::new("missing/file")), + Ok(root.join("missing/file")) + ); + } + + #[test] + fn relative_names_reject_unsafe_or_empty_paths_in_both_legacy_separator_styles() { + for name in [ + b"a\0b".as_slice(), + b"/root", + b"\\root", + b"C:\\root", + b"a/../b", + b"a\\..\\b", + ] { + assert!( + matches!(validate_relative(name), Err(Error::UnsafePath(_))), + "{name:?}" + ); + } + for name in [b"".as_slice(), b".", b".//.", b".\\.\\"] { + assert_eq!( + validate_relative(name), + Err(Error::InvalidArgument("empty archive path")) + ); + } + for name in [ + b"a/b".as_slice(), + b"a\\b", + b".//a", + b"a\\.\\b", + b"legacy-\xff", + ] { + validate_relative(name).unwrap(); + } + assert!(matches!(decode_rar50(b"plain-name"), Cow::Borrowed(_))); + assert!(matches!(decode_rar50(b"bad-\xff"), Cow::Borrowed(_))); + } + + #[test] + fn rar50_mapping_is_reversible_and_does_not_alias_unicode() { + let mut bytes: Vec = (0x80..=0xff).collect(); + bytes.extend_from_slice("/日本語/\u{fffe}\u{e080}".as_bytes()); + for name in [ + bytes.as_slice(), + "literal-\u{fffe}".as_bytes(), + b"bad-\xff", + "\u{e0ff}".as_bytes(), + ] { + let encoded = encode_rar50(name); + assert!(std::str::from_utf8(&encoded).is_ok()); + assert_eq!(decode_rar50(&encoded).as_ref(), name); + } + assert_ne!( + encode_rar50(b"bad-\xff"), + encode_rar50("bad-\u{fffd}".as_bytes()) + ); + assert!(matches!( + encode_rar50("cafe\u{301}/日本語".as_bytes()), + Cow::Borrowed(_) + )); + } + + #[test] + fn rar50_mapping_accepts_reference_mixed_names_and_marker_positions() { + for encoded in ["a\u{fffe}\u{e0ff}日本語", "a\u{e0ff}日本語\u{fffe}"] { + assert_eq!( + decode_rar50(encoded.as_bytes()).as_ref(), + b"a\xff\xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e" + ); + } + // Only high bytes are mapped; lower private-use characters stay Unicode. + assert_eq!( + decode_rar50("\u{fffe}\u{e02f}".as_bytes()).as_ref(), + "\u{e02f}".as_bytes() + ); + } +} + +/// Explicit source encoding for legacy names without a Unicode representation. +/// These mappings are locale independent and shared by every binding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum LegacyNameEncoding { + Cp437, + Cp850, + Cp852, + Cp866, + Windows1251, + Windows1252, + Utf8, +} + +impl std::str::FromStr for LegacyNameEncoding { + type Err = Error; + fn from_str(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "cp437" => Ok(Self::Cp437), + "cp850" => Ok(Self::Cp850), + "cp852" => Ok(Self::Cp852), + "cp866" => Ok(Self::Cp866), + "cp1251" | "windows-1251" => Ok(Self::Windows1251), + "cp1252" | "windows-1252" => Ok(Self::Windows1252), + "utf8" | "utf-8" => Ok(Self::Utf8), + _ => Err(Error::InvalidArgument("unsupported legacy name encoding (use cp437, cp850, cp852, cp866, windows-1251, windows-1252 or utf-8)")), + } + } +} + +impl LegacyNameEncoding { + /// Strict decoding: undefined bytes never become replacement characters. + pub fn decode(self, name: &[u8]) -> Result { + let table = match self { + Self::Utf8 => { + return std::str::from_utf8(name) + .map(str::to_owned) + .map_err(|_| Error::InvalidArgument("legacy name is not valid UTF-8")); + } + Self::Cp437 => &CP437, + Self::Cp850 => &CP850, + Self::Cp852 => &CP852, + Self::Cp866 => &CP866, + Self::Windows1251 => &CP1251, + Self::Windows1252 => &CP1252, + }; + name.iter() + .map(|&byte| { + if byte < 128 { + Ok(char::from(byte)) + } else { + char::from_u32(u32::from(table[usize::from(byte - 128)])).ok_or( + Error::InvalidArgument("undefined byte in selected legacy name encoding"), + ) + } + }) + .collect() + } +} + +/// Interpret a legacy name without changing its stored identity. `unicode` means +/// the format already supplied Unicode, which always takes precedence. Without +/// an explicit encoding the existing bytes are returned unchanged. +pub fn decoded_name( + name: &[u8], + unicode: bool, + encoding: Option, +) -> Result> { + match encoding { + Some(encoding) if !unicode => encoding + .decode(name) + .map(|text| Cow::Owned(text.into_bytes())) + .map_err(|error| error.at_entry(name.to_vec(), "decoding legacy filename")), + _ => Ok(Cow::Borrowed(name)), + } +} + +// High-byte mappings from Python's standard-library IBM/Microsoft codec tables. +// ASCII 0..127 is unchanged. 0xD800 marks an undefined code-page byte. +const CP437: [u16; 128] = [ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x00ec, 0x00c4, 0x00c5, 0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, + 0x00ff, 0x00d6, 0x00dc, 0x00a2, 0x00a3, 0x00a5, 0x20a7, 0x0192, 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb, + 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, + 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4, 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, 0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0, +]; +const CP850: [u16; 128] = [ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x00ec, 0x00c4, 0x00c5, 0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, + 0x00ff, 0x00d6, 0x00dc, 0x00f8, 0x00a3, 0x00d8, 0x00d7, 0x0192, 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x00ae, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb, + 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, 0x00c2, 0x00c0, 0x00a9, 0x2563, 0x2551, 0x2557, + 0x255d, 0x00a2, 0x00a5, 0x2510, 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x00e3, 0x00c3, + 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4, 0x00f0, 0x00d0, 0x00ca, 0x00cb, + 0x00c8, 0x0131, 0x00cd, 0x00ce, 0x00cf, 0x2518, 0x250c, 0x2588, 0x2584, 0x00a6, 0x00cc, 0x2580, + 0x00d3, 0x00df, 0x00d4, 0x00d2, 0x00f5, 0x00d5, 0x00b5, 0x00fe, 0x00de, 0x00da, 0x00db, 0x00d9, + 0x00fd, 0x00dd, 0x00af, 0x00b4, 0x00ad, 0x00b1, 0x2017, 0x00be, 0x00b6, 0x00a7, 0x00f7, 0x00b8, + 0x00b0, 0x00a8, 0x00b7, 0x00b9, 0x00b3, 0x00b2, 0x25a0, 0x00a0, +]; +const CP852: [u16; 128] = [ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x016f, 0x0107, 0x00e7, 0x0142, 0x00eb, 0x0150, 0x0151, + 0x00ee, 0x0179, 0x00c4, 0x0106, 0x00c9, 0x0139, 0x013a, 0x00f4, 0x00f6, 0x013d, 0x013e, 0x015a, + 0x015b, 0x00d6, 0x00dc, 0x0164, 0x0165, 0x0141, 0x00d7, 0x010d, 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x0104, 0x0105, 0x017d, 0x017e, 0x0118, 0x0119, 0x00ac, 0x017a, 0x010c, 0x015f, 0x00ab, 0x00bb, + 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, 0x00c2, 0x011a, 0x015e, 0x2563, 0x2551, 0x2557, + 0x255d, 0x017b, 0x017c, 0x2510, 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x0102, 0x0103, + 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4, 0x0111, 0x0110, 0x010e, 0x00cb, + 0x010f, 0x0147, 0x00cd, 0x00ce, 0x011b, 0x2518, 0x250c, 0x2588, 0x2584, 0x0162, 0x016e, 0x2580, + 0x00d3, 0x00df, 0x00d4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161, 0x0154, 0x00da, 0x0155, 0x0170, + 0x00fd, 0x00dd, 0x0163, 0x00b4, 0x00ad, 0x02dd, 0x02db, 0x02c7, 0x02d8, 0x00a7, 0x00f7, 0x00b8, + 0x00b0, 0x00a8, 0x02d9, 0x0171, 0x0158, 0x0159, 0x25a0, 0x00a0, +]; +const CP866: [u16; 128] = [ + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041a, 0x041b, + 0x041c, 0x041d, 0x041e, 0x041f, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, + 0x044c, 0x044d, 0x044e, 0x044f, 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040e, 0x045e, + 0x00b0, 0x2219, 0x00b7, 0x221a, 0x2116, 0x00a4, 0x25a0, 0x00a0, +]; +const CP1251: [u16; 128] = [ + 0x0402, 0x0403, 0x201a, 0x0453, 0x201e, 0x2026, 0x2020, 0x2021, 0x20ac, 0x2030, 0x0409, 0x2039, + 0x040a, 0x040c, 0x040b, 0x040f, 0x0452, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0xd800, 0x2122, 0x0459, 0x203a, 0x045a, 0x045c, 0x045b, 0x045f, 0x00a0, 0x040e, 0x045e, 0x0408, + 0x00a4, 0x0490, 0x00a6, 0x00a7, 0x0401, 0x00a9, 0x0404, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0407, + 0x00b0, 0x00b1, 0x0406, 0x0456, 0x0491, 0x00b5, 0x00b6, 0x00b7, 0x0451, 0x2116, 0x0454, 0x00bb, + 0x0458, 0x0405, 0x0455, 0x0457, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, 0x0420, 0x0421, 0x0422, 0x0423, + 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043a, 0x043b, + 0x043c, 0x043d, 0x043e, 0x043f, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, +]; +const CP1252: [u16; 128] = [ + 0x20ac, 0xd800, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, 0x2039, + 0x0152, 0xd800, 0x017d, 0xd800, 0xd800, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0xd800, 0x017e, 0x0178, 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00bf, 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, +]; + +#[cfg(all(test, windows))] +#[test] +fn windows_native_names_reject_unpaired_surrogates_without_replacement() { + use std::os::windows::ffi::OsStringExt; + for units in [&[0xd800][..], &[0xdc00], &[u16::from(b'a'), 0xd800]] { + let name = OsString::from_wide(units); + assert_eq!( + native_bytes(&name), + Err(Error::InvalidArgument( + "native filename cannot be represented as Unicode" + )) + ); + } + let paired = OsString::from_wide(&[0xd83d, 0xde00]); + assert_eq!(native_bytes(&paired).unwrap(), "😀".as_bytes()); +} + +/// Reject absolute paths, `..`, drive prefixes and NUL bytes without assuming +/// a filename encoding. Destination filesystem representability and RAR5 Unix +/// byte mapping are separate from archive member identity validation. +pub fn validate_entry_name(name: Vec) -> Result> { + crate::filename::validate_relative(&name)?; + Ok(name) +} + +/// The path a member name denotes below an output directory, or an error if it +/// denotes anywhere else. Backslashes are separators, because that is what a +/// DOS-era writer put in the header. On Unix non-UTF-8 bytes are preserved. +/// This is the legacy path convention, not a RAR5 wire-name decoder. +pub fn entry_relative_path(name: &[u8]) -> Result { + crate::filename::validate_relative(name)?; + relative_path(name, true) +} diff --git a/arca-rar/vendor/rars/src/filter.rs b/arca-rar/vendor/rars/src/filter.rs new file mode 100644 index 00000000..66ec0579 --- /dev/null +++ b/arca-rar/vendor/rars/src/filter.rs @@ -0,0 +1,237 @@ +//! Data filters: the reversible transforms a writer applies to a member before +//! compressing it, so the encoder sees something it can match on. +//! +//! Every RAR family that has filters at all draws from this set, but no family +//! encodes all of it. RAR 5 has four builtin filter types; the RAR 2.9 family +//! ships six filters as RarVM programs; delta and the two x86 filters are the +//! only ones common to both. Asking a writer for a filter it cannot encode is +//! rejected when the archive is written, not when the filter is named, because +//! the format is chosen at the same moment and often from the same argument. + +use std::ops::Range; + +/// A reversible transform applied to a member before it is compressed. +/// +/// Deliberately not `#[non_exhaustive]`: each format converts this into its own +/// narrower set with an exhaustive match and no wildcard arm, so an eighth +/// filter has to be decided about everywhere before the crate will build. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FilterKind { + /// Splits the member into `channels` planes and stores each plane as + /// differences, which turns a smooth ramp into a run of near-zero bytes. + Delta { channels: usize }, + /// Rewrites x86 `E8` call targets from relative to absolute, so the same + /// call made from different places becomes the same bytes. + E8, + /// As [`FilterKind::E8`], also converting `E9` jumps. + E8E9, + /// The ARM equivalent, rewriting `BL` branch targets. + Arm, + /// The Itanium equivalent. RAR 2.9 family only. + Itanium, + /// De-interleaves `width`-byte pixels and predicts each channel from its + /// neighbours. RAR 2.9 family only. + Rgb { width: usize, pos_r: usize }, + /// Predicts each sample from the previous ones in the same channel. + /// RAR 2.9 family only. + Audio { channels: usize }, +} + +impl FilterKind { + /// What this filter is called when a writer has to refuse it. + pub const fn name(self) -> &'static str { + match self { + Self::Delta { .. } => "the delta filter", + Self::E8 => "the x86 E8 filter", + Self::E8E9 => "the x86 E8/E9 filter", + Self::Arm => "the ARM filter", + Self::Itanium => "the Itanium filter", + Self::Rgb { .. } => "the RGB filter", + Self::Audio { .. } => "the audio filter", + } + } +} + +/// A filter and the bytes it covers. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FilterSpec { + pub kind: FilterKind, + /// The bytes the filter covers. `None` covers the whole member. + pub range: Option>, +} + +impl FilterSpec { + pub const fn whole(kind: FilterKind) -> Self { + Self { kind, range: None } + } + + pub const fn range(kind: FilterKind, range: Range) -> Self { + Self { + kind, + range: Some(range), + } + } +} + +impl From for FilterSpec { + fn from(kind: FilterKind) -> Self { + Self::whole(kind) + } +} + +/// How a writer decides which filter, if any, to apply to a member. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum FilterPolicy { + /// Compress the member as it stands. + #[default] + None, + /// Look for a filter that makes the member compress smaller, and use one + /// only if it does. + Auto, + /// Apply this filter whether or not it pays off. + Explicit(FilterSpec), +} + +impl FilterPolicy { + /// The common case: one filter over the whole member. + pub const fn explicit(kind: FilterKind) -> Self { + Self::Explicit(FilterSpec::whole(kind)) + } +} + +impl FilterKind { + /// Whether this format's writer can encode this filter. + /// + /// Answered by the same conversions the writers use, so the question and + /// the refusal cannot disagree. + pub fn is_supported_by(self, target: crate::ArchiveVersion) -> bool { + match target.family() { + crate::version::ArchiveFamily::Rar50Plus => { + crate::codec::rar50::Rar50Filter::try_from(self).is_ok() + } + crate::version::ArchiveFamily::Rar15To40 => { + // RAR 1.5 and 2.0 predate the filter VM entirely. + !matches!( + target, + crate::ArchiveVersion::Rar15 | crate::ArchiveVersion::Rar20 + ) && crate::codec::rar29::Rar29Filter::try_from(self).is_ok() + } + crate::version::ArchiveFamily::Rar13 => false, + } + } +} + +/// Every format whose writer can encode `kind`. +pub fn formats_supporting_filter(kind: FilterKind) -> Vec { + crate::ArchiveVersion::ALL + .into_iter() + .filter(|&target| kind.is_supported_by(target)) + .collect() +} + +/// A filter the target format has no way to encode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnsupportedFilterKind(pub FilterKind); + +#[cfg(test)] +mod tests { + use super::*; + use crate::codec::rar29::Rar29Filter; + use crate::codec::rar50::Rar50Filter; + + /// Exhaustive by construction: a new filter has to be added here, and the + /// two conversions below then have to say what each format does with it. + const ALL: [FilterKind; 7] = [ + FilterKind::Delta { channels: 1 }, + FilterKind::E8, + FilterKind::E8E9, + FilterKind::Arm, + FilterKind::Itanium, + FilterKind::Rgb { width: 3, pos_r: 0 }, + FilterKind::Audio { channels: 1 }, + ]; + + #[test] + fn kind_conversion_selects_the_whole_member_without_changing_parameters() { + for kind in ALL { + let spec = FilterSpec::from(kind); + assert_eq!(spec.kind, kind); + assert_eq!(spec.range, None); + } + } + + #[test] + fn every_filter_is_named_and_accepted_by_at_least_one_format() { + for kind in ALL { + assert!( + !kind.name().is_empty(), + "{kind:?} has no name to refuse it by" + ); + assert!( + Rar50Filter::try_from(kind).is_ok() || Rar29Filter::try_from(kind).is_ok(), + "{kind:?} is a filter no format can write" + ); + } + } + + /// The two families are complementary rather than nested, which is why a + /// rejection is worth naming the format that does support the filter. + #[test] + fn the_two_families_each_have_what_the_other_lacks() { + assert!(Rar50Filter::try_from(FilterKind::Arm).is_ok()); + assert!(Rar29Filter::try_from(FilterKind::Arm).is_err()); + for kind in [ + FilterKind::Itanium, + FilterKind::Rgb { width: 3, pos_r: 0 }, + FilterKind::Audio { channels: 1 }, + ] { + assert!(Rar29Filter::try_from(kind).is_ok(), "{kind:?}"); + assert_eq!( + Rar50Filter::try_from(kind), + Err(UnsupportedFilterKind(kind)), + "{kind:?}" + ); + } + } + + #[test] + fn each_filter_names_the_formats_that_can_write_it() { + use crate::ArchiveVersion; + assert_eq!( + formats_supporting_filter(FilterKind::Arm), + vec![ArchiveVersion::Rar50, ArchiveVersion::Rar70] + ); + assert_eq!( + formats_supporting_filter(FilterKind::Itanium), + vec![ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40 + ] + ); + // Delta is the one every filtering format has. + assert_eq!( + formats_supporting_filter(FilterKind::Delta { channels: 2 }), + vec![ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70 + ] + ); + for kind in ALL { + assert!(!formats_supporting_filter(kind).is_empty(), "{kind:?}"); + } + } + + #[test] + fn a_policy_defaults_to_leaving_the_data_alone() { + assert_eq!(FilterPolicy::default(), FilterPolicy::None); + assert_eq!( + FilterPolicy::explicit(FilterKind::E8), + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::E8)) + ); + } +} diff --git a/arca-rar/vendor/rars/src/filter_search.rs b/arca-rar/vendor/rars/src/filter_search.rs new file mode 100644 index 00000000..59eef09b --- /dev/null +++ b/arca-rar/vendor/rars/src/filter_search.rs @@ -0,0 +1,2292 @@ +//! Choosing a filter for a member, for any format that has filters. +//! +//! The expensive part of picking a filter is that judging one means compressing +//! the member with it. Doing that for every filter a format offers, times every +//! region a scanner proposes, is most of the cost of writing an archive and +//! nearly all of it is spent proving that filters which were never going to help +//! do not help. +//! +//! So the candidates are cut down before any whole-member encode happens. +//! Filters with a structural detector behind them get their regions from the +//! detector and a sample encode per region to confirm the detector was not +//! fooled by chance byte patterns and to drop the regions where the filter +//! would cost more than it saves. The x86 pair detect code by its opcodes; the +//! table scanner detects arrays of fixed-size records by the stride they +//! repeat at, and its regions become ranged delta filters at the record size. +//! Filters with no detector are measured on a sample of the member and have to +//! win by a clear margin to earn a whole-member encode. What survives is a +//! handful of finalists, each encoded once at the caller's real settings, +//! smallest wins. +//! +//! The unfiltered member is always one of the finalists, so the result can never +//! be larger than leaving the data alone. + +use crate::codec::workspace::{Allowance, Budget, Buffer}; +use crate::{FilterKind, FilterSpec, Result}; +use std::ops::Range; + +/// How much of the member the screens encode when deciding whether a filter is +/// worth a whole-member encode. +const SCREEN_SAMPLE_LEN: usize = 128 * 1024; + +/// Keeps the screen sample's delta planes aligned the way they fall in the whole +/// member: the lowest common multiple of the delta widths worth trying. +const SCREEN_SAMPLE_ALIGNMENT: usize = 12; + +/// How much smaller a filter has to make the sample before it earns a +/// whole-member encode, as a percentage. +/// +/// A filter that pays off does so by a mile: delta on 16-bit audio takes 43% off +/// and on interleaved counters 96%. A sample that comes out a fraction of a +/// percent smaller is measurement noise from the shorter history, and chasing it +/// costs a whole-member encode per filter to find out. +const SCREEN_MARGIN_PERCENT: usize = 1; + +/// How much of the member x86 detection has to cover before filtering the whole +/// thing is as good as filtering only the detected regions, as a fraction. +const X86_CODE_COVERAGE_RATIO: (usize, usize) = (9, 10); + +/// How many x86 regions the progress estimate assumes the scanner will find. +/// +/// Only used to scale a progress bar, and deliberately not measured: finding out +/// means scanning the member, which is a whole extra pass to sharpen a +/// percentage. Two to five is what the binaries measured came to, with an +/// unstripped 25 MB outlier at eleven. +const X86_ASSUMED_REGIONS: u64 = 4; + +/// Bytes at each end of a member that a ranged delta filter skips, because +/// container headers and trailers are not part of the sampled signal. +pub(crate) const AUTO_DELTA_EDGE_SKIP: usize = 64; + +/// How much of the member the table scanner judges at a time. +/// +/// A window is the resolution of the region boundaries: a table loses at most +/// one window at each end. Smaller windows find smaller tables and cost more +/// scanning per member. +const TABLE_SCAN_WINDOW: usize = 16 * 1024; + +/// The widest record stride the table scanner looks for. +/// +/// This is the scanner's own ceiling, traded against what it costs: every extra +/// stride is another comparison per byte of the member. What the *format* can +/// encode is a separate question, answered by +/// [`FilterSearch::max_delta_channels`], and the scan uses whichever is lower. +const MAX_TABLE_STRIDE: usize = 32; + +/// How close a byte has to be to the one a stride back to count as part of the +/// record structure. Not just equal: a field that creeps, like an address +/// column, is exactly what delta flattens. +const TABLE_STRIDE_NEAR: u8 = 2; + +/// How much of a window has to repeat at the winning stride before the window +/// counts as table-like, as a percentage. Code and text sit well under half +/// this at every stride; an array of structs sits well over it at the record +/// size, because most fields barely change from one record to the next. +const TABLE_STRIDE_HIT_PERCENT: usize = 40; + +/// How close to the best stride's score a shorter stride has to come to be +/// preferred, as a percentage. An 8-byte record repeats at 16, 24 and 32 too, +/// and the shortest period is the record size; the tolerance keeps noise from +/// picking a harmonic. +const TABLE_STRIDE_TIE_PERCENT: usize = 95; + +/// The smallest run of table-like windows worth a filter of its own. +const MIN_TABLE_REGION: usize = 2 * TABLE_SCAN_WINDOW; + +/// How many table regions the progress estimate assumes the scanner will find. +/// +/// Same reasoning as [`X86_ASSUMED_REGIONS`], and the same refusal to go and +/// measure: the binaries this was built for have one or two, being a +/// relocation table and sometimes a symbol table beside it. +const TABLE_ASSUMED_REGIONS: u64 = 2; + +/// What a writer has to tell the search about its format. +/// +/// Implementors describe a format at fixed encoder settings and hold no state, +/// so a search can run inside a parallel per-member encode. +pub(crate) trait FilterSearch { + /// The encoder settings a candidate is measured at. Compared, so the search + /// can tell when a screen's encode is also a real measurement. + type Options: Copy + PartialEq; + + /// Every filter this format can encode that has no detector of its own, so + /// the only way to know whether it helps is to try it. Given the member, so + /// a format can drop widths that do not fit or that its own cheap + /// statistics have already ruled out. + fn screened_kinds(&self, data: &[u8]) -> Vec; + + /// Whether this format can encode the x86 filters, which come with a + /// structural detector instead of a screen. + fn detects_x86(&self) -> bool { + true + } + + /// The most channels this format's delta filter can carry. + /// + /// The table scanner picks a stride from the data rather than from + /// [`FilterSearch::screened_kinds`], so this is how a format keeps it + /// inside what the writer can encode. Deliberately has no default: a + /// format that gets this wrong proposes a filter its own writer must then + /// refuse, and there is no safe guess to make on its behalf. + fn max_delta_channels(&self) -> usize; + + /// Cheaper settings for the screens, when ranking candidates at less than + /// full effort ranks them the same way. + /// + /// Defaults to the caller's own settings, because on RAR 5 the + /// reduced-effort parse measured *slower* than the full one: weaker matches + /// mean more for the entropy coder to do. On RAR 2.9 it really is cheaper, + /// so that format overrides this. + fn screen_options(&self, options: Self::Options) -> Self::Options { + options + } + + /// Apply the filters to a copy of `data`, without encoding it. + /// + /// This is how a screen prices a filter it can only afford to try on a slice + /// of the member: compress the transformed bytes and compare against the + /// unfiltered ones. Comparing a filtered *encode* with an unfiltered one used + /// to measure something else entirely. On a 396 KB binary the 128 KiB sample + /// said the E8E9 filter cost 10% (74322 plain against 81744 filtered) while + /// the same filter was worth 7.4% of the finished archive, so every x86 + /// member was screened out before anything measured it. + /// + /// That gap was the two paths cutting blocks at different sizes, fixed in + /// 6ce759c: they now agree to within a filter token per block. So a screen + /// that can encode the whole member goes straight to + /// [`FilterSearch::encode_filtered`] and keeps the result, and this stays for + /// the case it was written for, where the sample is a slice and there is + /// nothing to keep. + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result>; + + /// Encode without filters. + /// + /// Separate from [`FilterSearch::encode_filtered`] because carrying filters + /// can force shorter blocks, so encoding an empty filter list would not + /// produce what asking for no filter produces. The search compares every + /// candidate against leaving the data alone, and that has to mean the bytes + /// the writer would really emit. + fn encode_plain( + &self, + data: &[u8], + options: Self::Options, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result>; + + /// Encode with these filters applied. Never called with an empty list. + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: Self::Options, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result>; +} + +pub(crate) type FilterChoice = (Buffer, Buffer); +type Finalist = (Buffer, Option>); +type TableRegions = Buffer<(Range, usize), B>; + +/// Search owners cannot detach bounded byte storage while candidates compete. +pub(crate) trait OwnedSearch { + type Options: Copy + PartialEq; + type Memory: Budget; + fn allowance(&self) -> &Self::Memory; + fn screened_kinds(&self, data: &[u8]) -> Result>; + fn detects_x86(&self) -> bool; + fn max_delta_channels(&self) -> usize; + fn screen_options(&self, options: Self::Options) -> Self::Options; + fn filtered_bytes( + &self, + data: &[u8], + filters: &[FilterSpec], + ) -> Result>; + fn encode_plain( + &self, + data: &[u8], + options: Self::Options, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result>; + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: Self::Options, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result>; +} +struct UnlimitedSearch<'a, S>(&'a S, Allowance); +impl OwnedSearch for UnlimitedSearch<'_, S> { + type Options = S::Options; + type Memory = Allowance; + fn allowance(&self) -> &Allowance { + &self.1 + } + fn screened_kinds(&self, data: &[u8]) -> Result> { + Ok(Buffer::from_vec(self.0.screened_kinds(data))) + } + fn detects_x86(&self) -> bool { + self.0.detects_x86() + } + fn max_delta_channels(&self) -> usize { + self.0.max_delta_channels() + } + fn screen_options(&self, options: Self::Options) -> Self::Options { + self.0.screen_options(options) + } + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result> { + self.0.filtered_bytes(data, filters).map(Buffer::from_vec) + } + fn encode_plain( + &self, + data: &[u8], + options: Self::Options, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + self.0 + .encode_plain(data, options, progress) + .map(Buffer::from_vec) + } + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: Self::Options, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + self.0 + .encode_filtered(data, filters, options, progress) + .map(Buffer::from_vec) + } +} + +/// Internal events: codec positions are absolute only within one encode pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EncodeProgress { + PassStarted, + Advanced(usize), +} + +/// Adapt a single codec pass without inferring restarts from byte positions. +pub(crate) fn encode_pass( + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, + encode: impl FnOnce(Option<&mut dyn FnMut(usize) -> bool>) -> Result, +) -> Result { + match progress { + Some(progress) => { + if !progress(EncodeProgress::PassStarted) { + return Err(crate::Error::Cancelled); + } + encode(Some(&mut |position| { + progress(EncodeProgress::Advanced(position)) + })) + } + None => encode(None), + } +} + +/// Observe every encode in a search, including sample screens which previously +/// discarded the caller's progress/cancellation callback. The codec adapter and +/// search decisions remain unchanged. +struct ReportingSearch<'a, S> { + search: &'a S, + progress: std::cell::RefCell<&'a mut dyn FnMut(EncodeProgress) -> bool>, +} +impl OwnedSearch for ReportingSearch<'_, S> { + type Options = S::Options; + type Memory = S::Memory; + fn allowance(&self) -> &Self::Memory { + self.search.allowance() + } + fn screened_kinds(&self, data: &[u8]) -> Result> { + self.search.screened_kinds(data) + } + fn detects_x86(&self) -> bool { + self.search.detects_x86() + } + fn max_delta_channels(&self) -> usize { + self.search.max_delta_channels() + } + fn screen_options(&self, options: Self::Options) -> Self::Options { + self.search.screen_options(options) + } + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result> { + self.search.filtered_bytes(data, filters) + } + fn encode_plain( + &self, + data: &[u8], + options: Self::Options, + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + encode_pass(Some(&mut **self.progress.borrow_mut()), |progress| { + self.search.encode_plain(data, options, progress) + }) + } + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: Self::Options, + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + encode_pass(Some(&mut **self.progress.borrow_mut()), |progress| { + self.search + .encode_filtered(data, filters, options, progress) + }) + } +} + +/// Whether looking for a filter is worth starting at all. +/// +/// An empty member has nothing to filter, and text does not benefit from any of +/// them, so neither should cost a single encode to find that out. +pub(crate) fn search_applies(data: &[u8]) -> bool { + !data.is_empty() && !is_text_like(data) +} + +fn is_text_like(data: &[u8]) -> bool { + let sample_len = data.len().min(8192); + // `search_applies` screens empty members before calling this helper. + let sample = &data[..sample_len]; + let text_bytes = sample + .iter() + .filter(|&&byte| matches!(byte, b'\t' | b'\n' | b'\r' | 0x20..=0x7e)) + .count(); + text_bytes * 100 / sample_len >= 95 +} + +fn screen_wins(filtered: usize, baseline: usize) -> bool { + filtered * 100 < baseline * (100 - SCREEN_MARGIN_PERCENT) +} + +/// A window from the middle of the member, used to screen the filters that have +/// nothing but a trial encode to go on. +fn screen_sample(data: &[u8]) -> &[u8] { + if data.len() <= SCREEN_SAMPLE_LEN { + return data; + } + let middle = (data.len() - SCREEN_SAMPLE_LEN) / 2; + let start = middle / SCREEN_SAMPLE_ALIGNMENT * SCREEN_SAMPLE_ALIGNMENT; + &data[start..start + SCREEN_SAMPLE_LEN] +} + +/// What the screen made of one filter kind. +struct ScreenedKind { + kind: FilterKind, + /// The bytes, when the screen encoded the whole member the way the writer + /// would write it, so the search can take this as its measurement. + measured: Option>, + /// Whether it beat leaving the data alone by enough to be worth spending + /// another encode on a narrower range of the same filter. + worth_a_range: bool, +} + +/// What the screen learned, along with any encodes the search can reuse. +struct ScreenOutcome { + kinds: Buffer, B>, + plain: Option>, +} + +/// Which of the detectorless filters shrink a sample of the member. +/// +/// These used to cost a whole-member encode each to prove they made the member +/// bigger, which on a binary is most of the search. Whether they help is a local +/// property of the byte layout, so a slice of the member answers it for a few +/// per cent of the price. This measures the real filter rather than a statistic +/// standing in for it, because the wins do not all show up as entropy: RAR's +/// delta reorders into planes first, and what that buys is repetition the +/// encoder can match, not a flatter byte histogram. +/// +/// A member no bigger than one sample is a special case worth taking. There is +/// no slice to approximate with, so the screen encodes it the way the writer +/// would and every number it produces is the measurement the search was going to +/// spend an encode on anyway. That skips a second encode per surviving kind, and +/// it lets kinds that win by too little to be worth chasing stay in the running, +/// because keeping a candidate the screen has already paid for costs nothing. +fn screen_kinds( + search: &S, + data: &[u8], + options: S::Options, +) -> Result> { + let sample = screen_sample(data); + let mut outcome = ScreenOutcome { + kinds: Buffer::new(search.allowance()), + plain: None, + }; + if sample.len() < SCREEN_SAMPLE_ALIGNMENT { + return Ok(outcome); + } + let screen_options = search.screen_options(options); + // Only when the screen measures at the settings the finalists will use. A + // format that screens at reduced effort, which RAR 2.9 does, is measuring + // something else and keeps the two-encode path. + let measures_the_member = sample.len() == data.len() && screen_options == options; + let baseline = search.encode_plain(sample, screen_options, None)?; + for kind in search.screened_kinds(data)? { + let filters = [FilterSpec::whole(kind)]; + let packed = if measures_the_member { + search.encode_filtered(data, &filters, options, None)? + } else { + let transformed = search.filtered_bytes(sample, &filters)?; + search.encode_plain(&transformed, screen_options, None)? + }; + let worth_a_range = screen_wins(packed.len(), baseline.len()); + if measures_the_member { + outcome.kinds.try_push(ScreenedKind { + kind, + measured: Some(packed), + worth_a_range, + })?; + } else if worth_a_range { + // The screen measured the transform, not the encode the writer + // would emit, so this is evidence for a finalist rather than a + // measurement the search can reuse. + outcome.kinds.try_push(ScreenedKind { + kind, + measured: None, + worth_a_range, + })?; + } + } + if measures_the_member { + outcome.plain = Some(baseline); + } + Ok(outcome) +} + +/// Where the scanner thinks x86 code lives, merged into disjoint regions. +fn x86_code_regions(data: &[u8], allowance: &B) -> Result, B>> { + disjoint_ranges(crate::x86_filter_scan::ranges_with_allowance( + data, true, allowance, + )?) +} + +/// Which of the scanner's regions the x86 filter should cover, empty when none +/// of them is worth filtering. +/// +/// Byte patterns that look like call opcodes turn up in compressed data by +/// chance, and the scanner cannot tell those from a real code section. A sample +/// encode inside a region separates the two for a fraction of what a +/// whole-member encode costs. +/// +/// Every region gets its own sample, because on an unstripped binary the biggest +/// region is the debug data rather than the code. Screening only the largest, as +/// this did, declined the filter on every unstripped binary measured: it read +/// DWARF, saw no win, and stopped before anything measured the member. Regions +/// are disjoint, so the samples together never come to more than the member. +/// +/// A region the filter makes bigger is dropped rather than counted against the +/// rest, which is what keeps the debug data out of the ranged candidate. On a +/// 20 MB unstripped binary that turns a filter worth 0.2% into one worth +/// 0.4%; leaving it in was worse than not filtering at all. Ties are kept: a +/// sample with no convertible opcodes in it is not evidence against the region +/// it came from. +/// +/// A kept region is then asked the second question: does leaving the jump +/// opcodes alone beat converting them as well. That decides whether the E8-only +/// filter is worth a whole-member encode of its own, and the answer is usually +/// no. Over twenty-four members it won seven times and never by more than +/// 0.21%, while carrying the pair cost a third of the search; screening it here +/// keeps the wins and spends the encode only where a sample gives a reason to. +fn x86_screened_regions( + search: &S, + data: &[u8], + regions: &[Range], + options: S::Options, +) -> Result> { + let mut kept = Buffer::new(search.allowance()); + let mut helped = false; + for region in regions { + let sample = screen_sample(&data[region.clone()]); + if sample.len() < SCREEN_SAMPLE_ALIGNMENT { + // Scanner padding makes this possible only when the entire + // member is too short to provide a useful x86 sample. + continue; + } + // Measured at the caller's real settings, not the cheaper screen ones. + // The detectorless screens rank many candidates against each other, + // where a reduced parse ranks them the same way; this is a yes or no + // with no margin under it, and a sample encode is cheap enough to get + // right. + let baseline = search.encode_plain(sample, options, None)?; + let transformed = search.filtered_bytes(sample, &[FilterSpec::whole(FilterKind::E8E9)])?; + let filtered = search.encode_plain(&transformed, options, None)?; + if filtered.len() > baseline.len() { + continue; + } + // No margin here, unlike the detectorless filters: the scanner has + // already ruled on where code is, so a small win on this sample is + // evidence rather than noise. + helped |= filtered.len() < baseline.len(); + kept.try_push((region.clone(), filtered.len()))?; + } + let rejected_a_region = kept.len() < regions.len(); + if !helped { + return Ok(X86Screen { + kept: Buffer::new(search.allowance()), + rejected_a_region: false, + jumps_cost_more: false, + }); + } + + // Held back until the regions are known to be worth filtering at all. Asking + // per region inside the loop above priced the jump opcodes on members that + // then declined every filter, which is a sample encode each for an answer + // nothing reads. + let mut jumps_cost_more = false; + for (region, e8e9) in kept.iter() { + let sample = screen_sample(&data[region.clone()]); + let e8_only = search.filtered_bytes(sample, &[FilterSpec::whole(FilterKind::E8)])?; + let e8_only = search.encode_plain(&e8_only, options, None)?; + jumps_cost_more |= e8_only.len() < *e8e9; + } + + Ok(X86Screen { + rejected_a_region, + kept: Buffer::collect( + kept.into_iter().map(|(region, _)| region), + search.allowance(), + )?, + jumps_cost_more, + }) +} + +/// What the x86 screen made of the regions the scanner proposed. +struct X86Screen { + /// The regions worth filtering, empty when none of them is. + kept: Buffer, B>, + /// Whether any region came out bigger under the filter, which rules out + /// filtering the member end to end. + rejected_a_region: bool, + /// Whether any kept region did better with the jump opcodes left alone, + /// which is the only reason to price the E8-only filter separately. A tie + /// does not count: it usually means the sample held no jump opcodes at all, + /// so it is an absence of evidence, and acting on it costs a whole-member + /// encode. + jumps_cost_more: bool, +} + +/// The x86 filter specs worth measuring against the whole member, given what the +/// screen made of the regions the scanner proposed. +/// +/// The scanner proposes overlapping regions at several clustering distances, and +/// the search this replaced priced every one of them with its own whole-member +/// encode. Measured at full effort the whole spread is worth about a third of a +/// percent, so this keeps the two that are structurally different: filter +/// everything, or filter only where the screen saw code. +/// +/// Which of the two is worth an encode is mostly already known. Filtering +/// everything covers the regions the screen rejected as well, so once it has +/// rejected one, that candidate is asking to be told again that those bytes come +/// out bigger; on all seven binaries measured the kept regions beat it. Filtering +/// only the regions is the same thing as filtering everything when they cover +/// nearly the whole member. So each case is worth one candidate per kind, not +/// two. +/// +/// And usually one kind rather than two, because the screen has already asked +/// whether the jump opcodes are worth converting. That takes an unstripped +/// binary from five whole-member encodes to two. +fn x86_finalists( + data: &[u8], + screen: &X86Screen, +) -> Result, B>> { + let allowance = screen.kept.allowance(); + let regions = &screen.kept; + let covered: usize = regions.iter().map(|range| range.len()).sum(); + let (numerator, denominator) = X86_CODE_COVERAGE_RATIO; + let sparse = covered * denominator < data.len() * numerator; + let mut finalists = Buffer::new(&allowance); + for kind in [FilterKind::E8E9, FilterKind::E8] + .into_iter() + .take(if screen.jumps_cost_more { 2 } else { 1 }) + { + if screen.rejected_a_region || sparse { + finalists.try_push(Buffer::collect( + regions + .iter() + .map(|range| FilterSpec::range(kind, range.clone())), + &allowance, + )?)?; + } + if !screen.rejected_a_region { + finalists.try_push(Buffer::collect([FilterSpec::whole(kind)], &allowance)?)?; + } + } + Ok(finalists) +} + +/// The record stride this window repeats at, when it looks like an array of +/// fixed-size records at all. +/// +/// An array of structs repeats at its record size: most fields hold the same or +/// a slowly moving value from one record to the next, so the byte one stride +/// back predicts this one. Code and text have no such period. The score is how +/// many bytes land within [`TABLE_STRIDE_NEAR`] of the byte a stride back, +/// and the shortest stride within [`TABLE_STRIDE_TIE_PERCENT`] of the best +/// wins, because a period repeats at its own multiples. +fn table_stride(window: &[u8], max_stride: usize) -> Option { + // table_regions supplies a full scan window and caps the stride, so the + // only caller-side case without a candidate is a format with no delta. + if max_stride == 0 { + return None; + } + let mut hits = [0usize; MAX_TABLE_STRIDE + 1]; + for index in max_stride..window.len() { + let byte = window[index]; + for stride in 1..=max_stride { + let diff = (byte.wrapping_sub(window[index - stride]) as i8).unsigned_abs(); + hits[stride] += usize::from(diff <= TABLE_STRIDE_NEAR); + } + } + let total = window.len() - max_stride; + let best = *hits[1..=max_stride] + .iter() + .max() + .expect("there is at least one stride"); + if best * 100 < total * TABLE_STRIDE_HIT_PERCENT { + return None; + } + (1..=max_stride).find(|&stride| hits[stride] * 100 >= best * TABLE_STRIDE_TIE_PERCENT) +} + +/// Where the member holds arrays of fixed-size records, each with the stride +/// it repeats at. +/// +/// This is what the whole-member delta screens cannot see: a relocation table +/// is a few hundred kilobytes of 24-byte records inside a binary that delta +/// with stride 24 nearly halves, while the same filter over the whole member +/// is a disaster. On a 3.6 MB shared object the table packed to 32,767 bytes +/// against 59,512 unfiltered; delta 24 over the whole member came out 85% +/// *bigger* than no filter at all. The filter has to cover the table and stop. +fn table_regions( + data: &[u8], + max_stride: usize, + allowance: &B, +) -> Result, usize), B>> { + let max_stride = max_stride.min(MAX_TABLE_STRIDE); + let mut regions: Buffer<(Range, usize), B> = Buffer::new(allowance); + for start in (0..).map(|window| window * TABLE_SCAN_WINDOW) { + let Some(window) = data.get(start..start + TABLE_SCAN_WINDOW) else { + break; + }; + let Some(stride) = table_stride(window, max_stride) else { + continue; + }; + match regions.last_mut() { + Some((last, last_stride)) if *last_stride == stride && last.end == start => { + last.end = start + TABLE_SCAN_WINDOW; + } + _ => regions.try_push((start..start + TABLE_SCAN_WINDOW, stride))?, + } + } + regions.retain(|(range, _)| range.len() >= MIN_TABLE_REGION); + Ok(regions) +} + +/// Which of the scanner's table regions the delta filter actually shrinks. +/// +/// The scanner reads byte statistics, and repetition at a stride is also +/// something the match finder can sometimes reach on its own; a run of +/// constant bytes scores as a table at every stride and needs no filter at +/// all. A sample encode inside the region separates the tables delta helps +/// from the ones LZ was already handling, the same way the x86 screen checks +/// its scanner. +fn table_screened_regions( + search: &S, + data: &[u8], + regions: &[(Range, usize)], + options: S::Options, +) -> Result> { + let screen_options = search.screen_options(options); + let mut kept = Buffer::new(search.allowance()); + for (region, stride) in regions { + let sample = screen_sample(&data[region.clone()]); + if sample.len() < SCREEN_SAMPLE_ALIGNMENT { + continue; + } + let baseline = search.encode_plain(sample, screen_options, None)?; + let filters = [FilterSpec::whole(FilterKind::Delta { channels: *stride })]; + let transformed = search.filtered_bytes(sample, &filters)?; + let filtered = search.encode_plain(&transformed, screen_options, None)?; + if screen_wins(filtered.len(), baseline.len()) { + kept.try_push((region.clone(), *stride))?; + } + } + Ok(kept) +} + +/// `range` with the `removed` ranges cut out of it. `removed` must be sorted +/// and disjoint. +fn subtract_owned( + range: Range, + removed: &[Range], + allowance: &B, +) -> Result, B>> { + let mut kept = Buffer::new(allowance); + let mut start = range.start; + for cut in removed { + if cut.end <= start { + continue; + } + if cut.start >= range.end { + break; + } + if cut.start > start { + kept.try_push(start..cut.start)?; + } + start = start.max(cut.end); + } + if start < range.end { + kept.try_push(start..range.end)?; + } + Ok(kept) +} + +/// A finalist with the screened tables riding along: the table ranges are cut +/// out of whatever the specs covered, and delta at the record stride takes +/// their place. Empty specs come out as a tables-only candidate. +/// +/// The tables win where the two overlap. A verified table stride is worth +/// tens of percent of its region where the x86 filter is worth a few. This +/// used to build one bundled candidate that competed with the x86 finalists +/// instead, and on an unstripped binary the x86-only finalist won, taking the +/// tables down with the debug data they were bundled against. Grafting the +/// tables into every finalist lets them ride with whichever x86 variant wins. +fn graft_owned( + specs: Buffer, + tables: &[(Range, usize)], + member: usize, +) -> Result> { + if tables.is_empty() { + return Ok(specs); + } + let allowance = specs.allowance(); + let table_ranges = Buffer::collect(tables.iter().map(|(range, _)| range.clone()), &allowance)?; + let mut grafted = Buffer::new(&allowance); + for spec in specs { + let covered = spec.range.clone().unwrap_or(0..member); + for range in subtract_owned(covered, &table_ranges, &allowance)? { + grafted.try_push((grafted.len(), FilterSpec::range(spec.kind, range)))?; + } + } + for (range, stride) in tables { + grafted.try_push(( + grafted.len(), + FilterSpec::range(FilterKind::Delta { channels: *stride }, range.clone()), + ))?; + } + // Explicit original order preserves stable ties without an unaccounted sort allocation. + grafted.sort_unstable_by_key(|(order, spec)| { + (spec.range.as_ref().map_or(0, |range| range.start), *order) + }); + Ok(Buffer::collect( + grafted.into_iter().map(|(_, spec)| spec), + &allowance, + )?) +} + +/// The filter specs worth measuring against the whole member, paired with the +/// bytes for any the screen already measured. +/// +/// Everything expensive happens downstream of this, one whole-member encode per +/// unmeasured finalist, so the job here is to hand back a handful rather than the +/// several dozen the scanner and the delta widths can between them suggest. +fn finalists( + search: &S, + data: &[u8], + options: S::Options, +) -> Result, S::Memory>> { + let screen = screen_kinds(search, data, options)?; + let allowance = search.allowance(); + let mut finalists = Buffer::collect([(Buffer::new(allowance), screen.plain)], allowance)?; + + let table_regions = table_regions(data, search.max_delta_channels(), allowance)?; + let tables = table_screened_regions(search, data, &table_regions, options)?; + + // The tables graft into every x86 finalist rather than competing against + // them, so when the x86 screen kept anything they cost no encode of their + // own. Only a member with tables and no code pays for a tables-only + // candidate. + let mut tables_carried = false; + if search.detects_x86() { + let screen = + x86_screened_regions(search, data, &x86_code_regions(data, allowance)?, options)?; + if !screen.kept.is_empty() { + tables_carried = !tables.is_empty(); + for specs in x86_finalists(data, &screen)? { + finalists.try_push((graft_owned(specs, &tables, data.len())?, None))?; + } + } + } + if !tables.is_empty() && !tables_carried { + finalists.try_push(( + graft_owned(Buffer::new(allowance), &tables, data.len())?, + None, + ))?; + } + + for screened in screen.kinds { + finalists.try_push(( + Buffer::collect([FilterSpec::whole(screened.kind)], allowance)?, + screened.measured, + ))?; + if let (true, FilterKind::Delta { channels }) = (screened.worth_a_range, screened.kind) { + if let Some(range) = auto_delta_filter_range(data, channels) { + finalists.try_push(( + Buffer::collect([FilterSpec::range(screened.kind, range)], allowance)?, + None, + ))?; + } + } + } + + Ok(finalists) +} + +/// The candidate filter lists, screened but not ranked. +/// +/// The screens still run: whether a filter has any chance on this member is a +/// property of the member's own bytes, and a sample answers it the same way +/// whoever is going to measure the survivors. What this hands back is the +/// ranking, for a caller whose real encoder is not the one +/// [`FilterSearch::encode_filtered`] describes. +/// +/// A solid chain is that caller. Its members code against everything before +/// them, so a candidate encoded on its own has not been measured at all, and +/// the bytes [`choose_filter`] returns are bytes the chain would never write. +pub(crate) fn filter_candidates( + search: &S, + data: &[u8], + options: S::Options, +) -> Result>> { + Ok(finalists( + &UnlimitedSearch(search, Allowance::default()), + data, + options, + )? + .into_iter() + .map(|(specs, _measured)| specs.into_vec()) + .collect()) +} + +/// Picks the filter for a member, returning the winning specs together with the +/// bytes they produced so the caller does not have to encode the winner again. +/// +/// Which filter suits a member is a property of the data, not of how hard the +/// encoder is trying, so this is worth doing once even when several encoder +/// settings are going to be compared afterwards. +pub(crate) fn choose_filter( + search: &S, + data: &[u8], + options: S::Options, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, +) -> Result<(Vec, Vec)> { + choose_owned( + &UnlimitedSearch(search, Allowance::default()), + data, + options, + progress, + ) + .map(|(specs, bytes)| (specs.into_vec(), bytes.into_vec())) +} + +pub(crate) fn choose_owned( + search: &S, + data: &[u8], + options: S::Options, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, +) -> Result> { + match progress { + Some(progress) => choose_filter_inner( + &ReportingSearch { + search, + progress: std::cell::RefCell::new(progress), + }, + data, + options, + ), + None => choose_filter_inner(search, data, options), + } +} + +fn choose_filter_inner( + search: &S, + data: &[u8], + options: S::Options, +) -> Result> { + let mut best: Option> = None; + for (specs, measured) in finalists(search, data, options)? { + let packed = match measured { + Some(packed) => packed, + None if specs.is_empty() => search.encode_plain(data, options, None)?, + None => search.encode_filtered(data, &specs, options, None)?, + }; + if best + .as_ref() + .is_none_or(|(_, best)| packed.len() < best.len()) + { + best = Some((specs, packed)); + } + } + Ok(best.expect("the unfiltered member is always a finalist")) +} + +/// How many bytes the encoder will walk while this search runs. +/// +/// Progress is reported by encoder position, and the search walks the member +/// several times over, so the reporter needs the total to scale by. What +/// survives the screens is not knowable without doing the work, so this assumes +/// one detectorless filter does, and that the x86 detector finds code and +/// proposes its usual pair of specs. +/// +/// Deliberately does not run the detector to find out. This is called for every +/// member before anything is compressed, and scanning here only to scan again +/// inside the search cost a second pass over the member to sharpen a percentage. +/// A progress bar that finishes a little early or a little late is not worth +/// that. +pub(crate) fn walk_bytes_for_kinds(data: &[u8], screened: u64, encoder_candidates: usize) -> u64 { + let member = data.len() as u64; + let encoder_candidates = encoder_candidates.max(1) as u64; + let sample = screen_sample(data).len() as u64; + // Two sample encodes per region for the x86 screen, and the two specs it + // proposes when the detector does find code. How many regions there are is + // not knowable without scanning, so this assumes [`X86_ASSUMED_REGIONS`], + // bounded by the member: the regions are disjoint, so however many the + // scanner finds, their samples together cannot come to more than that. + let x86_screen = (sample * X86_ASSUMED_REGIONS).min(member) * 2; + let x86_finalists = 2; + // And two more per table region, on the same assumption and with the same + // bound. Tables graft into the x86 finalists rather than earning an encode + // of their own, and this already assumes the detector finds code, so they + // add nothing beyond their screens here. + let table_screen = (sample * TABLE_ASSUMED_REGIONS).min(member) * 2; + let screen = sample * (screened + 1) + x86_screen + table_screen; + // The unfiltered member, the x86 finalists, and one assumed screen + // survivor. When the sample is the whole member the screen encodes it the + // way the writer would, so the unfiltered member and the survivor are + // already counted in `screen`. That reads the sample rather than asking + // whether this format screens at full effort, which needs the caller's + // settings; RAR 5 is the only format that reports progress through here + // and it always does. + let finalists = if sample == member { + x86_finalists + } else { + 2 + x86_finalists + }; + screen + member * (finalists + encoder_candidates - 1) +} + +/// The bytes a ranged delta filter covers: the member minus its edges, trimmed +/// so the planes stay aligned. +pub(crate) fn auto_delta_filter_range(data: &[u8], channels: usize) -> Option> { + if channels == 0 || data.len() <= AUTO_DELTA_EDGE_SKIP * 2 + channels * 8 { + return None; + } + let start = AUTO_DELTA_EDGE_SKIP; + let end = data.len() - AUTO_DELTA_EDGE_SKIP; + let aligned_start = start + ((channels - start % channels) % channels); + let aligned_end = end - (end - aligned_start) % channels; + (aligned_start + channels * 8 <= aligned_end).then_some(aligned_start..aligned_end) +} + +/// Merges overlapping and touching ranges. +/// +/// Merging rather than dropping: two ranges that overlap describe one region, +/// and keeping only the first loses coverage of the rest of it. +#[cfg(test)] +pub(crate) fn disjoint_filter_ranges(ranges: Vec>) -> Vec> { + disjoint_ranges(Buffer::from_vec(ranges)) + .expect("unlimited ranges") + .into_vec() +} +fn disjoint_ranges( + mut ranges: Buffer, B>, +) -> Result, B>> { + ranges.sort_unstable_by_key(|range| (range.start, range.end)); + let mut disjoint: Buffer, B> = Buffer::new(&ranges.allowance()); + for range in ranges { + if let Some(last) = disjoint.last_mut() { + if range.start <= last.end { + last.end = last.end.max(range.end); + continue; + } + } + disjoint.try_push(range)?; + } + Ok(disjoint) +} + +#[cfg(test)] +mod tests { + fn screen_kinds( + search: &S, + data: &[u8], + options: S::Options, + ) -> Result> { + super::screen_kinds( + &UnlimitedSearch(search, Allowance::default()), + data, + options, + ) + } + fn x86_code_regions(data: &[u8]) -> Vec> { + super::x86_code_regions(data, &Allowance::default()) + .unwrap() + .into_vec() + } + fn x86_screened_regions( + search: &S, + data: &[u8], + regions: &[Range], + options: S::Options, + ) -> Result> { + super::x86_screened_regions( + &UnlimitedSearch(search, Allowance::default()), + data, + regions, + options, + ) + } + fn x86_finalists(data: &[u8], screen: &X86Screen) -> Vec> { + super::x86_finalists(data, screen) + .unwrap() + .into_iter() + .map(Buffer::into_vec) + .collect() + } + fn delta_table_regions(data: &[u8], max_stride: usize) -> Vec<(Range, usize)> { + table_regions(data, max_stride, &Allowance::default()) + .unwrap() + .into_vec() + } + fn table_screened_regions( + search: &S, + data: &[u8], + regions: &[(Range, usize)], + options: S::Options, + ) -> Result, usize)>> { + super::table_screened_regions( + &UnlimitedSearch(search, Allowance::default()), + data, + regions, + options, + ) + } + fn subtract_ranges(range: Range, removed: &[Range]) -> Vec> { + subtract_owned(range, removed, &Allowance::default()) + .unwrap() + .into_vec() + } + fn graft_tables( + specs: Vec, + tables: &[(Range, usize)], + member: usize, + ) -> Vec { + graft_owned(Buffer::from_vec(specs), tables, member) + .unwrap() + .into_vec() + } + + #[test] + fn search_reports_each_sample_pass_and_can_cancel_during_screening() { + use std::cell::RefCell; + struct CountingSearch(RefCell>); + impl FilterSearch for CountingSearch { + type Options = (); + fn screened_kinds(&self, _: &[u8]) -> Vec { + vec![FilterKind::Delta { channels: 1 }] + } + fn detects_x86(&self) -> bool { + false + } + fn max_delta_channels(&self) -> usize { + 1 + } + fn filtered_bytes(&self, data: &[u8], _: &[FilterSpec]) -> Result> { + Ok(data.to_vec()) + } + fn encode_plain( + &self, + data: &[u8], + _: (), + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + self.0.borrow_mut().push(data.len()); + // Short codec passes may report only their final position. + if progress.is_some_and(|report| !report(data.len())) { + return Err(crate::Error::Cancelled); + } + Ok(vec![0; 10]) + } + fn encode_filtered( + &self, + data: &[u8], + _: &[FilterSpec], + options: (), + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + self.encode_plain(data, options, progress) + } + } + let data = vec![0; SCREEN_SAMPLE_LEN * 2]; + let search = CountingSearch(RefCell::new(Vec::new())); + let mut events = Vec::new(); + let observed = choose_filter( + &search, + &data, + (), + Some(&mut |event| { + events.push(event); + true + }), + ) + .unwrap(); + let calls = search.0.borrow().clone(); + assert!(calls.len() >= 3); + assert!(calls[..calls.len() - 1] + .iter() + .all(|&len| len == SCREEN_SAMPLE_LEN)); + assert_eq!(calls.last(), Some(&data.len())); + assert_eq!( + events, + calls + .iter() + .flat_map(|&len| [EncodeProgress::PassStarted, EncodeProgress::Advanced(len)]) + .collect::>() + ); + assert_eq!(observed, choose_filter(&search, &data, (), None).unwrap()); + search.0.borrow_mut().clear(); + assert_eq!( + choose_filter( + &search, + &data, + (), + Some(&mut |event| matches!(event, EncodeProgress::PassStarted)) + ), + Err(crate::Error::Cancelled) + ); + assert_eq!(*search.0.borrow(), [SCREEN_SAMPLE_LEN]); + } + + use super::*; + use crate::codec::rar50::{ + encode_lz_member_with_options, encode_lz_member_with_options_and_progress, EncodeOptions, + Unpack50Encoder, + }; + + /// A stand-in format that measures candidates the way RAR 5 does. The + /// driver is generic, but it still has to be exercised against a real + /// encoder for the screens to mean anything. + #[derive(Clone, Copy)] + struct TestSearch { + cheap_screens: bool, + } + + impl FilterSearch for TestSearch { + type Options = EncodeOptions; + + fn screened_kinds(&self, _data: &[u8]) -> Vec { + vec![ + FilterKind::Arm, + FilterKind::Delta { channels: 1 }, + FilterKind::Delta { channels: 2 }, + FilterKind::Delta { channels: 3 }, + FilterKind::Delta { channels: 4 }, + ] + } + + fn max_delta_channels(&self) -> usize { + crate::codec::rar50::MAX_DELTA_CHANNELS + } + + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result> { + crate::codec::rar50::filtered_lz_member(data, filters) + .map(|(filtered, _)| filtered) + .map_err(crate::Error::from) + } + + fn screen_options(&self, options: EncodeOptions) -> EncodeOptions { + if self.cheap_screens { + EncodeOptions::new(4).with_max_match_distance(options.max_match_distance) + } else { + options + } + } + + fn encode_plain( + &self, + data: &[u8], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + match progress { + Some(progress) => { + encode_lz_member_with_options_and_progress(data, 0, options, progress) + } + None => encode_lz_member_with_options(data, 0, options), + } + .map_err(crate::Error::from) + } + + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: EncodeOptions, + _progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Unpack50Encoder::with_options(options) + .encode_member_with_filters(data, 0, filters) + .map_err(crate::Error::from) + } + } + + const FULL: TestSearch = TestSearch { + cheap_screens: false, + }; + + #[test] + fn tiny_scanner_regions_do_not_trigger_sample_encodes() { + let data = b"abcdefgh"; + let region = 0..data.len(); + let x86 = + x86_screened_regions(&FULL, data, std::slice::from_ref(®ion), options()).unwrap(); + assert!(x86.kept.is_empty()); + assert!(!x86.rejected_a_region); + let tables = table_screened_regions(&FULL, data, &[(0..data.len(), 1)], options()).unwrap(); + assert!(tables.is_empty()); + assert!(auto_delta_filter_range(data, 0).is_none()); + assert!(!search_applies(b"")); + } + + fn options() -> EncodeOptions { + EncodeOptions::new(64).with_max_match_distance(128 * 1024) + } + + /// Interleaved counters: RAR's delta reorders them into planes of near + /// constants, which is a huge win the byte histogram cannot see. + fn interleaved_counters() -> Vec { + let mut data = Vec::new(); + for index in 0..20_000u32 { + data.push(0xe8); + data.extend_from_slice(&index.to_le_bytes()); + data.extend_from_slice(b"\x55\x89\xe5"); + } + data + } + + fn incompressible(len: usize) -> Vec { + let mut state = 0x2545_f491_4f6c_dd1du64; + std::iter::repeat_with(|| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .take(len) + .collect() + } + + #[test] + fn the_screen_keeps_a_delta_width_that_pays_off() { + let kinds: Vec<_> = screen_kinds(&FULL, &interleaved_counters(), options()) + .unwrap() + .kinds + .into_iter() + .map(|screened| screened.kind) + .collect(); + assert!( + kinds.contains(&FilterKind::Delta { channels: 4 }), + "delta 4 turns this into planes of constants, so it has to survive: {kinds:?}" + ); + } + + #[test] + fn the_screen_rejects_filters_on_incompressible_data() { + assert!( + screen_kinds(&FULL, &incompressible(400_000), options()) + .unwrap() + .kinds + .is_empty(), + "nothing helps random bytes, and finding that out must not cost whole-member encodes" + ); + } + + /// A member that fits in one sample has no slice to approximate with, so + /// the screen encodes it the way the writer would and the search keeps every + /// number rather than paying for the winners twice. + #[test] + fn a_short_member_is_measured_by_the_screen_rather_than_encoded_twice() { + let data = interleaved_counters()[..100_000].to_vec(); + assert!(data.len() <= SCREEN_SAMPLE_LEN); + let screen = screen_kinds(&FULL, &data, options()).unwrap(); + + assert!(screen.plain.is_some(), "the unfiltered encode is reusable"); + assert_eq!( + screen.kinds.len(), + FULL.screened_kinds(&data).len(), + "a kind the screen has already encoded is worth keeping whether or \ + not it won by the margin" + ); + for screened in screen.kinds.iter() { + let measured = screened + .measured + .as_ref() + .unwrap_or_else(|| panic!("{:?} came back without its bytes", screened.kind)); + // Not just present: the same bytes the writer would emit for it. + // Anything else is a number the search cannot stand on. + let written = FULL + .encode_filtered(&data, &[FilterSpec::whole(screened.kind)], options(), None) + .unwrap(); + assert_eq!(*measured, written, "{:?}", screened.kind); + } + } + + /// A screen run at cheaper settings measured something the finalists will + /// not be measured at, so it cannot stand in for a real measurement even + /// when the sample happens to be the whole member. + #[test] + fn cheaper_screens_are_not_reused_as_measurements() { + let data = interleaved_counters()[..100_000].to_vec(); + let cheap = TestSearch { + cheap_screens: true, + }; + let screen = screen_kinds(&cheap, &data, options()).unwrap(); + assert!(screen.plain.is_none()); + assert!(screen + .kinds + .iter() + .all(|screened| screened.measured.is_none())); + } + + /// A member big enough that the screens have to sample it, with a filter in + /// it worth finding. The call targets here are absolute rather than + /// relative, so it is the delta filter that pays off on it, not the x86 + /// one; see [`calls_to_fixed_addresses`] for code shaped the way the x86 + /// filter wants. + fn x86_like(len: usize) -> Vec { + let mut state = 0x2545_f491u32; + let mut data = Vec::with_capacity(len); + while data.len() < len { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + if state.is_multiple_of(11) { + // A call to one of a few fixed addresses, which the filter turns + // into a handful of repeated relative offsets. + let target = 0x4000u32 + (state >> 28) * 0x400; + data.push(0xe8); + data.extend_from_slice(&target.to_le_bytes()); + } else { + data.extend_from_slice(&[0x48, 0x89, (state >> 16) as u8, 0xe5]); + } + } + data.truncate(len); + data + } + + /// The search has to end up with the filter when one plainly helps. + #[test] + fn the_screen_finds_a_filter_worth_having_on_a_sampled_member() { + let data = x86_like(SCREEN_SAMPLE_LEN * 4); + let (specs, packed) = choose_filter(&FULL, &data, options(), None).unwrap(); + let plain = FULL.encode_plain(&data, options(), None).unwrap(); + + assert!( + !specs.is_empty(), + "the search left the filter on the table: {} against {} unfiltered", + packed.len(), + plain.len() + ); + assert!(packed.len() < plain.len()); + } + + /// Calls to a handful of fixed addresses, in bodies that repeat. Unfiltered + /// the call breaks every repeat, because the displacement counts from wherever + /// the call sits; converted to an absolute address the bodies match each + /// other again. That is what the x86 filter is for, and it is why a stretch + /// of real code screens as a win. + fn calls_to_fixed_addresses(len: usize) -> Vec { + const BODIES: [[u8; 11]; 4] = [ + [ + 0x55, 0x48, 0x89, 0xe5, 0x48, 0x83, 0xec, 0x20, 0x89, 0x7d, 0xfc, + ], + [ + 0x48, 0x8b, 0x45, 0xf8, 0x48, 0x8b, 0x00, 0x48, 0x89, 0xc7, 0x90, + ], + [ + 0x8b, 0x45, 0xfc, 0x83, 0xc0, 0x01, 0x89, 0x45, 0xfc, 0x66, 0x90, + ], + [ + 0x48, 0x8d, 0x35, 0x00, 0x00, 0x00, 0x00, 0x31, 0xc0, 0x0f, 0x1f, + ], + ]; + let mut state = 0x2545_f491u32; + let mut data = Vec::with_capacity(len); + while data.len() < len { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + data.extend_from_slice(&BODIES[(state >> 28) as usize % BODIES.len()]); + // Far enough ahead that the address stays in the range the filter + // converts however the member is sliced. + let target = 0x0010_0000u32 + ((state >> 24) & 7) * 0x400; + let call_end = (data.len() + 5) as u32; + data.push(0xe8); + data.extend_from_slice(&target.wrapping_sub(call_end).to_le_bytes()); + } + data.truncate(len); + data + } + + /// Debug data: strings and symbol names, with enough stray call opcodes in + /// the numbers between them for the scanner to call it code. + /// + /// The numbers behind those opcodes repeat, so converting them by position + /// is the wrong way round: it turns a handful of repeated values into a + /// different one every time and costs the encoder the matches. That is why + /// filtering a whole unstripped binary makes it bigger. + fn debug_like(len: usize) -> Vec { + const WORDS: [&[u8]; 6] = [ + b"_ZN4llvm12FunctionPassE", + b"/usr/include/c++/14/bits/", + b"DW_AT_decl_file", + b"unsigned long long int", + b"__gnu_cxx::__normal_iterator", + b"DW_TAG_subprogram", + ]; + let mut state = 0x9e37_79b9u32; + let mut data = Vec::with_capacity(len); + while data.len() < len { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + if state.is_multiple_of(37) { + data.push(0xe8); + data.extend_from_slice( + &(0x0004_0000u32 + ((state >> 20) & 3) * 0x40).to_le_bytes(), + ); + } else { + data.extend_from_slice(WORDS[(state >> 28) as usize % WORDS.len()]); + data.push(0); + data.extend_from_slice(&state.rotate_left(7).to_le_bytes()); + } + } + data.truncate(len); + data + } + + /// An unstripped binary in miniature: a code section, then a debug section + /// several times its size that carries enough stray call opcodes to look + /// like code to the scanner. Returns the member and where the code ends. + fn code_then_debug() -> (Vec, usize) { + let code_len = 132 * 1024; + // Wider than the scanner's clustering gap, so the two stay separate + // regions rather than merging into one. + let gap = 48 * 1024; + let mut data = calls_to_fixed_addresses(code_len); + data.resize(code_len + gap, 0x5a); + data.extend_from_slice(&debug_like(220 * 1024)); + (data, code_len) + } + + /// The screen has to look past the biggest region. On an unstripped binary + /// the biggest thing the scanner calls code is the debug data, and sampling + /// only that declined the filter on every unstripped binary measured. + #[test] + fn the_x86_screen_looks_past_the_largest_region_to_the_code() { + let (data, code_len) = code_then_debug(); + let regions = x86_code_regions(&data); + let largest = regions + .iter() + .max_by_key(|range| range.len()) + .expect("the scanner has to find something"); + assert!( + largest.start >= code_len, + "this proves nothing unless the debug section is the largest region: {regions:?}" + ); + + let screen = x86_screened_regions(&FULL, &data, ®ions, options()).unwrap(); + assert_eq!( + screen.kept.len(), + 1, + "the code region is the only one worth filtering: {:?}", + screen.kept + ); + assert!( + screen.kept[0].end <= code_len + 1024, + "kept {:?}", + screen.kept + ); + assert!(screen.rejected_a_region); + } + + #[test] + fn x86_region_scan_releases_all_storage_on_each_admission_failure() { + use crate::codec::workspace::RefusingBudget; + + let data = calls_to_fixed_addresses(256 * 1024); + let baseline = RefusingBudget::new(usize::MAX); + let regions = super::x86_code_regions(&data, &baseline).unwrap(); + assert!(!regions.is_empty()); + drop(regions); + assert_eq!(baseline.used(), 0); + assert!(baseline.attempts() > 0); + + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + assert_eq!( + super::x86_code_regions(&data, &budget).unwrap_err(), + crate::Error::Cancelled, + "admission {index}" + ); + assert_eq!(budget.used(), 0, "admission {index}"); + } + } + + #[test] + fn x86_candidate_construction_releases_storage_on_each_admission_failure() { + use crate::codec::workspace::RefusingBudget; + + let run = |budget: &RefusingBudget| -> Result<()> { + let screen = X86Screen { + rejected_a_region: true, + kept: Buffer::collect(std::iter::once(0..3), budget)?, + jumps_cost_more: true, + }; + let finalists = super::x86_finalists(&[0; 100], &screen)?; + assert_eq!(finalists.len(), 2); + assert!(finalists.iter().all(|specs| specs.len() == 1)); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + assert_eq!( + run(&budget), + Err(crate::Error::Cancelled), + "admission {index}" + ); + assert_eq!(budget.used(), 0, "admission {index}"); + } + } + + #[test] + fn table_grafting_releases_storage_on_each_admission_failure() { + use crate::codec::workspace::RefusingBudget; + + let run = |budget: &RefusingBudget| -> Result<()> { + let specs = Buffer::collect([FilterSpec::whole(FilterKind::E8E9)], budget)?; + let grafted = super::graft_owned(specs, &[(20..40, 2)], 100)?; + assert_eq!(grafted.len(), 3); + assert_eq!(grafted[0].range, Some(0..20)); + assert_eq!(grafted[1].range, Some(20..40)); + assert_eq!(grafted[2].range, Some(40..100)); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + assert_eq!( + run(&budget), + Err(crate::Error::Cancelled), + "admission {index}" + ); + assert_eq!(budget.used(), 0, "admission {index}"); + } + } + + #[test] + fn candidate_search_preserves_codec_cancellation() { + struct CancelledCodec; + + impl FilterSearch for CancelledCodec { + type Options = (); + + fn screened_kinds(&self, _: &[u8]) -> Vec { + Vec::new() + } + + fn detects_x86(&self) -> bool { + false + } + + fn max_delta_channels(&self) -> usize { + 0 + } + + fn filtered_bytes(&self, _: &[u8], _: &[FilterSpec]) -> Result> { + unreachable!("no filters are proposed") + } + + fn encode_plain( + &self, + _: &[u8], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Err(crate::Error::Cancelled) + } + + fn encode_filtered( + &self, + _: &[u8], + _: &[FilterSpec], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + unreachable!("no filters are proposed") + } + } + + assert_eq!( + filter_candidates(&CancelledCodec, b"binary\0member", ()), + Err(crate::Error::Cancelled) + ); + } + + #[test] + fn bounded_delta_finalists_release_storage_on_each_admission_failure() { + use crate::codec::workspace::RefusingBudget; + + struct DeltaSearch(RefusingBudget); + impl OwnedSearch for DeltaSearch { + type Options = (); + type Memory = RefusingBudget; + + fn allowance(&self) -> &Self::Memory { + &self.0 + } + fn screened_kinds(&self, _: &[u8]) -> Result> { + Ok(Buffer::collect( + [1, 2, 3].map(|channels| FilterKind::Delta { channels }), + &self.0, + )?) + } + fn detects_x86(&self) -> bool { + false + } + fn max_delta_channels(&self) -> usize { + 0 + } + fn screen_options(&self, options: ()) { + options + } + fn filtered_bytes( + &self, + _: &[u8], + _: &[FilterSpec], + ) -> Result> { + unreachable!("the whole member is measured directly") + } + fn encode_plain( + &self, + _: &[u8], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Ok(Buffer::collect([0; 10], &self.0)?) + } + fn encode_filtered( + &self, + _: &[u8], + _: &[FilterSpec], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Ok(Buffer::collect([0], &self.0)?) + } + } + + let run = |budget: &RefusingBudget| -> Result<()> { + let finalists = super::finalists(&DeltaSearch(budget.clone()), &[0; 512], ())?; + assert_eq!(finalists.len(), 7); + assert!(finalists[1].0[0].range.is_none()); + assert!(finalists[2].0[0].range.is_some()); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + assert_eq!( + run(&budget), + Err(crate::Error::Cancelled), + "admission {index}" + ); + assert_eq!(budget.used(), 0, "admission {index}"); + } + } + + #[test] + fn x86_screen_releases_storage_on_each_admission_failure() { + use crate::codec::workspace::RefusingBudget; + use std::cell::Cell; + + struct ScreenSearch { + budget: RefusingBudget, + encodes: Cell, + } + impl OwnedSearch for ScreenSearch { + type Options = (); + type Memory = RefusingBudget; + + fn allowance(&self) -> &Self::Memory { + &self.budget + } + fn screened_kinds(&self, _: &[u8]) -> Result> { + Ok(Buffer::new(&self.budget)) + } + fn detects_x86(&self) -> bool { + true + } + fn max_delta_channels(&self) -> usize { + 0 + } + fn screen_options(&self, options: ()) { + options + } + fn filtered_bytes( + &self, + data: &[u8], + _: &[FilterSpec], + ) -> Result> { + Ok(Buffer::collect(data.iter().copied(), &self.budget)?) + } + fn encode_plain( + &self, + _: &[u8], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + let pass = self.encodes.get(); + self.encodes.set(pass + 1); + let size = [10, 5, 6][pass]; + Ok(Buffer::collect(std::iter::repeat_n(0, size), &self.budget)?) + } + fn encode_filtered( + &self, + _: &[u8], + _: &[FilterSpec], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + unreachable!("the x86 screen encodes filtered bytes as plain data") + } + } + + let run = |budget: &RefusingBudget| -> Result<()> { + let search = ScreenSearch { + budget: budget.clone(), + encodes: Cell::new(0), + }; + let region = 0..64; + let result = + super::x86_screened_regions(&search, &[0; 64], std::slice::from_ref(®ion), ())?; + assert_eq!(result.kept.len(), 1); + assert!(!result.jumps_cost_more); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + assert_eq!( + run(&budget), + Err(crate::Error::Cancelled), + "admission {index}" + ); + assert_eq!(budget.used(), 0, "admission {index}"); + } + } + + #[test] + fn table_only_candidate_releases_storage_on_each_admission_failure() { + use crate::codec::workspace::RefusingBudget; + + struct TableSearch(RefusingBudget); + impl OwnedSearch for TableSearch { + type Options = (); + type Memory = RefusingBudget; + + fn allowance(&self) -> &Self::Memory { + &self.0 + } + fn screened_kinds(&self, _: &[u8]) -> Result> { + Ok(Buffer::new(&self.0)) + } + fn detects_x86(&self) -> bool { + false + } + fn max_delta_channels(&self) -> usize { + MAX_TABLE_STRIDE + } + fn screen_options(&self, options: ()) { + options + } + fn filtered_bytes( + &self, + _: &[u8], + _: &[FilterSpec], + ) -> Result> { + Ok(Buffer::collect([0], &self.0)?) + } + fn encode_plain( + &self, + data: &[u8], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + let size = if data.len() == 1 { 1 } else { 10 }; + Ok(Buffer::collect(std::iter::repeat_n(0, size), &self.0)?) + } + fn encode_filtered( + &self, + _: &[u8], + _: &[FilterSpec], + _: (), + _: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + unreachable!("the candidate search only screens the table") + } + } + + let data = reloc_table(2048); + let run = |budget: &RefusingBudget| -> Result<()> { + let finalists = super::finalists(&TableSearch(budget.clone()), &data, ())?; + assert_eq!(finalists.len(), 2); + assert!(matches!( + &finalists[1].0[..], + [FilterSpec { + kind: FilterKind::Delta { channels: 24 }, + range: Some(_), + }] + )); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + assert_eq!( + run(&budget), + Err(crate::Error::Cancelled), + "admission {index}" + ); + assert_eq!(budget.used(), 0, "admission {index}"); + } + } + + /// The E8-only filter costs a whole-member encode to ask whether leaving + /// the jump opcodes alone packs better. Over twenty-four members it won + /// seven times and never by more than 0.21%, so it only earns that encode + /// when a sample says it might. + #[test] + fn the_jump_opcodes_are_priced_on_a_sample_before_they_earn_an_encode() { + let (data, _) = code_then_debug(); + let regions = x86_code_regions(&data); + let screen = x86_screened_regions(&FULL, &data, ®ions, options()).unwrap(); + let kinds: Vec<_> = x86_finalists(&data, &screen) + .iter() + .map(|specs| specs[0].kind) + .collect(); + + // Every call here goes to a fixed address, which is what the jump + // conversion is for, so this member is one where converting both pays. + assert!(!screen.jumps_cost_more, "{kinds:?}"); + assert_eq!( + kinds, + [FilterKind::E8E9], + "the E8-only candidate cost an encode nothing asked for" + ); + } + + #[test] + fn dense_x86_code_uses_a_whole_member_finalist() { + let data = calls_to_fixed_addresses(256 * 1024); + let regions = x86_code_regions(&data); + let screen = x86_screened_regions(&FULL, &data, ®ions, options()).unwrap(); + assert!(!screen.rejected_a_region, "{regions:?}"); + let finalists = x86_finalists(&data, &screen); + assert!(!finalists.is_empty()); + assert!(finalists + .iter() + .all(|specs| specs.iter().all(|spec| spec.range.is_none()))); + } + + #[test] + fn sparse_x86_code_prices_region_and_whole_member_finalists() { + let code = calls_to_fixed_addresses(32 * 1024); + let mut data = code.clone(); + data.resize(512 * 1024, 0); + let regions = x86_code_regions(&data); + let screen = x86_screened_regions(&FULL, &data, ®ions, options()).unwrap(); + assert_eq!(screen.kept.len(), 1, "{regions:?}"); + assert!(!screen.rejected_a_region); + let finalists = x86_finalists(&data, &screen); + assert!(finalists + .iter() + .any(|specs| specs.iter().all(|spec| spec.range.is_some()))); + assert!(finalists + .iter() + .any(|specs| specs.iter().all(|spec| spec.range.is_none()))); + } + + /// And the member as a whole has to come out smaller for it. + #[test] + fn the_search_filters_the_code_in_an_unstripped_binary() { + let (data, _) = code_then_debug(); + let plain = FULL.encode_plain(&data, options(), None).unwrap(); + let (specs, packed) = choose_filter(&FULL, &data, options(), None).unwrap(); + + assert!( + matches!( + specs.as_slice(), + [FilterSpec { + kind: FilterKind::E8E9 | FilterKind::E8, + range: Some(_) + }] + ), + "expected a ranged x86 filter, got {specs:?} at {} against {} unfiltered", + packed.len(), + plain.len() + ); + assert!(packed.len() * 100 < plain.len() * 97); + } + + /// A relocation table in miniature: 24-byte records of three 8-byte fields + /// that creep or repeat from one record to the next. What makes it a filter + /// case is that the creeping fields never repeat exactly, so the match + /// finder gets nothing, while their stride-24 differences are constants. + fn reloc_table(records: usize) -> Vec { + let mut data = Vec::with_capacity(records * 24); + for index in 0..records as u64 { + data.extend_from_slice(&(0x7f80_1234_0000 + index * 24).to_le_bytes()); + data.extend_from_slice(&0x0102_0304_0506_0708u64.to_le_bytes()); + data.extend_from_slice(&(0x4455_6677_0000 | ((index * 7) & 0xffff)).to_le_bytes()); + } + data + } + + #[test] + fn a_struct_table_scans_as_its_record_stride() { + let data = reloc_table(20_000); + let regions = delta_table_regions(&data, MAX_TABLE_STRIDE); + assert_eq!(regions.len(), 1, "{regions:?}"); + let (region, stride) = ®ions[0]; + assert_eq!(*stride, 24, "the record size is the shortest true period"); + assert!( + region.len() * 10 >= data.len() * 9, + "the region has to cover the table: {region:?} of {}", + data.len() + ); + } + + #[test] + fn a_table_without_code_gets_its_own_filter_candidate() { + let mut data = reloc_table(2048); + for byte in &mut data { + if matches!(*byte, 0xe8 | 0xe9) { + *byte = 0xe7; + } + } + assert!(x86_code_regions(&data).is_empty()); + let candidates = filter_candidates(&FULL, &data, options()).unwrap(); + assert!(candidates.iter().any(|specs| matches!( + specs.as_slice(), + [FilterSpec { + kind: FilterKind::Delta { channels: 24 }, + range: Some(_), + }] + ))); + } + + #[test] + fn adjacent_struct_tables_with_different_record_sizes_stay_separate() { + let mut data = reloc_table(4096); + let boundary = data.len(); + for index in 0..8192u64 { + data.extend_from_slice(&(0x7f80_1234_0000 + index * 8).to_le_bytes()); + } + let regions = delta_table_regions(&data, MAX_TABLE_STRIDE); + assert_eq!(regions.len(), 2, "{regions:?}"); + assert_eq!(regions[0].1, 24); + assert_eq!(regions[1].1, 8); + assert_eq!(regions[0].0.end, boundary); + assert_eq!(regions[1].0.start, boundary); + } + + #[test] + fn matching_table_strides_do_not_merge_across_a_non_table_gap() { + let mut data = reloc_table(4096); + let first_end = data.len(); + data.extend(incompressible(2 * TABLE_SCAN_WINDOW)); + let second_start = data.len(); + data.extend(reloc_table(4096)); + let regions = delta_table_regions(&data, MAX_TABLE_STRIDE); + assert_eq!(regions.len(), 2, "{regions:?}"); + assert_eq!(regions[0], (0..first_end, 24)); + assert_eq!(regions[1], (second_start..data.len(), 24)); + } + + #[test] + fn text_and_random_bytes_do_not_scan_as_tables() { + for data in [ + b"the quick brown fox jumps over the lazy dog ".repeat(6_000), + incompressible(256 * 1024), + ] { + assert_eq!( + delta_table_regions(&data, MAX_TABLE_STRIDE), + vec![], + "{} bytes", + data.len() + ); + } + } + + /// x86 instructions are variable length, so code has no byte period for the + /// scanner to lock onto. [`calls_to_fixed_addresses`] is no use for proving + /// that: it emits a fixed 16-byte unit, which really is a record array as + /// far as any stride test can tell. This mixes instruction lengths the way + /// a compiler does, so a false table here would be a false table on a real + /// `.text`. + fn variable_length_code(len: usize) -> Vec { + const INSTRUCTIONS: [&[u8]; 7] = [ + &[0x55], // push rbp + &[0x48, 0x89, 0xe5], // mov rbp, rsp + &[0x8b, 0x45, 0xfc], // mov eax, [rbp-4] + &[0x48, 0x83, 0xec, 0x20], // sub rsp, 32 + &[0x0f, 0xb6, 0x54, 0x18, 0x01], // movzx edx, [rax+rbx+1] + &[0x48, 0x8d, 0x35, 0x12, 0x00, 0x00, 0x00], // lea rsi, [rip+18] + &[0x66, 0x0f, 0x1f, 0x84, 0x00, 0, 0, 0, 0], // nop word [rax+rax] + ]; + let mut state = 0x2545_f491u32; + let mut data = Vec::with_capacity(len); + while data.len() < len { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + data.extend_from_slice(INSTRUCTIONS[(state >> 27) as usize % INSTRUCTIONS.len()]); + } + data.truncate(len); + data + } + + #[test] + fn real_shaped_code_does_not_scan_as_a_table() { + let data = variable_length_code(256 * 1024); + assert_eq!( + delta_table_regions(&data, MAX_TABLE_STRIDE), + vec![], + "variable-length instructions have no record stride to find" + ); + } + + /// And the property that actually matters: even where the scanner does fire + /// on code, nothing reaches the encoder unless a sample encode agrees. + #[test] + fn code_never_reaches_the_encoder_as_a_delta_candidate() { + for data in [ + variable_length_code(256 * 1024), + calls_to_fixed_addresses(256 * 1024), + ] { + let regions = delta_table_regions(&data, MAX_TABLE_STRIDE); + assert_eq!( + table_screened_regions(&FULL, &data, ®ions, options()).unwrap(), + vec![], + "the screen let a delta filter onto code: {regions:?}" + ); + } + } + + /// A format that cannot encode a wide delta must never be handed one. The + /// scan is bounded by what the writer can say, not only by the data. + #[test] + fn the_scan_stays_inside_what_the_format_can_encode() { + let data = reloc_table(20_000); + assert_eq!(delta_table_regions(&data, MAX_TABLE_STRIDE)[0].1, 24); + for (region, stride) in delta_table_regions(&data, 8) { + assert!(stride <= 8, "{region:?} asked for {stride} channels"); + } + assert_eq!(delta_table_regions(&data, 0), vec![]); + } + + /// Constant bytes repeat at every stride, so the scanner calls them a + /// table, but the encoder handles a run perfectly well on its own. The + /// sample encode is what stops a useless filter there. + #[test] + fn a_run_of_constant_bytes_screens_out_of_the_table_regions() { + let data = vec![0u8; 200 * 1024]; + let regions = delta_table_regions(&data, MAX_TABLE_STRIDE); + assert!(!regions.is_empty(), "the scanner sees repetition in a run"); + assert_eq!( + table_screened_regions(&FULL, &data, ®ions, options()).unwrap(), + vec![] + ); + } + + #[test] + fn subtracting_ranges_cuts_the_tables_out_of_the_code() { + assert_eq!( + subtract_ranges(0..1000, &[200..300, 600..700]), + vec![0..200, 300..600, 700..1000] + ); + assert_eq!( + subtract_ranges(100..200, &[0..250, 300..400]), + Vec::>::new() + ); + assert_eq!( + subtract_ranges(100..200, &[0..50, 300..400]), + vec![100..200] + ); + assert_eq!( + subtract_ranges(100..200, &[150..400, 500..600]), + vec![100..150] + ); + } + + /// The point of the whole scanner: a binary with a table in it comes out + /// with delta at the record stride over the table, and only there. + #[test] + fn the_search_deltas_the_table_inside_a_binary() { + let code_len = 132 * 1024; + let mut data = calls_to_fixed_addresses(code_len); + let table_start = data.len(); + data.extend_from_slice(&reloc_table(20_000)); + + let plain = FULL.encode_plain(&data, options(), None).unwrap(); + let (specs, packed) = choose_filter(&FULL, &data, options(), None).unwrap(); + + let delta = specs + .iter() + .find(|spec| matches!(spec.kind, FilterKind::Delta { channels: 24 })) + .unwrap_or_else(|| panic!("no stride-24 delta among {specs:?}")); + let range = delta.range.clone().expect("the table filter is ranged"); + assert!( + range.start >= table_start.saturating_sub(TABLE_SCAN_WINDOW) + && range.start < table_start + TABLE_SCAN_WINDOW, + "the filter starts at the table, not the code: {range:?} against {table_start}" + ); + for spec in &specs { + let other = spec.range.clone().expect("every spec here is ranged"); + assert!( + spec == delta || other.end <= range.start || other.start >= range.end, + "{specs:?} overlap" + ); + } + assert!( + packed.len() * 100 < plain.len() * 90, + "the table filter is worth a lot more than this: {} against {}", + packed.len(), + plain.len() + ); + } + + #[test] + fn grafting_cuts_the_tables_out_of_whatever_the_specs_covered() { + let tables = [(200..300, 24), (600..700, 8)]; + assert_eq!( + graft_tables(vec![FilterSpec::whole(FilterKind::E8E9)], &tables, 1000), + vec![ + FilterSpec::range(FilterKind::E8E9, 0..200), + FilterSpec::range(FilterKind::Delta { channels: 24 }, 200..300), + FilterSpec::range(FilterKind::E8E9, 300..600), + FilterSpec::range(FilterKind::Delta { channels: 8 }, 600..700), + FilterSpec::range(FilterKind::E8E9, 700..1000), + ] + ); + assert_eq!( + graft_tables(Vec::new(), &tables[..1], 1000), + vec![FilterSpec::range( + FilterKind::Delta { channels: 24 }, + 200..300 + )] + ); + let code_only = vec![FilterSpec::range(FilterKind::E8, 0..100)]; + assert_eq!(graft_tables(code_only.clone(), &[], 1000), code_only); + } + + /// The member this change exists for: an unstripped binary with a + /// relocation table in it. The bundled candidate this replaces lost to the + /// x86-only finalist here, because it carried the tables and the trimmed + /// code coverage as one bet, and the table went unfiltered with it. + #[test] + fn the_winner_filters_the_code_and_deltas_the_table_in_one_member() { + let (mut data, code_len) = code_then_debug(); + let table_start = data.len(); + data.extend_from_slice(&reloc_table(20_000)); + + let (specs, _) = choose_filter(&FULL, &data, options(), None).unwrap(); + let filters_the_code = specs.iter().any(|spec| { + matches!(spec.kind, FilterKind::E8 | FilterKind::E8E9) + && spec + .range + .as_ref() + .is_some_and(|range| range.start < code_len) + }); + let deltas_the_table = specs.iter().any(|spec| { + matches!(spec.kind, FilterKind::Delta { channels: 24 }) + && spec + .range + .as_ref() + .is_some_and(|range| range.start >= table_start - TABLE_SCAN_WINDOW) + }); + assert!( + filters_the_code && deltas_the_table, + "one of the two regions went unfiltered: {specs:?}" + ); + } + + #[test] + fn the_search_never_loses_to_no_filter() { + for data in [ + interleaved_counters(), + b"the quick brown fox ".repeat(20_000), + (0..300_000u32).map(|index| (index / 3) as u8).collect(), + incompressible(200_000), + ] { + let plain = FULL.encode_plain(&data, options(), None).unwrap(); + let (specs, packed) = choose_filter(&FULL, &data, options(), None).unwrap(); + assert!( + packed.len() <= plain.len(), + "{specs:?} came out at {} against {} unfiltered", + packed.len(), + plain.len() + ); + } + } + + #[test] + fn the_screen_sample_sits_in_the_middle_and_keeps_delta_planes_aligned() { + let data = vec![0u8; 5 * SCREEN_SAMPLE_LEN + 7]; + let sample = screen_sample(&data); + assert_eq!(sample.len(), SCREEN_SAMPLE_LEN); + let start = sample.as_ptr() as usize - data.as_ptr() as usize; + assert_eq!(start % SCREEN_SAMPLE_ALIGNMENT, 0); + assert!(start > 0 && start + sample.len() < data.len()); + } + + #[test] + fn a_ranged_delta_filter_skips_container_edges_and_aligns_planes() { + let data = vec![0u8; 512]; + let range = auto_delta_filter_range(&data, 3).unwrap(); + + assert!(range.start >= AUTO_DELTA_EDGE_SKIP); + assert!(range.end <= data.len() - AUTO_DELTA_EDGE_SKIP); + assert_eq!(range.start % 3, 0); + assert_eq!((range.end - range.start) % 3, 0); + assert!(auto_delta_filter_range(&data[..80], 3).is_none()); + } + + /// Two ranges describing one region must come out as one range. Dropping + /// the second instead, which one copy of this used to do, loses coverage of + /// everything past the overlap. + #[test] + fn overlapping_ranges_merge_rather_than_drop() { + assert_eq!( + disjoint_filter_ranges(vec![0..100, 50..200, 400..500]), + vec![0..200, 400..500] + ); + assert_eq!(disjoint_filter_ranges(vec![0..100, 100..200]), vec![0..200]); + } +} diff --git a/arca-rar/vendor/rars/src/io_util.rs b/arca-rar/vendor/rars/src/io_util.rs new file mode 100644 index 00000000..80950d83 --- /dev/null +++ b/arca-rar/vendor/rars/src/io_util.rs @@ -0,0 +1,33 @@ +use crate::{Error, Result}; +use std::io::{Read, Seek, SeekFrom}; + +pub(crate) fn read_exact_at( + file: &mut (impl Read + Seek), + offset: usize, + len: usize, +) -> Result> { + file.seek(SeekFrom::Start(offset as u64))?; + let mut data = vec![0; len]; + file.read_exact(&mut data)?; + Ok(data) +} + +pub(crate) fn read_u16(input: &[u8], offset: usize) -> Result { + let end = offset.checked_add(2).ok_or(Error::TooShort)?; + let bytes = input.get(offset..end).ok_or(Error::TooShort)?; + Ok(u16::from_le_bytes([bytes[0], bytes[1]])) +} + +pub(crate) fn read_u32(input: &[u8], offset: usize) -> Result { + let end = offset.checked_add(4).ok_or(Error::TooShort)?; + let bytes = input.get(offset..end).ok_or(Error::TooShort)?; + Ok(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) +} + +#[cfg(feature = "encryption")] +pub(crate) fn align16(value: usize, overflow_message: &'static str) -> Result { + value + .checked_add(15) + .map(|value| value & !15) + .ok_or(Error::InvalidHeader(overflow_message)) +} diff --git a/arca-rar/vendor/rars/src/lib.rs b/arca-rar/vendor/rars/src/lib.rs new file mode 100644 index 00000000..c3efb74b --- /dev/null +++ b/arca-rar/vendor/rars/src/lib.rs @@ -0,0 +1,5822 @@ +//! High-level RAR archive API. +//! +//! This crate is the supported public Rust API for `rars`. It is the facade +//! over the version-specific format modules, detects archive families, exposes +//! common member metadata, and streams extraction or recovery output to +//! caller-provided writers without requiring callers to buffer whole archives +//! in memory. New Rust users should depend on this crate rather than the +//! lower-level `rars-*` implementation crates, which ended at 0.3.x. +//! +//! All capabilities are enabled by default. Set `default-features = false` for +//! sequential parsing and decoding without encoders, recovery algorithms, crypto +//! dependencies or Rayon. Select `write`, `recovery`, `encryption` and `parallel` +//! independently as needed. Encrypted writing needs `write` and `encryption`; +//! writing recovery records needs `write` and `recovery`. Recovery repair does +//! not require `write`. Unavailable encrypted or recovery operations report +//! [`Error::FeatureDisabled`], while metadata remains readable where the format +//! permits it. Buffered extraction falls back to sequential execution without +//! `parallel`, and on bare WebAssembly. + +/// Where the unit tests write their files. Shared with the integration tests +/// by path rather than by API. +#[cfg(test)] +#[path = "../tests/support/scratch.rs"] +mod scratch; + +#[cfg(test)] +mod test_features; + +#[cfg(all(test, feature = "write", feature = "encryption"))] +#[path = "../tests/support/read_errors.rs"] +mod read_errors; + +mod atomic64; +#[cfg(feature = "write")] +pub mod builder; +#[doc(hidden)] +pub mod codec; +pub mod crc32; +#[doc(hidden)] +pub mod crypto; +pub mod detect; +#[cfg(any(feature = "write", all(feature = "recovery", feature = "encryption")))] +mod entropy; +pub mod error; +pub mod features; +pub mod filename; +pub mod filter; +#[cfg(feature = "write")] +mod filter_search; +mod io_util; +mod member; +pub use member::{ArchiveIndex, ArchiveMemberRef, ArchiveMemberRefs}; +mod member_read; +pub use member_read::{read_volume_members_at, read_volume_members_at_with_options}; +mod output_limit; +mod parallel; +mod parse_budget; +#[cfg(any(feature = "write", feature = "recovery"))] +mod pending_archive; +#[cfg(any(feature = "write", feature = "recovery"))] +mod progress; +mod read_control; +pub use read_control::ReadCancellation; +mod extraction_control; +mod file_times; +pub mod rar13; +pub mod rar15_40; +pub mod rar50; +mod reader_scratch; +#[doc(hidden)] +pub mod recovery; +#[cfg(feature = "write")] +mod rewrite; +#[cfg(feature = "write")] +mod rewrite_staging; +#[cfg(feature = "write")] +pub use rewrite_staging::RewriteStaging; +mod source; +#[cfg(feature = "write")] +mod streaming; +mod temp_file; +pub mod timestamp; +pub use file_times::{FileTimes, FileTimestamp}; +mod tzif; +pub mod version; +mod volume_extract; +#[cfg(feature = "write")] +pub mod write_plan; +#[cfg(feature = "write")] +mod write_progress; +mod writer_option; +pub use writer_option::WriterOption; +#[cfg(feature = "write")] +mod write_stream; +#[cfg(feature = "write")] +mod x86_filter_scan; + +#[cfg(feature = "write")] +pub use builder::Builder; +pub use detect::{detect_archive_family, find_archive_start, ArchiveSignature, SFX_SCAN_LIMIT}; +pub use error::{Error, ErrorKind, Result}; +pub use extraction_control::{ExtractionDecision, ExtractionErrorAction, ExtractionOutcome}; +pub use features::{Feature, FeatureSet}; +pub use filename::{entry_relative_path, validate_entry_name}; +pub use filter::{ + formats_supporting_filter, FilterKind, FilterPolicy, FilterSpec, UnsupportedFilterKind, +}; +#[cfg(any(feature = "write", feature = "recovery"))] +pub use progress::{WriteOperation, WriteProgress, WriteProgressEvent}; +pub use reader_scratch::Rar50Scratch; +use std::io::{Read, Write}; +use std::path::Path; +#[cfg(feature = "write")] +pub use streaming::output::{WriterOutput, WriterVolumes}; +#[cfg(feature = "write")] +pub use streaming::{ + EntryReader, EntrySource, WriteCancellation, WriterResources, DEFAULT_WRITER_MEMORY_LIMIT, +}; +pub use timestamp::StoredTimestamp; +pub use version::{ArchiveFamily, ArchiveVersion}; +#[cfg(feature = "write")] +pub use write_plan::{formats_supporting, supported_features, supports, MemberCoding, PlanShape}; + +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +/// Options used while parsing archives, extracting members, decoding comments or repairing. +pub struct ArchiveReadOptions<'a> { + /// Filename interpretation for application display/path adapters. Parsing and + /// core extraction callbacks retain original names; use `filename::decoded_name`. + pub legacy_name_encoding: Option, + /// Password bytes used for encrypted headers or payloads. + pub password: Option<&'a [u8]>, + /// Cooperative cancellation for this parsing, extraction or repair call. Parsing + /// does not retain the token for later extraction. A cancelled token stays + /// cancelled; use a new token for a new operation. Partial output may remain. + /// Blocked caller I/O and indivisible library work cannot be preempted. + pub cancellation: Option<&'a ReadCancellation>, + /// Inclusive top-level header count for one physical archive parse. + /// Counts main, encryption, file/directory, service, unknown and end headers; + /// standalone signatures/markers and nested records are not separate headers. + /// None preserves defaults; zero refuses even an empty archive's main header. + /// Each parsing call (including each independently parsed volume) starts fresh. + /// Extraction does not apply this policy retroactively. + pub max_header_count: Option, + /// Inclusive cumulative plaintext header bytes for one physical archive parse. + /// Includes CRC/size fields, names and extras; nested records count once in + /// their enclosing header. Excludes payloads, SFX and standalone signatures, + /// and encryption salt/IV/padding. RAR1.3's embedded signature counts as part + /// of its main header. Encrypted sizes require first-block decryption. + /// + /// Admission precedes full-header allocation; bounded prefixes and encryption + /// framing can consume additional space. This is not a total RAM/CPU limit: + /// source copies, metadata overhead and key derivation are outside the quota. + /// None preserves defaults; zero refuses the main header. Resets per parse, + /// not across a caller's separately parsed volume set. No partial Archive is + /// returned on refusal. Existing end-record and tolerant-tail handling stays. + pub max_header_bytes: Option, + /// Optional RAR 5 whole-member buffered decode limit, including logical + /// members split across volumes. This does not bound decoder dictionaries. + /// + /// Compressed members above this limit use the streaming path. Without + /// `rar50_scratch`, filtered streams return a typed buffered-decode-limit + /// error. With scratch enabled, member bytes and filter records live on disk + /// and each filter uses the scratch policy's separate workspace limit. + pub rar50_buffered_decode_limit: Option, + /// Optional disk-backed decoding above the whole-member buffering threshold. + /// Uses separate scratch-disk and filter-workspace quotas; it does not bound + /// dictionaries or aggregate process RAM. Scratch-backed members publish only + /// after integrity verification and parallel entry points run sequentially. + pub rar50_scratch: Option<&'a Rar50Scratch>, + /// Inclusive ceiling on a compressed RAR5/7 member's declared dictionary size. + /// `None` leaves dictionary sizes unrestricted; zero rejects compressed members. + /// Stored entries, directories and redirections are exempt. This is not a + /// total-memory budget: history copies, buffered output and parallel jobs + /// consume additional memory. + /// + /// Supply this option to extraction, including volume extraction. Parsing + /// does not retain it. Options-aware comment decoding also applies it. + /// Legacy formats, recovery helpers and direct + /// codec calls are outside its scope. Earlier members may already be emitted + /// when a later member exceeds the limit; its output callback is not opened. + pub rar50_dictionary_size_limit: Option, + /// Inclusive logical member output ceiling for every archive family, independent of RAM usage. + /// None preserves defaults; zero permits empty output. Apply to extraction, + /// not parsing. Known oversized members and unsupported unknown-size members + /// are refused before opening output. Runtime failures can leave partial output. + /// The ceiling resets per logical member, not per volume fragment. Discarding + /// a solid member's bytes does not exempt it; retries and history copies are + /// not counted twice. Direct codecs, password-only default wrappers and + /// recovery helpers are outside this explicit policy. Options-aware comment + /// decoding treats the archive comment as one logical member. + pub max_member_output_bytes: Option, + /// Inclusive total logical output ceiling for one extraction call, across + /// all members and volumes. Counts bytes accepted by output writers, including + /// discarded solid contents; retries and history copies do not count twice. + /// None preserves defaults; zero permits empty output. Known sizes are + /// admitted before opening output; unknown-size logical members are refused. + /// Errors can leave earlier output and a failing member's prefix. + /// Short writes charge only accepted bytes; refused chunks need not fill + /// the remaining allowance. A per-member refusal takes precedence when both + /// output ceilings reject the same admission or write. Separate extraction + /// calls, including calls for nested archives, do not share this budget. + /// + /// Configuring this option makes parallel entry points extract sequentially + /// for deterministic admission and accounting. This can reduce throughput. + /// It is not a CPU/RAM budget: buffered decoding can precede the output guard. + /// Parsing does not retain policy; password-only wrappers, direct codecs and + /// recovery helpers keep defaults. Each extraction starts a new budget. + /// Options-aware comment decoding starts a separate single-comment budget. + pub max_total_output_bytes: Option, + /// Aggregate capacity ceiling for reader-owned payload workspace in one + /// extraction or options-aware comment call, across all members and volumes. + /// Includes decoder/model state, packed and cipher staging, filter scratch, + /// checkpoints, split cursors and queued parallel results. None is unlimited. + /// Capacity growth charges old and replacement allocations while both live. + /// Parsed archive/source storage, metadata, caller sinks and collected final + /// output, allocator overhead and executor/control storage are excluded. + /// This is not a process RAM limit. Parsing and direct codecs do not apply it. + /// Parallel jobs receive fixed allowances before dispatch; a worker cannot + /// borrow a sibling's spare. Insufficient allowance returns ResourceLimit; + /// increase the limit or use sequential extraction. Partial output can remain. + pub max_reader_workspace_bytes: Option, +} + +impl<'a> ArchiveReadOptions<'a> { + /// Uses a shared cancellation signal without retaining policy in the archive. + pub fn with_cancellation(mut self, token: &'a ReadCancellation) -> Self { + self.cancellation = Some(token); + self + } + + pub(crate) fn check_cancelled(&self) -> Result<()> { + if self + .cancellation + .is_some_and(ReadCancellation::is_cancelled) + { + return Err(Error::Cancelled); + } + Ok(()) + } + /// Creates read options without a password. + pub fn new() -> Self { + Self::default() + } + + /// Sets the top-level header ceiling for each physical archive parse. + pub fn with_max_header_count(mut self, limit: u64) -> Self { + self.max_header_count = Some(limit); + self + } + + /// Sets the cumulative plaintext-header byte ceiling for each parse. + pub fn with_max_header_bytes(mut self, limit: u64) -> Self { + self.max_header_bytes = Some(limit); + self + } + + /// Creates read options with a password. + pub fn with_password(password: &'a [u8]) -> Self { + Self { + password: Some(password), + ..Self::default() + } + } + + /// Creates read options with an optional password. + pub fn with_optional_password(password: Option<&'a [u8]>) -> Self { + Self { + password, + ..Self::default() + } + } + + /// Sets the aggregate reader workspace capacity ceiling for each call. + pub fn with_max_reader_workspace_bytes(mut self, limit: u64) -> Self { + self.max_reader_workspace_bytes = Some(limit); + self + } + + /// Sets the logical member output ceiling. + pub fn with_max_member_output_bytes(mut self, limit: u64) -> Self { + self.max_member_output_bytes = Some(limit); + self + } + + /// Sets the total logical output ceiling and selects sequential extraction. + pub fn with_max_total_output_bytes(mut self, limit: u64) -> Self { + self.max_total_output_bytes = Some(limit); + self + } + + /// Sets the RAR5/7 declared dictionary-size ceiling for member extraction. + pub fn with_rar50_dictionary_size_limit(mut self, limit: u64) -> Self { + self.rar50_dictionary_size_limit = Some(limit); + self + } + + /// Sets the RAR 5 whole-member buffered decode limit. + pub fn with_rar50_buffered_decode_limit(mut self, limit: u64) -> Self { + self.rar50_buffered_decode_limit = Some(limit); + self + } + + /// Enables caller-configured disk scratch for large RAR5/7 members. + pub fn with_rar50_scratch(mut self, scratch: &'a Rar50Scratch) -> Self { + self.rar50_scratch = Some(scratch); + self + } +} + +#[derive(Debug, Clone)] +#[non_exhaustive] +/// A parsed RAR archive, preserving the concrete archive family. +pub enum Archive { + /// RAR 1.3/1.4 archive. + Rar13(rar13::Archive), + /// RAR 1.5 through RAR 4.x archive. + Rar15To40(rar15_40::Archive), + /// RAR 5.0 or later archive, including RAR 7 archives. + Rar50Plus(rar50::Archive), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +/// Metadata supplied to extraction and verification callbacks. +pub struct ExtractedEntryMeta { + /// The format supplied Unicode; legacy encoding overrides must not apply. + pub name_is_unicode: bool, + /// Raw entry name bytes as stored by the archive family. + pub name: Vec, + /// Stored modification time: DOS/FAT for legacy RAR, Unix seconds for RAR5. + /// `None` means absent; `Some(0)` is a valid RAR5 Unix epoch timestamp. + pub file_time: Option, + /// File attributes widened to a common integer type, exactly as stored. + pub file_attr: u64, + /// How to read `file_attr`, from the host OS the entry records. + pub attr_source: AttrSource, + /// Detail `file_time` is too coarse to hold, when the archive carries it. + pub mtime_refinement: Option, + /// Whether the entry is a directory. + pub is_directory: bool, +} + +impl ExtractedEntryMeta { + /// Creates common metadata for extraction callbacks. + /// Pass `None` for an absent timestamp; a numeric value means it is present. + pub fn new( + name: Vec, + file_time: impl Into>, + file_attr: u64, + is_directory: bool, + ) -> Self { + Self { + name_is_unicode: false, + name, + file_time: file_time.into(), + file_attr, + attr_source: AttrSource::Unknown, + mtime_refinement: None, + is_directory, + } + } + + /// Records how `file_attr` should be read. + #[must_use] + pub fn with_attr_source(mut self, attr_source: AttrSource) -> Self { + self.attr_source = attr_source; + self + } + + /// Records detail `file_time` is too coarse to hold. + #[must_use] + pub fn with_mtime_refinement(mut self, refinement: Option) -> Self { + self.mtime_refinement = refinement; + self + } + + /// Raw entry name bytes as stored by the archive family. + pub fn name_bytes(&self) -> &[u8] { + &self.name + } + + /// Returns the entry name with invalid UTF-8 replaced for display only. + /// + /// Use [`Self::name_bytes`] when exact archive bytes matter. + pub fn name_lossy(&self) -> String { + String::from_utf8_lossy(&self.name).into_owned() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +/// Common member view plus family-specific detail. +pub struct ArchiveMember { + /// Metadata shared across archive families. + pub meta: ArchiveMemberMeta, + /// Extra metadata that is meaningful only for one archive family. + pub detail: ArchiveMemberDetail, +} + +impl ArchiveMember { + /// Whether this member declares a dependency on preceding decoded history. + /// RAR 1.3 has no member-level solid flag; consult its archive main header. + pub fn is_solid(&self) -> bool { + match &self.detail { + ArchiveMemberDetail::Rar15To40 { solid, .. } => *solid, + ArchiveMemberDetail::Rar50Plus { + compression_info, .. + } => compression_info & 0x40 != 0, + ArchiveMemberDetail::Rar13 { .. } => false, + } + } + + /// Name interpretation for display or destination adapters. Stored bytes and + /// rewrite identity remain unchanged; Unicode metadata takes precedence. + pub fn decoded_name( + &self, + encoding: Option, + ) -> Result> { + filename::decoded_name(&self.meta.name, self.name_is_unicode(), encoding) + } + + /// Whether the format supplied Unicode rather than unspecified legacy bytes. + pub fn name_is_unicode(&self) -> bool { + match &self.detail { + ArchiveMemberDetail::Rar13 { .. } => false, + ArchiveMemberDetail::Rar15To40 { unicode_name, .. } => unicode_name.is_some(), + _ => true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +/// Family-independent metadata for a file-like archive member. +pub struct ArchiveMemberMeta { + /// Archive family that produced this member. + pub family: ArchiveFamily, + /// Raw entry name bytes as stored by the archive. + pub name: Vec, + /// Packed payload size in bytes. + pub packed_size: u64, + /// Unpacked file size in bytes. + pub unpacked_size: u64, + /// DOS local wall-clock fields for RAR 1.3-4.x, or Unix seconds for RAR5. + /// RAR5 includes the extended modification-time fallback; `None` is distinct + /// from an explicitly stored Unix epoch (`Some(0)`). + pub file_time: Option, + /// Odd-second and subsecond detail, kept separate from the raw time field. + pub mtime_refinement: Option, + /// File attributes widened to a common integer type. + pub file_attr: u64, + /// Host OS discriminator when present in the archive format. + pub host_os: Option, + /// Whether the member is a directory. + pub is_directory: bool, + /// Whether the member carries a RAR5 link or other redirection record. + pub is_redirection: bool, + /// Whether the member payload is encrypted. + pub is_encrypted: bool, + /// Whether the member payload is stored without compression. + pub is_stored: bool, + /// Whether the member continues from a previous volume. + pub is_split_before: bool, + /// Whether the member continues into the next volume. + pub is_split_after: bool, +} + +impl ArchiveMemberMeta { + /// Stored modification time with its encoding attached, before refinements. + /// + /// Legacy DOS values remain local wall-clock fields; RAR5 values are Unix + /// seconds, including the existing extended-time fallback. This accessor + /// does not validate or reinterpret the raw value. Absence stays `None`, + /// and odd-second/subsecond detail remains in `mtime_refinement`. + pub fn stored_modification_time(&self) -> Option { + self.file_time + .map(|raw| StoredTimestamp::from_family(self.family, raw)) + } + + /// Modification time as an instant. Legacy DOS values use the same local + /// zone and refinement policy as CLI extraction; RAR5 values are absolute. + pub fn modification_time(&self) -> Option { + let raw = self.file_time?; + if self.family == ArchiveFamily::Rar50Plus { + let nanos = self.mtime_refinement.map_or(0, |detail| detail.nanoseconds); + return std::time::UNIX_EPOCH + .checked_add(std::time::Duration::new(u64::from(raw), nanos)); + } + timestamp::extracted_system_time(self.family, Some(raw), self.mtime_refinement) + } + + /// How to interpret this member's attributes, using the archive family's + /// host numbering and the same compatibility rules as extraction. + pub fn attr_source(&self) -> AttrSource { + match self.family { + ArchiveFamily::Rar13 => AttrSource::Dos, + ArchiveFamily::Rar15To40 => self + .host_os + .and_then(|host| u8::try_from(host).ok()) + .map(AttrSource::rar15_40) + .unwrap_or_default(), + ArchiveFamily::Rar50Plus => self.host_os.map(AttrSource::rar50).unwrap_or_default(), + } + } + + /// Raw member name bytes as stored by the archive family. + pub fn name_bytes(&self) -> &[u8] { + &self.name + } + + /// Returns the member name with invalid UTF-8 replaced for display only. + /// + /// Use [`Self::name_bytes`] when exact archive bytes matter. + pub fn name_lossy(&self) -> String { + String::from_utf8_lossy(&self.name).into_owned() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +/// Family-specific member metadata. +pub enum ArchiveMemberDetail { + /// RAR 1.3/1.4 member fields. + #[non_exhaustive] + Rar13 { + /// Compression method byte from the file header. + method: u8, + /// Minimum unpacker version byte from the file header. + unpack_version: u8, + /// Legacy 16-bit file checksum. + file_checksum: u16, + /// Whether the member carries a file-comment extension. + has_file_comment: bool, + }, + /// RAR 1.5 through RAR 4.x member fields. + #[non_exhaustive] + Rar15To40 { + /// Compression method byte from the file header. + method: u8, + /// Minimum unpacker version byte from the file header. + unpack_version: u8, + /// Stored CRC-32 of the unpacked data. + crc32: u32, + /// Whether this member participates in a solid stream. + solid: bool, + /// Original Unicode wire name, including the legacy fallback. + unicode_name: Option>, + /// Raw legacy extended timestamp record. + extended_times: Vec, + /// Per-file salt when file encryption is used. + salt: Option<[u8; 8]>, + /// Whether the member carries a file-comment extension. + has_file_comment: bool, + }, + /// RAR 5.0 and later member fields. + #[non_exhaustive] + Rar50Plus { + /// Raw compression-info field from the RAR5 file header. + compression_info: u64, + /// Stored CRC-32 when present. + crc32: Option, + /// Complete supported RAR5 high-precision file times. + file_times: Option, + /// Redirection metadata, including the raw target name. + redirection: Option, + /// Strong file hash when present. + hash: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +/// Strong hash metadata attached to an archive member. +pub enum ArchiveMemberHash { + /// RAR5 BLAKE2sp file hash. + Blake2sp([u8; 32]), + /// Unknown hash record retained for inspection. + Other { hash_type: u64, data: Vec }, +} + +#[derive(Debug, Clone)] +#[non_exhaustive] +/// Lazy iterator returned by [`Archive::members`]. +pub struct ArchiveMembers<'a> { + inner: ArchiveMemberRefs<'a>, +} + +impl Iterator for ArchiveMembers<'_> { + type Item = ArchiveMember; + + fn next(&mut self) -> Option { + self.inner.next().map(|member| member.to_owned()) + } + + fn nth(&mut self, n: usize) -> Option { + self.inner.nth(n).map(|member| member.to_owned()) + } +} + +/// A `Write` that appends into a buffer someone else holds, for the extraction +/// callbacks that hand their writer away and need the bytes back. +struct SharedBuffer(std::sync::Arc>>>); + +impl SharedBuffer { + fn lock(&self) -> std::sync::MutexGuard<'_, Option>> { + // Only private buffer operations run under this lock. A panic while + // appending unwinds extraction before its owner can read the result; + // no callback runs with the guard held or can resume that extraction. + self.0.lock().expect("member buffer mutex is not poisoned") + } +} + +impl Write for SharedBuffer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.lock() + .get_or_insert_with(Vec::new) + .extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl Archive { + /// Returns the detected archive family. + pub fn family(&self) -> ArchiveFamily { + match self { + Self::Rar13(_) => ArchiveFamily::Rar13, + Self::Rar15To40(_) => ArchiveFamily::Rar15To40, + Self::Rar50Plus(_) => ArchiveFamily::Rar50Plus, + } + } + + /// Returns the byte offset where the RAR archive begins after any SFX stub. + pub fn sfx_offset(&self) -> usize { + match self { + Self::Rar13(archive) => archive.sfx_offset, + Self::Rar15To40(archive) => archive.sfx_offset, + Self::Rar50Plus(archive) => archive.sfx_offset, + } + } + + /// Iterates over file-like members using a common cross-version metadata view. + pub fn members(&self) -> ArchiveMembers<'_> { + ArchiveMembers { + inner: self.member_refs(), + } + } + + /// Traverses member headers without allocating names or copying metadata. + /// The borrow prevents mutation of the archive while these views are in use. + pub fn member_refs(&self) -> ArchiveMemberRefs<'_> { + ArchiveMemberRefs::new(self) + } + + /// Builds an optional zero-copy metadata index for repeated lookup. + /// The archive cannot be mutated while this index remains in use. + /// + /// ```compile_fail + /// # fn example(archive: &mut rars::Archive) { + /// let index = archive.index(); + /// if let rars::Archive::Rar13(legacy) = archive { + /// legacy.entries.clear(); + /// } + /// println!("{}", index.len()); + /// # } + /// ``` + pub fn index(&self) -> ArchiveIndex<'_> { + ArchiveIndex::new(self) + } + + /// Streams extracted entries to caller-provided writers. + /// + /// Output is not transactional: integrity checks can fail after all bytes of + /// a member have been written, and decode/filter or sink failures can leave + /// a partial member. Earlier entries and callback side effects are not rolled + /// back. The amount written before an error depends on the decoding path; + /// even a complete payload must not be treated as verified on failure. + /// + /// For verified publication, write to caller-owned staging storage and + /// publish it only after extraction succeeds. Output quotas do not provide + /// staging or rollback. This method does not flush or sync returned writers; + /// callers must check any required flush/sync before publishing staged output. + pub fn extract_to(&self, password: Option<&[u8]>, open: F) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + self.extract_to_with_options(read_options(password), open) + } + + /// Streams extracted entries to caller-provided writers with read options. + /// Has the failure and staging semantics of [`Self::extract_to`]. + pub fn extract_to_with_options( + &self, + options: ArchiveReadOptions<'_>, + mut open: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + match self { + Self::Rar13(archive) => { + archive.extract_to_with_options(options, |meta| open(&rar13_meta(meta))) + } + Self::Rar15To40(archive) => { + archive.extract_to(options, |meta| open(&rar15_40_meta(meta))) + } + Self::Rar50Plus(archive) => archive.extract_to(options, |meta| open(&rar50_meta(meta))), + } + } + + /// Visits members in archive order, choosing whether to extract, skip or stop. + /// + /// The callback receives full metadata before payload limits, dictionary + /// admission, password checks or decoding. It can skip an oversized or + /// encrypted independent member without reading its payload. Skipped output + /// does not consume output quotas and is not verified. Parsing limits and + /// passwords needed to read encrypted headers still apply before this call. + /// + /// `Extract` uses the same decoding and output accounting as [`Self::extract_to`]. + /// Its writer can already have been opened when admission fails. Callback, + /// source, decoding, integrity and sink errors stop extraction immediately; + /// partial output has the failure semantics documented on that method. + /// `Stop` returns [`ExtractionOutcome::Stopped`], distinct from reaching the end. + /// Cancellation takes precedence over a successful callback decision. + /// + /// Skipping file data is conservatively refused in any solid archive with + /// [`Error::CannotSkipSolidMember`]. To discard solid output while retaining + /// history, return `Extract(Box::new(std::io::sink()))`; this still decodes, + /// verifies, requires passwords and counts against output quotas. + /// Directories can be skipped. RAR5 redirections can be skipped or stop the + /// scan but cannot be extracted through this writer-only callback. + /// This sequential API handles a single physical archive; extracting split + /// members still requires the existing multivolume extraction API. + /// + /// ``` + /// # fn scan(archive: &rars::Archive) -> rars::Result<()> { + /// use rars::{ArchiveReadOptions, ExtractionDecision}; + /// archive.extract_with_control(ArchiveReadOptions::new(), |member| { + /// if member.meta.is_encrypted || member.meta.unpacked_size > 1024 * 1024 { + /// Ok(ExtractionDecision::Skip) + /// } else { + /// Ok(ExtractionDecision::Extract(Box::new(std::io::sink()))) + /// } + /// })?; + /// # Ok(()) + /// # } + /// ``` + pub fn extract_with_control( + &self, + options: ArchiveReadOptions<'_>, + mut decide: F, + ) -> Result + where + F: FnMut(&ArchiveMember) -> Result, + { + match self { + Self::Rar13(archive) => archive.extract_controlled(options, &mut decide, None), + Self::Rar15To40(archive) => archive.extract_controlled(options, &mut decide, None), + Self::Rar50Plus(archive) => archive.extract_controlled(options, &mut decide, None), + } + } + + /// Like [`Self::extract_with_control`], with explicit recovery after independent + /// member failures. `on_error` can return [`ExtractionErrorAction::Continue`] + /// to attempt the next member with fresh decoder state. Partial output is not + /// removed, and bytes already accepted remain charged to the total quota. + /// `Complete` means the traversal ended, not that every member was verified. + /// + /// Solid or split-member failures, resource-limit failures, cancellation and + /// errors returned by either callback are fatal and cannot be continued. + /// Parsing errors occur before this API and cannot be recovered here. + pub fn extract_with_control_and_errors( + &self, + options: ArchiveReadOptions<'_>, + mut decide: F, + mut on_error: E, + ) -> Result + where + F: FnMut(&ArchiveMember) -> Result, + E: FnMut(&ArchiveMember, &Error) -> Result, + { + match self { + Self::Rar13(archive) => { + archive.extract_controlled(options, &mut decide, Some(&mut on_error)) + } + Self::Rar15To40(archive) => { + archive.extract_controlled(options, &mut decide, Some(&mut on_error)) + } + Self::Rar50Plus(archive) => { + archive.extract_controlled(options, &mut decide, Some(&mut on_error)) + } + } + } + + /// Extracts independent non-solid members in parallel, buffering decoded + /// file bytes before replaying writes in archive order. + /// Batches are published before decoding the next batch, so a later failure + /// can leave earlier output published. Legacy RAR 1.5–4.0 batches retain at + /// most one result per worker; this is not a byte or total-memory ceiling. + /// + /// Solid archives, split members, multivolume sets, and RAR 1.3/1.4 + /// archives use the regular streaming extractor. + pub fn extract_to_parallel_buffered(&self, password: Option<&[u8]>, open: F) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + self.extract_to_parallel_buffered_with_options(read_options(password), open) + } + + /// Extracts independent non-solid members in parallel with read options. + /// Uses the batch publication semantics of [`Self::extract_to_parallel_buffered`]. + /// A configured total output ceiling selects sequential extraction. + pub fn extract_to_parallel_buffered_with_options( + &self, + options: ArchiveReadOptions<'_>, + mut open: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + match self { + Self::Rar13(archive) => { + archive.extract_to_with_options(options, |meta| open(&rar13_meta(meta))) + } + Self::Rar15To40(archive) => { + archive.extract_to_parallel_buffered(options, |meta| open(&rar15_40_meta(meta))) + } + Self::Rar50Plus(archive) => { + archive.extract_to_parallel_buffered(options, |meta| open(&rar50_meta(meta))) + } + } + } + + /// Returns one member's decoded bytes, or `None` when the archive has no + /// file of that name. + /// + /// Only the selected payload and solid predecessors are decoded and verified. + /// Unrelated independent files and later payloads are skipped. Use [`test`](Self::test) + /// to verify the whole archive. Duplicate names select the last payload member; + /// use [`read_member_at`](Self::read_member_at) for unambiguous identity. + /// Reading several solid members separately repeats predecessor decoding; + /// use [`extract_to`](Self::extract_to) to take them all in one pass. + pub fn read_member(&self, name: &[u8], password: Option<&[u8]>) -> Result>> { + self.read_member_with_options(name, read_options(password)) + } + + /// Read one named payload with cancellation and extraction resource policies. + /// Selection matches [`read_member`](Self::read_member): the last payload with + /// this name, plus required solid predecessors. Output budgets include those + /// predecessors even though their bytes are discarded. Parsing-only limits + /// are not reapplied. No partial byte buffer is returned on failure. + pub fn read_member_with_options( + &self, + name: &[u8], + options: ArchiveReadOptions<'_>, + ) -> Result>> { + options.check_cancelled()?; + let index = self + .member_refs() + .enumerate() + .filter(|(_, member)| { + member.name_bytes() == name && !member.is_directory() && !member.is_redirection() + }) + .map(|(index, _)| index) + .last(); + match index { + Some(index) => self.read_member_at_with_options(index, options), + None => Ok(None), + } + } + + /// Returns one member's decoded bytes by archive-order index. + /// + /// Unlike name lookup this remains unambiguous when an archive contains + /// duplicate names or names that are not valid UTF-8. + /// Indices include directories and redirections, which return no file bytes. + /// Missing indices also return `None` without decoding. Only the selected + /// payload and solid predecessors are verified; later payloads are not read. + pub fn read_member_at(&self, index: usize, password: Option<&[u8]>) -> Result>> { + self.read_member_at_with_options(index, read_options(password)) + } + + /// Identity-based counterpart to [`read_member_with_options`](Self::read_member_with_options). + /// Indices include directories and redirections, which return `None`. + /// Cancellation is checked even for a missing or non-payload selection. + /// A failed call returns no buffer; each call starts fresh output budgets. + pub fn read_member_at_with_options( + &self, + index: usize, + options: ArchiveReadOptions<'_>, + ) -> Result>> { + options.check_cancelled()?; + let Some(member) = self.member_refs().nth(index) else { + return Ok(None); + }; + if member.is_directory() || member.is_redirection() { + return Ok(None); + } + // Match controlled extraction's conservative solid admission policy, + // including member flags in archives without a main solid flag. + let solid = self.is_solid(); + let collected = std::sync::Arc::new(std::sync::Mutex::new(None::>)); + let mut current = 0usize; + self.extract_with_control(options, |member| { + let this = current; + current += 1; + if this > index { + return Ok(ExtractionDecision::Stop); + } + if this != index { + return Ok( + if solid && !member.meta.is_directory && !member.meta.is_redirection { + ExtractionDecision::Extract(Box::new(std::io::sink())) + } else { + ExtractionDecision::Skip + }, + ); + } + let sink = SharedBuffer(std::sync::Arc::clone(&collected)); + *sink.lock() = Some(Vec::new()); + Ok(ExtractionDecision::Extract(Box::new(sink))) + })?; + let taken = collected + .lock() + .expect("member buffer mutex is not poisoned") + .take(); + Ok(taken) + } + + /// Decodes every member and discards the bytes, so a bad checksum or a + /// wrong password is reported and nothing is written. + pub fn test(&self, password: Option<&[u8]>) -> Result<()> { + self.test_with_options(read_options(password)) + } + + /// Verify all member payloads with cancellation and extraction resource policies. + /// Discarding output does not exempt it from member or total output limits. + /// Parsing-only limits are not reapplied; comments and recovery are outside + /// this payload test. Each call starts fresh output budgets. + pub fn test_with_options(&self, options: ArchiveReadOptions<'_>) -> Result<()> { + self.test_with_options_and_callback(options, |_| Ok(())) + } + + /// Verifies payloads and reports regular entries and directories once, in + /// archive order. Redirections are skipped, as in ordinary extraction. + /// + /// Callbacks do not receive payload bytes. A callback can precede that + /// member's verification; earlier callbacks can have run when the call fails. + /// Only successful completion verifies every payload. Cancellation is checked + /// around callbacks, and a callback error aborts the operation. + /// + /// RAR5 verification discards each worker's decoded output rather than + /// retaining another buffer for publication. Buffered, streaming and scratch + /// decoding retain their extraction routing. All output, dictionary and + /// workspace limits also apply to discarded output. + pub fn test_with_options_and_callback( + &self, + options: ArchiveReadOptions<'_>, + mut tested: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result<()>, + { + options.check_cancelled()?; + if let Self::Rar50Plus(archive) = self { + return archive.test_to(options, |meta| tested(&rar50_meta(meta))); + } + self.extract_to_parallel_buffered_with_options(options, |meta| { + tested(meta)?; + Ok(Box::new(std::io::sink()) as Box) + }) + } + + /// The archive comment, decrypting its payload with the supplied password. + pub fn comment(&self, password: Option<&[u8]>) -> Result>> { + self.comment_with_options(ArchiveReadOptions::with_optional_password(password)) + } + + /// Decodes the archive comment with per-call cancellation and output limits. + /// Both output ceilings apply to this single comment, with fresh budgets. + /// Admission precedes payload allocation; no partial comment is returned. + /// RAR5 dictionary and buffered/scratch policies also apply. Parsing limits + /// are not reapplied. Existing format-specific checksum behaviour is retained. + /// The default RAR5 comment path remains buffered; set an explicit buffering + /// threshold to use streaming/scratch decoding. The returned Vec, packed input + /// and decoder workspace are not an aggregate RAM quota. Unknown-size RAR5 + /// comments are refused when either output ceiling is configured. + /// + /// ```no_run + /// # fn example(archive: &rars::Archive) -> rars::Result<()> { + /// let options = rars::ArchiveReadOptions::new() + /// .with_max_member_output_bytes(64 * 1024) + /// .with_rar50_dictionary_size_limit(8 * 1024 * 1024); + /// let comment = archive.comment_with_options(options)?; + /// # Ok(()) + /// # } + /// ``` + pub fn comment_with_options(&self, options: ArchiveReadOptions<'_>) -> Result>> { + match self { + Self::Rar13(archive) => archive.archive_comment_with_options(options), + Self::Rar15To40(archive) => archive.archive_comment_with_options(options), + Self::Rar50Plus(archive) => archive.archive_comment_with_options(options), + } + } + + /// Returns full repaired archive bytes using the archive's embedded + /// recovery records. + pub fn repair_recovery(&self) -> Result> { + Ok(self.repair_recovery_with_report(None)?.data) + } + + /// Repaired archive bytes together with what the repair had to change. + /// + /// `password` unlocks the recovery record of a header-encrypted RAR 5 + /// archive, and frames its replacement end-of-archive header. + pub fn repair_recovery_with_report( + &self, + password: Option<&[u8]>, + ) -> Result { + self.repair_recovery_with_options(ArchiveReadOptions::with_optional_password(password)) + } + + /// Repairs embedded recovery data with a password and cooperative cancellation. + /// Only password and cancellation apply; parsing and member-output limits do + /// not describe repair workspace or the complete repaired archive. + pub fn repair_recovery_with_options( + &self, + options: ArchiveReadOptions<'_>, + ) -> Result { + if !cfg!(feature = "recovery") { + return Err(Error::FeatureDisabled { + feature: "recovery", + }); + } + options.check_cancelled()?; + match self { + Self::Rar15To40(archive) => archive.repair_protect_head_with_options(options), + Self::Rar50Plus(archive) => archive.repair_recovery_with_options(options), + Self::Rar13(_) => Err(Error::UnsupportedFamilyFeature { + family: ArchiveFamily::Rar13, + feature: "recovery repair for RAR 1.3/1.4 archives", + }), + } + } + + /// Streams full repaired archive bytes to `writer` using embedded recovery + /// records. + pub fn repair_recovery_to(&self, writer: &mut dyn Write) -> Result<()> { + self.repair_recovery_to_with_report(writer, None) + .map(|_| ()) + } + + pub fn repair_recovery_to_with_report( + &self, + writer: &mut dyn Write, + password: Option<&[u8]>, + ) -> Result { + if !cfg!(feature = "recovery") { + return Err(Error::FeatureDisabled { + feature: "recovery", + }); + } + match self { + Self::Rar15To40(archive) => { + let repaired = archive.repair_protect_head_with_report()?; + writer.write_all(&repaired.data)?; + Ok(repaired.report) + } + Self::Rar50Plus(archive) => archive.repair_recovery_to_with_report(writer, password), + Self::Rar13(_) => Err(Error::UnsupportedFamilyFeature { + family: ArchiveFamily::Rar13, + feature: "recovery repair for RAR 1.3/1.4 archives", + }), + } + } + + /// Returns the concrete RAR 1.3/1.4 archive when this archive has that family. + pub fn as_rar13(&self) -> Option<&rar13::Archive> { + match self { + Self::Rar13(archive) => Some(archive), + Self::Rar15To40(_) => None, + Self::Rar50Plus(_) => None, + } + } + + /// Returns the concrete RAR 1.5 through RAR 4.x archive when applicable. + pub fn as_rar15_40(&self) -> Option<&rar15_40::Archive> { + match self { + Self::Rar13(_) => None, + Self::Rar15To40(archive) => Some(archive), + Self::Rar50Plus(_) => None, + } + } + + /// Returns the concrete RAR 5.0 or later archive when applicable. + pub fn as_rar50(&self) -> Option<&rar50::Archive> { + match self { + Self::Rar13(_) | Self::Rar15To40(_) => None, + Self::Rar50Plus(archive) => Some(archive), + } + } +} + +fn rar13_member(entry: &rar13::Entry) -> ArchiveMember { + ArchiveMember { + meta: ArchiveMemberMeta { + family: ArchiveFamily::Rar13, + name: entry.name.clone(), + packed_size: u64::from(entry.header.pack_size), + unpacked_size: u64::from(entry.header.unp_size), + file_time: Some(entry.header.file_time), + mtime_refinement: None, + file_attr: u64::from(entry.header.file_attr), + host_os: None, + is_directory: entry.is_directory(), + is_redirection: false, + is_encrypted: entry.is_encrypted(), + is_stored: entry.is_stored(), + is_split_before: entry.is_split_before(), + is_split_after: entry.is_split_after(), + }, + detail: ArchiveMemberDetail::Rar13 { + method: entry.header.method, + unpack_version: entry.header.unp_ver, + file_checksum: entry.header.file_crc, + has_file_comment: entry.has_file_comment(), + }, + } +} + +fn rar15_40_member(file: &rar15_40::FileHeader) -> ArchiveMember { + ArchiveMember { + meta: ArchiveMemberMeta { + family: ArchiveFamily::Rar15To40, + name: file.name.clone(), + packed_size: file.pack_size, + unpacked_size: file.unp_size, + file_time: Some(file.file_time), + mtime_refinement: file.mtime_refinement(), + file_attr: u64::from(file.attr), + host_os: Some(u64::from(file.host_os)), + is_directory: file.is_directory(), + is_redirection: false, + is_encrypted: file.is_encrypted(), + is_stored: file.is_stored(), + is_split_before: file.is_split_before(), + is_split_after: file.is_split_after(), + }, + detail: ArchiveMemberDetail::Rar15To40 { + method: file.method, + unpack_version: file.unp_ver, + crc32: file.file_crc, + solid: file.is_solid(), + salt: file.salt, + unicode_name: file.unicode_name.clone(), + extended_times: file.ext_time.clone(), + has_file_comment: file.has_file_comment(), + }, + } +} + +fn rar50_member(file: &rar50::FileHeader) -> ArchiveMember { + ArchiveMember { + meta: ArchiveMemberMeta { + family: ArchiveFamily::Rar50Plus, + name: file.name.clone(), + packed_size: file.packed_size(), + unpacked_size: file.unpacked_size, + file_time: file.modification_time(), + mtime_refinement: file.modification_time_refinement(), + file_attr: file.attributes, + host_os: Some(file.host_os), + is_directory: file.is_directory(), + is_redirection: file.is_redirection(), + is_encrypted: file.encrypted, + is_stored: file.is_stored(), + is_split_before: file.is_split_before(), + is_split_after: file.is_split_after(), + }, + detail: ArchiveMemberDetail::Rar50Plus { + compression_info: file.compression_info, + crc32: file.data_crc32, + hash: file.hash.as_ref().map(rar50_member_hash), + redirection: file.redirection.clone(), + file_times: file.file_times, + }, + } +} + +fn rar50_member_hash(hash: &rar50::FileHash) -> ArchiveMemberHash { + match hash.hash_type { + 0 if hash.data.len() == 32 => { + let mut data = [0; 32]; + data.copy_from_slice(&hash.data); + ArchiveMemberHash::Blake2sp(data) + } + _ => ArchiveMemberHash::Other { + hash_type: hash.hash_type, + data: hash.data.clone(), + }, + } +} + +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +/// Archive reader facade with signature-based dispatch. +pub struct ArchiveReader; + +/// Describes what an embedded recovery repair changed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RecoveryRepairReport { + pub changed: bool, + pub data_repaired: bool, + pub recovery_record_rebuilt: bool, + pub end_record_rebuilt: bool, + pub available_recovery_shards: Option, + pub expected_recovery_shards: Option, +} + +/// Repaired archive bytes together with a precise repair report. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecoveryRepairResult { + pub data: Vec, + pub report: RecoveryRepairReport, +} + +#[cfg(feature = "recovery")] +impl RecoveryRepairResult { + /// Publishes repaired bytes beside the destination, replacing it only after + /// writing, syncing and a final cancellation check succeed. Cancellation + /// after the final check cannot interrupt the rename. Temporary output is + /// removed on failure; the destination is untouched until publication. + pub fn write_to_path( + &self, + path: &std::path::Path, + cancellation: Option<&ReadCancellation>, + ) -> Result<()> { + let control = crate::read_control::ReadControl::new(cancellation); + self.write_to_path_controlled(path, &control) + } + + fn write_to_path_controlled( + &self, + path: &std::path::Path, + control: &crate::read_control::ReadControl, + ) -> Result<()> { + control.check()?; + let (mut pending, output) = crate::pending_archive::PendingArchive::create(path)?; + { + let mut output = output; + control.finish( + control + .write_all(&mut output, &self.data) + .map_err(Error::from), + )?; + output.sync_all()?; + } + control.check()?; + std::fs::rename(pending.path.as_ref().unwrap(), path)?; + pending.path = None; + Ok(()) + } +} + +impl ArchiveReader { + /// Detects the archive signature in a byte slice. + pub fn detect(input: &[u8]) -> Result { + detect_archive_family(input).ok_or(Error::UnsupportedSignature) + } + + /// Parses an archive from memory with default read options. + pub fn read(input: &[u8]) -> Result { + Self::read_with_options(input, ArchiveReadOptions::default()) + } + + /// Parses an archive from an owned memory buffer with default read options. + pub fn read_owned(input: Vec) -> Result { + Self::read_owned_with_options(input, ArchiveReadOptions::default()) + } + + /// Parses an archive from memory using explicit read options. + pub fn read_with_options(input: &[u8], options: ArchiveReadOptions<'_>) -> Result { + options.check_cancelled()?; + let signature = + find_archive_start(input, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + match signature.family { + ArchiveFamily::Rar13 => Ok(Archive::Rar13(rar13::Archive::parse_with_options( + input, options, + )?)), + ArchiveFamily::Rar15To40 => Ok(Archive::Rar15To40( + rar15_40::Archive::parse_with_options(input, options)?, + )), + ArchiveFamily::Rar50Plus => Ok(Archive::Rar50Plus(rar50::Archive::parse_with_options( + input, options, + )?)), + } + } + + /// Parses an archive from an owned memory buffer using explicit read options. + pub fn read_owned_with_options( + input: Vec, + options: ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let signature = + find_archive_start(&input, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + match signature.family { + ArchiveFamily::Rar13 => Ok(Archive::Rar13(rar13::Archive::parse_owned_with_options( + input, options, + )?)), + ArchiveFamily::Rar15To40 => Ok(Archive::Rar15To40( + rar15_40::Archive::parse_owned_with_options(input, options)?, + )), + ArchiveFamily::Rar50Plus => Ok(Archive::Rar50Plus( + rar50::Archive::parse_owned_with_options(input, options)?, + )), + } + } + + /// Parses an archive from a path with default read options. + /// + /// See [`Self::read_reader`] to retain an already-open file handle instead. + pub fn read_path(path: impl AsRef) -> Result { + Self::read_path_with_options(path, ArchiveReadOptions::default()) + } + + /// Parses an owned seekable source without copying the complete archive. + /// + /// Reads from offset zero regardless of the source's initial position. The + /// archive and its clones retain the source until they are dropped. Its + /// contents and length must remain unchanged throughout that lifetime. + /// Sources borrowed from a shorter-lived owner are not supported; an owned + /// file, memory-map wrapper, or shared virtual-file handle can be supplied. + /// + /// Source I/O is serialized, with a separate position per reader; parallel + /// decoding still runs concurrently. Parsing reads the signature scan and + /// headers, while extraction reads payloads on demand. Individual codecs + /// may buffer payloads. Cancellation cannot interrupt a blocked source call. + /// + /// ```no_run + /// # fn example() -> rars::Result<()> { + /// let file = std::fs::File::open("archive.rar")?; + /// let archive = rars::ArchiveReader::read_reader(file)?; + /// archive.extract_to(None, |_| Ok(Box::new(std::io::sink())))?; + /// # Ok(()) + /// # } + /// ``` + pub fn read_reader(reader: R) -> Result { + Self::read_reader_with_options(reader, ArchiveReadOptions::default()) + } + + /// Parses an owned seekable source with the same policies as [`Self::read_with_options`]. + /// Ownership, positioning and concurrency follow [`Self::read_reader`]. + pub fn read_reader_with_options( + reader: R, + options: ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let reader = crate::source::ReaderSource::new(reader)?; + let source = crate::source::ArchiveSource::Reader(reader.clone()); + let len = source.len()?; + let mut cursor = reader.cursor(); + let signature = read_archive_signature( + &mut crate::read_control::ReadControl::new(options.cancellation).reader(&mut cursor), + len.min(SFX_SCAN_LIMIT), + )?; + options.check_cancelled()?; + let signature = signature.ok_or(Error::UnsupportedSignature)?; + match signature.family { + ArchiveFamily::Rar13 => Ok(Archive::Rar13(rar13::Archive::parse_seekable( + cursor, + len as u64, + signature.offset, + source, + options, + )?)), + ArchiveFamily::Rar15To40 => Ok(Archive::Rar15To40(rar15_40::Archive::parse_seekable( + cursor, + len as u64, + signature.offset, + source, + options, + )?)), + ArchiveFamily::Rar50Plus => Ok(Archive::Rar50Plus(rar50::Archive::parse_file_backed( + &mut cursor, + len, + signature.offset, + source, + options, + )?)), + } + } + + /// Parses an archive from a path using explicit read options. + pub fn read_path_with_options( + path: impl AsRef, + options: ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let path = path.as_ref(); + let mut file = std::fs::File::open(path)?; + let len = file.metadata()?.len(); + let signature = read_archive_signature(&mut file, len.min(SFX_SCAN_LIMIT as u64) as usize)?; + options.check_cancelled()?; + let signature = signature.ok_or(Error::UnsupportedSignature)?; + match signature.family { + ArchiveFamily::Rar13 => Ok(Archive::Rar13( + rar13::Archive::parse_path_with_signature_and_options(path, signature, options)?, + )), + ArchiveFamily::Rar15To40 => Ok(Archive::Rar15To40( + rar15_40::Archive::parse_path_with_signature(path, signature, options)?, + )), + ArchiveFamily::Rar50Plus => Ok(Archive::Rar50Plus( + rar50::Archive::parse_path_with_signature(path, signature, options)?, + )), + } + } +} + +fn read_archive_signature( + reader: &mut impl Read, + scan_len: usize, +) -> Result> { + let mut prefix = [0; 8]; + let prefix_len = scan_len.min(prefix.len()); + reader.read_exact(&mut prefix[..prefix_len])?; + if let Some(signature) = detect_archive_family(&prefix[..prefix_len]) + .filter(|signature| signature.family != ArchiveFamily::Rar13) + { + // The first strong signature wins the full scan too. RAR 1.3 still + // scans because a later RAR 1.5/5 signature takes precedence over it. + return Ok(Some(signature)); + } + if scan_len <= prefix.len() { + return Ok(find_archive_start(&prefix[..prefix_len], SFX_SCAN_LIMIT)); + } + let mut scan = vec![0; scan_len]; + scan[..prefix_len].copy_from_slice(&prefix[..prefix_len]); + reader.read_exact(&mut scan[prefix_len..])?; + Ok(find_archive_start(&scan, SFX_SCAN_LIMIT)) +} + +#[cfg(test)] +#[path = "reader_signature_tests.rs"] +mod reader_signature_tests; + +fn read_options(password: Option<&[u8]>) -> ArchiveReadOptions<'_> { + match password { + Some(password) => ArchiveReadOptions::with_password(password), + None => ArchiveReadOptions::new(), + } +} + +/// Streams a multivolume archive set to caller-provided writers. +/// Has the failure and staging semantics of [`Archive::extract_to`], including +/// partial output from a split member when a later volume fails. +pub fn extract_volumes_to(archives: &[Archive], password: Option<&[u8]>, open: F) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + extract_volumes_to_with_options(archives, read_options(password), open) +} + +/// Streams a multivolume archive set to caller-provided writers with read options. +/// Has the failure and staging semantics of [`Archive::extract_to`]. +pub fn extract_volumes_to_with_options( + archives: &[Archive], + options: ArchiveReadOptions<'_>, + mut open: F, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + options.check_cancelled()?; + let Some(first) = archives.first() else { + return Err(Error::InvalidHeader("volume set is empty")); + }; + + match first.family() { + ArchiveFamily::Rar13 => { + let typed = rar13_volumes(archives)?; + rar13::extract_volumes_to_with_options(&typed, options, |meta| open(&rar13_meta(meta))) + } + ArchiveFamily::Rar15To40 => { + let typed = rar15_40_volumes(archives)?; + rar15_40::extract_volumes_to(&typed, options, |meta| open(&rar15_40_meta(meta))) + } + ArchiveFamily::Rar50Plus => { + let typed = rar50_volumes(archives)?; + rar50::extract_volumes_to(&typed, options, |meta| open(&rar50_meta(meta))) + } + } +} + +/// Verifies every logical payload in a volume set and discards decoded output. +/// Has the resource and cancellation policies of [`Archive::test_with_options`]. +pub fn test_volumes_with_options( + archives: &[Archive], + options: ArchiveReadOptions<'_>, +) -> Result<()> { + test_volumes_with_options_and_callback(archives, options, |_| Ok(())) +} + +/// Verifies a volume set with metadata callbacks. Split members produce one +/// callback at their final fragment, in logical archive order. Has the callback +/// and failure semantics of [`Archive::test_with_options_and_callback`]. +pub fn test_volumes_with_options_and_callback( + archives: &[Archive], + options: ArchiveReadOptions<'_>, + mut tested: F, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result<()>, +{ + options.check_cancelled()?; + let Some(first) = archives.first() else { + return Err(Error::InvalidHeader("volume set is empty")); + }; + if first.family() == ArchiveFamily::Rar50Plus { + let typed = rar50_volumes(archives)?; + return rar50::test_volumes_to(&typed, options, |meta| tested(&rar50_meta(meta))); + } + extract_volumes_to_with_options(archives, options, |meta| { + tested(meta)?; + Ok(Box::new(std::io::sink()) as Box) + }) +} + +/// Returns the logical members in a volume set in archive order. +/// +/// Continuation headers are folded into the member that began on an earlier +/// volume. Packed sizes are accumulated across all fragments. +pub fn volume_members(archives: &[Archive]) -> Result> { + let Some(first) = archives.first() else { + return Err(Error::InvalidHeader("volume set is empty")); + }; + if archives + .iter() + .any(|archive| archive.family() != first.family()) + { + return Err(Error::InvalidHeader("mixed archive families in volume set")); + } + + let mut members: Vec = Vec::new(); + for archive in archives { + for member in archive.members() { + if member.meta.is_split_before { + let Some(previous) = members.last_mut() else { + return Err(Error::InvalidHeader( + "volume set starts with a continuation", + )); + }; + previous.meta.packed_size = previous + .meta + .packed_size + .saturating_add(member.meta.packed_size); + previous.meta.is_split_after = member.meta.is_split_after; + } else { + members.push(member); + } + } + } + Ok(members) +} + +/// Returns one logical member from a volume set by archive-order index. +pub fn read_volume_member_at( + archives: &[Archive], + index: usize, + password: Option<&[u8]>, +) -> Result>> { + read_volume_member_at_with_options( + archives, + index, + ArchiveReadOptions::with_optional_password(password), + ) +} + +/// Reads one logical volume member with per-call policies. The current volume +/// traversal decodes the whole set; discarded output also counts against quotas. +pub fn read_volume_member_at_with_options( + archives: &[Archive], + index: usize, + options: ArchiveReadOptions<'_>, +) -> Result>> { + options.check_cancelled()?; + let mut output_indices = member_read::volume_output_indices(archives); + let collected = std::sync::Arc::new(std::sync::Mutex::new(None::>)); + extract_volumes_to_with_options(archives, options, |meta| { + if output_indices.next() != Some(index) || meta.is_directory { + return Ok(Box::new(std::io::sink()) as Box); + } + let sink = SharedBuffer(std::sync::Arc::clone(&collected)); + *sink.lock() = Some(Vec::new()); + Ok(Box::new(sink) as Box) + })?; + let taken = collected + .lock() + .expect("member buffer mutex is not poisoned") + .take(); + Ok(taken) +} + +/// Detail a coarse timestamp cannot hold, carried alongside it. +/// +/// A DOS timestamp counts in two-second steps, so RAR 1.5-4.x archives put the +/// odd second and any sub-second precision in a separate extended field. This +/// is that field decoded: add [`add_second`](Self::add_second) whole seconds +/// and then [`nanoseconds`](Self::nanoseconds) to the base time. +/// RAR5 extended times also use this detail, with `add_second` always false. +/// +/// Kept apart from the timestamp rather than folded into it because +/// [`ExtractedEntryMeta::file_time`] is the DOS value as stored, which has +/// nowhere to put either part. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct TimeRefinement { + /// Whether the true time is one second later than the DOS value. + pub add_second: bool, + /// Sub-second remainder, below 1_000_000_000. + pub nanoseconds: u32, +} + +/// How to read [`ExtractedEntryMeta::file_attr`], which depends on the host +/// that wrote the entry. +/// +/// The grouping is measured, not assumed, because it is not the obvious one. +/// Against RAR 7.12 on Linux at umask 022, extracting one file with only +/// `HOST_OS` and `ATTR` changed: +/// +/// | RAR 1.5-4.x `HOST_OS` | attr `0x21` | reading | +/// |---|---|---| +/// | 0 MS-DOS, 1 OS/2, 2 Win32, 4 Mac OS | 444 | DOS attributes | +/// | 3 Unix, 5 BeOS | 041 | raw `st_mode` | +/// | 6 WinCE, 7+ | 644 | ignored | +/// +/// Mac OS sits with the DOS hosts and BeOS with the Unix ones, the reverse of +/// how the two are usually grouped. RAR 5.0 numbers its hosts separately: 0 is +/// Windows, 1 is Unix, and RAR 7.12 ignores the attributes of anything else. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum AttrSource { + /// Windows `FILE_ATTRIBUTE_*` bits. + Dos, + /// POSIX `st_mode`. + Unix, + /// A host this build does not know; attributes carry no meaning. + #[default] + Unknown, +} + +impl AttrSource { + fn rar15_40(host_os: u8) -> Self { + match host_os { + 0 | 1 | 2 | 4 => Self::Dos, + 3 | 5 => Self::Unix, + _ => Self::Unknown, + } + } + + fn rar50(host_os: u64) -> Self { + match host_os { + 0 => Self::Dos, + 1 => Self::Unix, + _ => Self::Unknown, + } + } +} + +fn rar13_meta(meta: &rar13::ExtractedEntryMeta) -> ExtractedEntryMeta { + ExtractedEntryMeta { + name_is_unicode: false, + name: meta.name.clone(), + file_time: Some(meta.file_time), + file_attr: u64::from(meta.file_attr), + // RAR 1.3/1.4 is MS-DOS only. + attr_source: AttrSource::Dos, + // RAR 1.3/1.4 predates the extended time field. + mtime_refinement: None, + is_directory: meta.is_directory, + } +} + +fn rar15_40_meta(meta: &rar15_40::ExtractedEntryMeta) -> ExtractedEntryMeta { + ExtractedEntryMeta { + name_is_unicode: meta.name_is_unicode, + name: meta.name.clone(), + file_time: Some(meta.file_time), + file_attr: u64::from(meta.attr), + attr_source: AttrSource::rar15_40(meta.host_os), + mtime_refinement: meta.mtime_refinement, + is_directory: meta.is_directory, + } +} + +/// Converts RAR 5.0 entry metadata into the common form, resolving the +/// attributes against the host OS as extraction needs them. +/// +/// Public so a caller doing its own extraction gets the same resolution the +/// built-in paths do; getting it wrong loses the read-only bit or applies a +/// Windows attribute word as a Unix mode. +pub fn rar50_meta(meta: &rar50::ExtractedEntryMeta) -> ExtractedEntryMeta { + ExtractedEntryMeta { + name_is_unicode: true, + name: meta.name.clone(), + file_time: meta.file_time, + file_attr: meta.attr, + attr_source: AttrSource::rar50(meta.host_os), + mtime_refinement: meta.mtime_refinement, + is_directory: meta.is_directory, + } +} + +fn rar13_volumes(archives: &[Archive]) -> Result> { + archives + .iter() + .map(|archive| match archive { + Archive::Rar13(archive) => Ok(archive.clone()), + Archive::Rar15To40(_) | Archive::Rar50Plus(_) => { + Err(Error::InvalidHeader("mixed archive families in volume set")) + } + }) + .collect() +} + +fn rar15_40_volumes(archives: &[Archive]) -> Result> { + archives + .iter() + .map(|archive| match archive { + Archive::Rar15To40(archive) => Ok(archive.clone()), + Archive::Rar13(_) | Archive::Rar50Plus(_) => { + Err(Error::InvalidHeader("mixed archive families in volume set")) + } + }) + .collect() +} + +fn rar50_volumes(archives: &[Archive]) -> Result> { + archives + .iter() + .map(|archive| match archive { + Archive::Rar50Plus(archive) => Ok(archive.clone()), + Archive::Rar13(_) | Archive::Rar15To40(_) => { + Err(Error::InvalidHeader("mixed archive families in volume set")) + } + }) + .collect() +} + +#[cfg(all(test, feature = "write"))] +mod tests { + use super::*; + use std::cell::RefCell; + use std::path::{Path, PathBuf}; + use std::rc::Rc; + + #[test] + fn facade_buffer_flush_preserves_written_bytes() { + let bytes = std::sync::Arc::new(std::sync::Mutex::new(None)); + let mut writer = SharedBuffer(bytes.clone()); + writer.flush().unwrap(); + assert!(bytes.lock().unwrap().is_none()); + assert_eq!(writer.write(b"").unwrap(), 0); + writer.write_all(b"a").unwrap(); + writer.flush().unwrap(); + writer.write_all(b"bc").unwrap(); + writer.flush().unwrap(); + writer.flush().unwrap(); + assert_eq!(*bytes.lock().unwrap(), Some(b"abc".to_vec())); + } + + #[test] + fn facade_family_accessors_and_public_hash_models_preserve_information() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.as_rar13().is_some(), + version == ArchiveVersion::Rar13 + ); + assert_eq!( + archive.as_rar15_40().is_some(), + version == ArchiveVersion::Rar29 + ); + assert_eq!( + archive.as_rar50().is_some(), + version == ArchiveVersion::Rar50 + ); + if !matches!(archive, Archive::Rar50Plus(_)) { + continue; + } + for (hash_type, data) in [ + (0, vec![7; 32]), + (0, vec![7; 31]), + (0, vec![7; 33]), + (1, vec![7; 32]), + (u64::MAX, vec![]), + ] { + let Archive::Rar50Plus(modern) = &mut archive else { + unreachable!() + }; + let file = modern + .blocks + .iter_mut() + .find_map(|block| match block { + rar50::Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + file.hash = Some(rar50::FileHash { + hash_type, + data: data.clone(), + }); + let member = archive.members().next().unwrap(); + let ArchiveMemberDetail::Rar50Plus { hash, .. } = member.detail else { + panic!("modern member must retain its family") + }; + let expected = if hash_type == 0 && data.len() == 32 { + ArchiveMemberHash::Blake2sp([7; 32]) + } else { + ArchiveMemberHash::Other { hash_type, data } + }; + assert_eq!(hash, Some(expected)); + } + } + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_publication_rename_failure_preserves_destination_and_cleans_staging() { + let root = crate::scratch::case("recovery-publication-rename-failure"); + let destination = root.join("destination"); + std::fs::create_dir(&destination).unwrap(); + let sentinel = destination.join("existing"); + std::fs::write(&sentinel, b"keep these bytes").unwrap(); + let result = RecoveryRepairResult { + data: b"repaired bytes".to_vec(), + report: RecoveryRepairReport::default(), + }; + let error = result.write_to_path(&destination, None).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"keep these bytes"); + assert_eq!(std::fs::read_dir(&destination).unwrap().count(), 1); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 1); + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_publication_cancellation_at_each_checkpoint_preserves_destination() { + let root = crate::scratch::case("recovery-publication-checkpoints"); + let destination = root.join("archive.rar"); + let result = RecoveryRepairResult { + // Three writes, including a partial final chunk. + data: vec![7; 2 * 64 * 1024 + 1], + report: RecoveryRepairReport::default(), + }; + for successful_checks in 0..=6 { + std::fs::write(&destination, b"original archive").unwrap(); + let token = ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(successful_checks); + let published = result.write_to_path_controlled(&destination, &control); + if successful_checks < 6 { + assert_eq!(published.unwrap_err(), Error::Cancelled); + assert!(token.is_cancelled()); + assert_eq!(std::fs::read(&destination).unwrap(), b"original archive"); + } else { + published.unwrap(); + assert!(!token.is_cancelled()); + assert_eq!(std::fs::read(&destination).unwrap(), result.data); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 1); + } + } + + #[test] + fn member_mtime_matches_rar50_extraction_without_losing_absence() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"time".to_vec(), vec![], None, None) + .unwrap(); + let archive = rar50::Archive::parse(&builder.to_bytes().unwrap()).unwrap(); + let mut file = archive.files().next().unwrap().clone(); + for (base, extended, expected) in [ + (None, None, None), + (Some(0), None, Some(0)), + (None, Some(0), Some(0)), + (None, Some(1_700_000_002), Some(1_700_000_002)), + (Some(123), Some(456), Some(123)), + (Some(0), Some(456), Some(0)), + ] { + file.mtime = base; + file.htime_mtime = extended; + assert_eq!(rar50_member(&file).meta.file_time, expected); + assert_eq!( + rar50_member(&file).meta.stored_modification_time(), + expected.map(StoredTimestamp::UnixSeconds) + ); + assert_eq!(file.metadata().file_time, expected); + } + } + + struct CollectWriter { + data: Rc>>, + } + + #[derive(Debug, Clone, PartialEq, Eq)] + // Historical numeric snapshots; timestamp presence has dedicated regression tests. + struct CollectedEntry { + name: Vec, + data: Vec, + file_time: u32, + file_attr: u64, + is_directory: bool, + } + + fn deterministic_noise(len: usize) -> Vec { + let mut state = 0x1234_5678u32; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect() + } + + fn rar15_40_fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar15_40") + .join(name) + } + + /// Which hosts store DOS attributes and which store `st_mode` is measured + /// against RAR 7.12, and the grouping is not the obvious one: Mac OS reads + /// as a DOS host and BeOS as a Unix one. + #[test] + fn attr_source_follows_the_measured_host_grouping() { + for host in [0u8, 1, 2, 4] { + assert_eq!(AttrSource::rar15_40(host), AttrSource::Dos, "host {host}"); + } + for host in [3u8, 5] { + assert_eq!(AttrSource::rar15_40(host), AttrSource::Unix, "host {host}"); + } + for host in [6u8, 7, 255] { + assert_eq!( + AttrSource::rar15_40(host), + AttrSource::Unknown, + "host {host}" + ); + } + + assert_eq!(AttrSource::rar50(0), AttrSource::Dos); + assert_eq!(AttrSource::rar50(1), AttrSource::Unix); + for host in [2u64, 5, 99] { + assert_eq!(AttrSource::rar50(host), AttrSource::Unknown, "host {host}"); + } + + // A caller building metadata by hand gets no host, and no host means + // no attribute is applied rather than one guessed at. + assert_eq!( + ExtractedEntryMeta::new(b"x".to_vec(), 0, 0x21, false).attr_source, + AttrSource::Unknown + ); + } + + #[test] + fn extracted_entry_meta_exposes_raw_and_lossy_names() { + let meta = ExtractedEntryMeta { + name_is_unicode: false, + name: vec![0xff, b'.', b't', b'x', b't'], + file_time: None, + file_attr: 0, + attr_source: AttrSource::Unknown, + mtime_refinement: None, + is_directory: false, + }; + + assert_eq!(meta.name_bytes(), [0xff, b'.', b't', b'x', b't']); + assert_eq!(meta.name_lossy(), "\u{fffd}.txt"); + } + + impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.data.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + fn collect_extract(archive: &Archive, password: Option<&[u8]>) -> Result> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(password, |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time.unwrap_or(0), + file_attr: meta.file_attr, + is_directory: meta.is_directory, + }) + .collect()) + } + + fn collect_rar15_40(archive: &rar15_40::Archive) -> Result> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(ArchiveReadOptions::default(), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + file_attr: u64::from(meta.attr), + is_directory: meta.is_directory, + }) + .collect()) + } + + fn collect_rar15_40_volumes( + archives: &[rar15_40::Archive], + password: Option<&[u8]>, + ) -> Result> { + let entries = RefCell::new(Vec::new()); + rar15_40::extract_volumes_to(archives, read_options(password), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + file_attr: u64::from(meta.attr), + is_directory: meta.is_directory, + }) + .collect()) + } + + fn collect_rar50_volumes( + archives: &[rar50::Archive], + password: Option<&[u8]>, + ) -> Result> { + let entries = RefCell::new(Vec::new()); + rar50::extract_volumes_to(archives, read_options(password), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time.unwrap_or(0), + file_attr: meta.attr, + is_directory: meta.is_directory, + }) + .collect()) + } + + fn collect_rar50_file( + archive: &rar50::Archive, + file: &rar50::FileHeader, + ) -> Result { + let meta = file.metadata(); + let data = Rc::new(RefCell::new(Vec::new())); + file.write_to( + archive, + None, + &mut CollectWriter { + data: Rc::clone(&data), + }, + )?; + let data = data.borrow().clone(); + Ok(CollectedEntry { + name: meta.name, + data, + file_time: meta.file_time.unwrap_or(0), + file_attr: meta.attr, + is_directory: meta.is_directory, + }) + } + + fn rar13_options(target: ArchiveVersion) -> rar13::WriterOptions { + rar13::WriterOptions::new(target, FeatureSet::store_only()) + } + + fn rar15_options(target: ArchiveVersion) -> rar15_40::WriterOptions { + rar15_options_with_features(target, FeatureSet::store_only()) + } + + fn rar15_options_with_features( + target: ArchiveVersion, + features: FeatureSet, + ) -> rar15_40::WriterOptions { + rar15_40::WriterOptions::new(target, features) + } + + fn rar50_options(target: ArchiveVersion) -> rar50::WriterOptions { + rar50_options_with_features(target, FeatureSet::store_only()) + } + + fn rar50_options_with_features( + target: ArchiveVersion, + features: FeatureSet, + ) -> rar50::WriterOptions { + rar50::WriterOptions::new(target, features) + } + + /// Builds a member from bytes the test already holds. + fn rar50_entry(name: &[u8], data: &[u8]) -> rar50::ArchiveEntry { + rar50::ArchiveEntry::new( + name.to_vec(), + EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data.to_vec())), + ) + } + + fn write_rar50_volume_set( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + max_payload_per_volume: u64, + recovery_percent: Option, + ) -> Vec> { + let mut sink = rar50::CollectedVolumes::new(); + rar50::write_streaming_volumes_to( + entries, + options, + rar50::ArchiveExtras::default().with_recovery_percent(recovery_percent), + max_payload_per_volume, + &mut sink, + &WriterResources::default(), + ) + .unwrap(); + sink.take() + } + + fn write_rar29_filter( + options: rar15_40::WriterOptions, + entries: &[rar15_40::FileEntry<'_>], + kind: rar15_40::FilterKind, + ) -> Result> { + rar15_40::write_rar29_compressed_archive_with_filter_policy( + entries, + options, + rar15_40::FilterPolicy::Explicit(rar15_40::FilterSpec::whole(kind)), + ) + } + + fn write_rar29_filter_range( + options: rar15_40::WriterOptions, + entries: &[rar15_40::FileEntry<'_>], + kind: rar15_40::FilterKind, + range: std::ops::Range, + ) -> Result> { + rar15_40::write_rar29_compressed_archive_with_filter_policy( + entries, + options, + rar15_40::FilterPolicy::Explicit(rar15_40::FilterSpec::range(kind, range)), + ) + } + + #[cfg(feature = "recovery")] + fn assert_rar50_volume_recovery_records(archives: &[rar50::Archive], percent: u64) { + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives + .iter() + .all(|archive| archive.main.has_recovery_record())); + for archive in archives { + let service = archive.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, percent); + let data = collect_rar50_file(archive, service).unwrap().data; + assert!(data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(data[0x0c..0x10].try_into().unwrap()) as usize, + data.len() + ); + } + } + + #[test] + fn direct_writer_creates_rar15_stored_archive() { + let bytes = rar15_40::write_stored_archive( + &[rar15_40::StoredEntry { + name: b"hello.txt", + data: b"hello via facade\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar15), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert_eq!(archive.family(), ArchiveFamily::Rar15To40); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"hello via facade\n"); + } + + #[test] + fn archive_reader_accepts_owned_buffers_without_changing_dispatch() { + let rar13_bytes = rar13::write_stored_archive( + &[rar13::StoredEntry { + name: b"old.txt", + data: b"owned rar13\n", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }], + rar13_options(ArchiveVersion::Rar14), + ) + .unwrap(); + let rar13_archive = ArchiveReader::read_owned(rar13_bytes).unwrap(); + assert_eq!(rar13_archive.family(), ArchiveFamily::Rar13); + assert_eq!( + collect_extract(&rar13_archive, None).unwrap()[0].data, + b"owned rar13\n" + ); + + let rar15_bytes = rar15_40::write_stored_archive( + &[rar15_40::StoredEntry { + name: b"mid.txt", + data: b"owned rar15\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar15), + ) + .unwrap(); + let rar15_archive = ArchiveReader::read_owned(rar15_bytes).unwrap(); + assert_eq!(rar15_archive.family(), ArchiveFamily::Rar15To40); + assert_eq!( + collect_extract(&rar15_archive, None).unwrap()[0].data, + b"owned rar15\n" + ); + + let rar50_bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entries( + [rar50_entry(b"new.txt", b"owned rar50\n") + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .finish() + .unwrap(); + let rar50_archive = ArchiveReader::read_owned(rar50_bytes).unwrap(); + assert_eq!(rar50_archive.family(), ArchiveFamily::Rar50Plus); + assert_eq!( + collect_extract(&rar50_archive, None).unwrap()[0].data, + b"owned rar50\n" + ); + } + + #[test] + fn direct_writer_keeps_rar13_methods_version_typed() { + let err = + rar13::write_stored_archive(&[], rar13_options(ArchiveVersion::Rar15)).unwrap_err(); + + assert!(matches!( + err, + Error::UnsupportedVersion(ArchiveVersion::Rar15) + )); + } + + #[test] + fn archive_members_exposes_rar13_common_metadata_and_typed_detail() { + let bytes = rar13::write_stored_archive( + &[rar13::StoredEntry { + name: b"old.txt", + data: b"old rar member", + file_time: 0x1234_5678, + file_attr: 0x20, + password: None, + file_comment: Some(b"note"), + }], + rar13_options(ArchiveVersion::Rar14), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + + assert_eq!(members.len(), 1); + assert_eq!(members[0].meta.family, ArchiveFamily::Rar13); + assert_eq!(members[0].meta.name, b"old.txt"); + assert_eq!(members[0].meta.name_bytes(), b"old.txt"); + assert_eq!(members[0].meta.name_lossy(), "old.txt"); + assert_eq!(members[0].meta.packed_size, b"old rar member".len() as u64); + assert_eq!( + members[0].meta.unpacked_size, + b"old rar member".len() as u64 + ); + assert_eq!(members[0].meta.file_time, Some(0x1234_5678)); + assert_eq!(members[0].meta.file_attr, 0x20); + assert_eq!(members[0].meta.host_os, None); + assert!(members[0].meta.is_stored); + assert!(!members[0].meta.is_encrypted); + assert!(!members[0].meta.is_split_before); + assert!(!members[0].meta.is_split_after); + assert!(matches!( + members[0].detail, + ArchiveMemberDetail::Rar13 { + method: 0, + unpack_version: _, + file_checksum: _, + has_file_comment: true, + .. + } + )); + } + + #[test] + fn archive_members_exposes_rar15_40_common_metadata_and_typed_detail() { + let features = FeatureSet::store_only(); + let payload = b"rar 2.9 member metadata ".repeat(32); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"newer.txt", + data: &payload, + file_time: 0x0102_0304, + file_attr: 0x20, + host_os: 2, + password: None, + file_comment: Some(b"rar29 note"), + }], + rar15_options_with_features(ArchiveVersion::Rar29, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + + assert_eq!(members.len(), 1); + assert_eq!(members[0].meta.family, ArchiveFamily::Rar15To40); + assert_eq!(members[0].meta.name, b"newer.txt"); + assert_eq!(members[0].meta.unpacked_size, payload.len() as u64); + assert_eq!(members[0].meta.file_time, Some(0x0102_0304)); + assert_eq!(members[0].meta.file_attr, 0x20); + assert_eq!(members[0].meta.host_os, Some(2)); + assert!(!members[0].meta.is_stored); + assert!(!members[0].meta.is_encrypted); + assert!(matches!( + members[0].detail, + ArchiveMemberDetail::Rar15To40 { + method: 0x33 | 0x35, + unpack_version: 29, + solid: false, + salt: None, + has_file_comment: true, + .. + } + )); + } + + #[test] + fn archive_members_exposes_rar50_common_metadata_and_typed_detail() { + let bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entries( + [rar50_entry(b"five.txt", b"rar 5 member metadata") + .with_mtime(Some(0x1111_2222)) + .with_attributes(0x1_0000_0020) + .with_host_os(3)] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + + assert_eq!(members.len(), 1); + assert_eq!(members[0].meta.family, ArchiveFamily::Rar50Plus); + assert_eq!(members[0].meta.name, b"five.txt"); + assert_eq!( + members[0].meta.packed_size, + b"rar 5 member metadata".len() as u64 + ); + assert_eq!( + members[0].meta.unpacked_size, + b"rar 5 member metadata".len() as u64 + ); + assert_eq!(members[0].meta.file_time, Some(0x1111_2222)); + assert_eq!(members[0].meta.file_attr, 0x1_0000_0020); + assert_eq!(members[0].meta.host_os, Some(3)); + assert!(members[0].meta.is_stored); + assert!(!members[0].meta.is_encrypted); + assert!(matches!( + members[0].detail, + ArchiveMemberDetail::Rar50Plus { .. } + )); + } + + #[test] + fn extraction_metadata_preserves_rar50_u64_file_attributes() { + let bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entries( + [rar50_entry(b"wide-attrs.txt", b"wide RAR5 file attributes") + .with_mtime(Some(0)) + .with_attributes(0x1_0000_0020) + .with_host_os(3)] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"wide-attrs.txt"); + assert_eq!(extracted[0].file_attr, 0x1_0000_0020); + } + + #[test] + fn direct_writer_creates_rar15_compressed_archive() { + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"text.txt", + data: b"facade compressed facade compressed facade compressed\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar15), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert_eq!(archive.family(), ArchiveFamily::Rar15To40); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!( + extracted[0].data, + b"facade compressed facade compressed facade compressed\n" + ); + } + + #[test] + fn direct_writer_creates_rar29_compressed_archive_with_default_auto_policy() { + let payload = + b"facade rar29 default auto text alpha beta gamma alpha beta gamma\n".repeat(256); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar29-default-auto.txt", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar29), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + // The byte is the level that was asked for, and nothing here asked for + // one, so it is the default 0x33. Which engine answered is signalled in + // the stream, which is why the round trip below is what proves PPMd + // read back. WinRAR stamps 0x34 on the PPMd archive it writes at -m4. + assert_eq!(raw.files().next().unwrap().method, 0x33); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_e8_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0facade rar29 e8 filter payload\n".repeat(12); + let bytes = write_rar29_filter( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-e8.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::E8, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_auto_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0facade rar29 auto filter payload\n".repeat(12); + let bytes = rar15_40::write_rar29_compressed_archive_with_filter_policy( + &[rar15_40::FileEntry { + name: b"rar29-auto.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar29), + rar15_40::FilterPolicy::Auto, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_ppmd_compressed_archive() { + let payload = b"facade rar29 ppmd text payload alpha beta gamma\n".repeat(64); + let bytes = rar15_40::write_rar29_compressed_archive_with_filter_policy( + &[rar15_40::FileEntry { + name: b"rar29-ppmd.txt", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar29).with_method(rar15_40::Rar29Method::Ppmd), + rar15_40::FilterPolicy::None, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + // Forcing PPMd does not change the level, so this is still the default + // 0x33; extracting it is what shows PPMd was used. + assert_eq!(file.method, 0x33); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_segmented_e8_filtered_compressed_archive() { + let mut payload = b"facade unfiltered prefix before x86 segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(b"\xe8\0\0\0\0facade segmented e8 filter payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"facade unfiltered suffix after x86 segment\n"); + let bytes = write_rar29_filter_range( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-segmented-e8.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::E8, + filter_start..filter_end, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_solid_e8_filtered_compressed_archive() { + let first = b"\xe8\0\0\0\0facade rar29 solid e8 first payload\n".repeat(12); + let second = b"\xe8\0\0\0\0facade rar29 solid e8 second payload\n".repeat(12); + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_rar29_filter( + rar15_options_with_features(ArchiveVersion::Rar29, features), + &[ + rar15_40::FileEntry { + name: b"rar29-solid-e8-first.bin", + data: &first, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_40::FileEntry { + name: b"rar29-solid-e8-second.bin", + data: &second, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ], + rar15_40::FilterKind::E8, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let files: Vec<_> = raw.files().collect(); + assert!(raw.main.is_solid()); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar29_encrypted_e8_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0facade rar29 encrypted e8 payload\n".repeat(12); + let features = FeatureSet::store_only(); + let bytes = write_rar29_filter( + rar15_options_with_features(ArchiveVersion::Rar29, features), + &[rar15_40::FileEntry { + name: b"rar29-encrypted-e8.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_40::FilterKind::E8, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert!(file.is_encrypted()); + assert!(file.salt.is_some()); + assert!(matches!( + collect_extract(&archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar30_header_encrypted_e8_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0facade rar30 header encrypted e8 payload\n".repeat(12); + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_rar29_filter( + rar15_options_with_features(ArchiveVersion::Rar30, features), + &[rar15_40::FileEntry { + name: b"rar30-header-encrypted-e8.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_40::FilterKind::E8, + ) + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(raw.main.has_encrypted_headers()); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_e8e9_filtered_compressed_archive() { + let payload = b"\xe9\0\0\0\0facade rar29 e8e9 filter payload\n".repeat(12); + let bytes = write_rar29_filter( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-e8e9.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::E8E9, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_delta_filtered_compressed_archive() { + let payload: Vec = (0..384).map(|index| (index * 19 + 5) as u8).collect(); + let bytes = write_rar29_filter( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-delta.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Delta { channels: 3 }, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_segmented_delta_filtered_compressed_archive() { + let mut payload = b"facade unfiltered prefix before delta segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..384).map(|index| (index * 19 + 5) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"facade unfiltered suffix after delta segment\n"); + let bytes = write_rar29_filter_range( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-segmented-delta.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Delta { channels: 3 }, + filter_start..filter_end, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_itanium_filtered_compressed_archive() { + let mut payload = vec![0u8; 48]; + payload[16] = 22; + payload[21] = 20; + payload.extend_from_slice(b"facade rar29 itanium filter payload\n"); + let bytes = write_rar29_filter( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-itanium.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Itanium, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_segmented_itanium_filtered_compressed_archive() { + let mut payload = b"facade unfiltered prefix before itanium segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(&[0; 48]); + payload[filter_start + 16] = 22; + payload[filter_start + 21] = 20; + payload.extend_from_slice(b"facade segmented itanium filter payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"facade unfiltered suffix after itanium segment\n"); + let bytes = write_rar29_filter_range( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-segmented-itanium.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Itanium, + filter_start..filter_end, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_rgb_filtered_compressed_archive() { + let width = 12; + let payload: Vec = (0..96).map(|index| (index * 37 + 17) as u8).collect(); + let bytes = write_rar29_filter( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-rgb.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Rgb { width, pos_r: 0 }, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_segmented_rgb_filtered_compressed_archive() { + let width = 12; + let mut payload = b"facade unfiltered prefix before rgb segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..96).map(|index| (index * 37 + 17) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"facade unfiltered suffix after rgb segment\n"); + let bytes = write_rar29_filter_range( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-segmented-rgb.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Rgb { width, pos_r: 0 }, + filter_start..filter_end, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_audio_filtered_compressed_archive() { + let payload: Vec = (0..160) + .map(|index| (index * 11 + index / 7) as u8) + .collect(); + let bytes = write_rar29_filter( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-audio.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Audio { channels: 2 }, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_segmented_audio_filtered_compressed_archive() { + let mut payload = b"facade unfiltered prefix before audio segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..160).map(|index| (index * 11 + index / 7) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"facade unfiltered suffix after audio segment\n"); + let bytes = write_rar29_filter_range( + rar15_options(ArchiveVersion::Rar29), + &[rar15_40::FileEntry { + name: b"rar29-segmented-audio.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_40::FilterKind::Audio { channels: 2 }, + filter_start..filter_end, + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar20_compressed_archive() { + let payload = b"facade rar20 literal compressed payload\n".repeat(32); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar20.txt", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar20), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert_eq!(archive.family(), ArchiveFamily::Rar15To40); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.unp_ver, 20); + assert_eq!(file.method, 0x33); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_compressed_archive() { + let payload = b"facade rar29 literal compressed payload\n".repeat(32); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar29.txt", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar29), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert_eq!(archive.family(), ArchiveFamily::Rar15To40); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.unp_ver, 29); + assert!(matches!(file.method, 0x33 | 0x35)); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar29_solid_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = rar15_40::write_compressed_archive( + &[ + rar15_40::FileEntry { + name: b"one.txt", + data: b"facade rar29 solid one alpha beta\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_40::FileEntry { + name: b"two.txt", + data: b"facade rar29 solid two alpha beta\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ], + rar15_options_with_features(ArchiveVersion::Rar29, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(raw.main.is_solid()); + let files: Vec<_> = raw.files().collect(); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar29 solid one alpha beta\n"); + assert_eq!(extracted[1].data, b"facade rar29 solid two alpha beta\n"); + } + + #[test] + fn direct_writer_creates_rar20_solid_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let first = b"facade rar20 solid shared line alpha beta gamma\n".repeat(48); + let second = b"facade rar20 solid shared line alpha beta gamma\nsecond\n".repeat(24); + let bytes = rar15_40::write_compressed_archive( + &[ + rar15_40::FileEntry { + name: b"one.txt", + data: &first, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_40::FileEntry { + name: b"two.txt", + data: &second, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ], + rar15_options_with_features(ArchiveVersion::Rar20, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(raw.main.is_solid()); + let files: Vec<_> = raw.files().collect(); + assert_eq!(files[0].unp_ver, 20); + assert_eq!(files[1].unp_ver, 20); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); + } + + #[test] + fn direct_writer_creates_rar15_archive_comment() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive_with_comment( + &[rar15_40::FileEntry { + name: b"commented.txt", + data: b"facade commented payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar15, features), + Some(b"facade note\n"), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let archive = archive.as_rar15_40().unwrap(); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"facade note\n"[..]) + ); + assert_eq!( + collect_rar15_40(archive).unwrap()[0].data, + b"facade commented payload\n" + ); + } + + #[test] + fn direct_writer_creates_rar15_file_comment() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_stored_archive( + &[rar15_40::StoredEntry { + name: b"file-comment.txt", + data: b"facade file comment payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"facade file note"), + }], + rar15_options_with_features(ArchiveVersion::Rar15, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let archive = archive.as_rar15_40().unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!( + file.file_comment().unwrap().as_deref(), + Some(&b"facade file note"[..]) + ); + assert_eq!( + collect_rar15_40(archive).unwrap()[0].data, + b"facade file comment payload\n" + ); + } + + #[test] + fn direct_writer_creates_rar20_old_style_comments() { + let archive_features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive_with_comment( + &[rar15_40::FileEntry { + name: b"rar20-commented.txt", + data: b"facade rar20 archive comment payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar20, archive_features), + Some(b"facade rar20 archive note"), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(raw.main.has_archive_comment()); + assert_eq!( + raw.archive_comment().unwrap().as_deref(), + Some(b"facade rar20 archive note".as_slice()) + ); + + let file_features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar20-file-commented.txt", + data: b"facade rar20 file comment payload payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"facade rar20 file note"), + }], + rar15_options_with_features(ArchiveVersion::Rar20, file_features), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.unp_ver, 20); + assert_eq!( + file.file_comment().unwrap().as_deref(), + Some(b"facade rar20 file note".as_slice()) + ); + } + + #[test] + fn direct_writer_creates_rar29_old_style_comments() { + let archive_features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive_with_comment( + &[rar15_40::FileEntry { + name: b"rar29-commented.txt", + data: b"facade rar29 archive comment payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar29, archive_features), + Some(b"facade rar29 archive note"), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(raw.main.has_archive_comment()); + assert_eq!( + raw.archive_comment().unwrap().as_deref(), + Some(b"facade rar29 archive note".as_slice()) + ); + + let file_features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar29-file-commented.txt", + data: b"facade rar29 file comment payload payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"facade rar29 file note"), + }], + rar15_options_with_features(ArchiveVersion::Rar29, file_features), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.unp_ver, 29); + assert_eq!( + file.file_comment().unwrap().as_deref(), + Some(b"facade rar29 file note".as_slice()) + ); + } + + #[test] + fn direct_writer_creates_rar30_newsub_archive_comment() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive_with_comment( + &[rar15_40::FileEntry { + name: b"rar30-commented.txt", + data: b"facade rar30 NEWSUB archive comment payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar30, features), + Some(b"facade rar30 NEWSUB note"), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(!raw.main.has_archive_comment()); + let subblock = raw.new_subs().next().unwrap(); + assert_eq!(subblock.kind, rar15_40::NewSubKind::ArchiveComment); + assert_eq!(subblock.file.name, b"CMT"); + assert_eq!( + raw.archive_comment().unwrap().as_deref(), + Some(b"facade rar30 NEWSUB note".as_slice()) + ); + } + + #[test] + fn direct_writer_creates_rar15_solid_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = rar15_40::write_compressed_archive( + &[ + rar15_40::FileEntry { + name: b"one.txt", + data: b"shared facade prefix one\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_40::FileEntry { + name: b"two.txt", + data: b"shared facade prefix two\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ], + rar15_options_with_features(ArchiveVersion::Rar15, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"shared facade prefix one\n"); + assert_eq!(extracted[1].data, b"shared facade prefix two\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar15_encrypted_compressed_archive() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"secret.txt", + data: b"facade encrypted payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar15, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert!(matches!( + collect_extract(&archive, None), + Err(Error::NeedPassword) + )); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"facade encrypted payload\n"); + } + + #[test] + fn direct_writer_creates_rar15_stored_volumes() { + let parts = rar15_40::write_stored_volumes( + rar15_40::StoredEntry { + name: b"split.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_options(ArchiveVersion::Rar15), + 10, + ) + .unwrap(); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse(part).unwrap()) + .collect(); + let extracted = collect_rar15_40_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"split.bin"); + assert_eq!(extracted[0].data, b"abcdefghijklmnopqrstuvwxyz0123456789"); + } + + #[test] + fn direct_writer_creates_rar20_compressed_volumes() { + let data = b"facade rar20 split phrase alpha beta gamma\n".repeat(32); + let parts = rar15_40::write_compressed_volumes( + rar15_40::FileEntry { + name: b"split-rar20.txt", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_options(ArchiveVersion::Rar20), + 8, + ) + .unwrap(); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse(part).unwrap()) + .collect(); + let first_file = archives[0].files().next().unwrap(); + assert_eq!(first_file.unp_ver, 20); + assert!(first_file.is_split_after()); + + let extracted = collect_rar15_40_volumes(&archives, None).unwrap(); + assert_eq!(extracted[0].name, b"split-rar20.txt"); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn direct_writer_creates_rar29_compressed_volumes() { + let data = b"facade rar29 split phrase alpha beta gamma\n".repeat(32); + let parts = rar15_40::write_compressed_volumes( + rar15_40::FileEntry { + name: b"split-rar29.txt", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + rar15_options(ArchiveVersion::Rar29), + 8, + ) + .unwrap(); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse(part).unwrap()) + .collect(); + let first_file = archives[0].files().next().unwrap(); + assert_eq!(first_file.unp_ver, 29); + assert!(first_file.is_split_after()); + + let extracted = collect_rar15_40_volumes(&archives, None).unwrap(); + assert_eq!(extracted[0].name, b"split-rar29.txt"); + assert_eq!(extracted[0].data, data); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar29_encrypted_compressed_volumes() { + let features = FeatureSet::store_only(); + let parts = rar15_40::write_compressed_volumes( + rar15_40::FileEntry { + name: b"split-rar29-secret.txt", + data: b"facade rar29 encrypted split facade rar29 encrypted split\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + rar15_options_with_features(ArchiveVersion::Rar29, features), + 8, + ) + .unwrap(); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse(part).unwrap()) + .collect(); + + assert!(matches!( + collect_rar15_40_volumes(&archives, None), + Err(Error::NeedPassword) + )); + let extracted = collect_rar15_40_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-rar29-secret.txt"); + assert_eq!( + extracted[0].data, + b"facade rar29 encrypted split facade rar29 encrypted split\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar15_encrypted_compressed_volumes() { + let features = FeatureSet::store_only(); + let parts = rar15_40::write_compressed_volumes( + rar15_40::FileEntry { + name: b"split-secret.txt", + data: b"facade encrypted split facade encrypted split\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + rar15_options_with_features(ArchiveVersion::Rar15, features), + 8, + ) + .unwrap(); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse(part).unwrap()) + .collect(); + + assert!(matches!( + collect_rar15_40_volumes(&archives, None), + Err(Error::NeedPassword) + )); + let extracted = collect_rar15_40_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-secret.txt"); + assert_eq!( + extracted[0].data, + b"facade encrypted split facade encrypted split\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar30_encrypted_compressed_volumes() { + let features = FeatureSet::store_only(); + let parts = rar15_40::write_compressed_volumes( + rar15_40::FileEntry { + name: b"split-rar30-secret.txt", + data: b"facade rar30 encrypted split facade rar30 encrypted split\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + rar15_options_with_features(ArchiveVersion::Rar30, features), + 8, + ) + .unwrap(); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse(part).unwrap()) + .collect(); + + assert!(matches!( + collect_rar15_40_volumes(&archives, None), + Err(Error::NeedPassword) + )); + let extracted = collect_rar15_40_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-rar30-secret.txt"); + assert_eq!( + extracted[0].data, + b"facade rar30 encrypted split facade rar30 encrypted split\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar30_header_encrypted_compressed_volumes() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let parts = rar15_40::write_compressed_volumes( + rar15_40::FileEntry { + name: b"split-rar30-header-secret.txt", + data: b"facade rar30 header encrypted split facade rar30 header encrypted split\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + rar15_options_with_features(ArchiveVersion::Rar30, features), + 8, + ) + .unwrap(); + assert!(matches!( + rar15_40::Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + let archives: Vec<_> = parts + .iter() + .map(|part| rar15_40::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + + let extracted = collect_rar15_40_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-rar30-header-secret.txt"); + assert_eq!( + extracted[0].data, + b"facade rar30 header encrypted split facade rar30 header encrypted split\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar30_aes_encrypted_compressed_archive() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar30-secret.txt", + data: b"facade rar30 aes encrypted payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar30, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert!(matches!( + collect_extract(&archive, None), + Err(Error::NeedPassword) + )); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"facade rar30 aes encrypted payload\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar29_aes_encrypted_compressed_archive() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar29-secret.txt", + data: b"facade rar29 aes encrypted payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar29, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.unp_ver, 29); + assert!(file.is_encrypted()); + assert!(file.salt.is_some()); + assert!(matches!( + collect_extract(&archive, None), + Err(Error::NeedPassword) + )); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"facade rar29 aes encrypted payload\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar20_encrypted_compressed_archive() { + let features = FeatureSet::store_only(); + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar20-secret.txt", + data: b"facade rar20 encrypted payload payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar20, features), + ) + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar15_40().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.unp_ver, 20); + assert!(file.is_encrypted()); + assert!(matches!( + collect_extract(&archive, None), + Err(Error::NeedPassword) + )); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar20 encrypted payload payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar30_header_encrypted_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar15_40::write_compressed_archive( + &[rar15_40::FileEntry { + name: b"rar30-header-secret.txt", + data: b"facade rar30 header encrypted payload\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }], + rar15_options_with_features(ArchiveVersion::Rar30, features), + ) + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar15_40().unwrap(); + assert!(raw.main.has_encrypted_headers()); + assert_eq!(raw.files().next().unwrap().name, b"rar30-header-secret.txt"); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar30 header encrypted payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar30_solid_header_encrypted_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let bytes = rar15_40::write_compressed_archive( + &[ + rar15_40::FileEntry { + name: b"solid-header-one.txt", + data: b"facade solid header encrypted one one one\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + rar15_40::FileEntry { + name: b"solid-header-two.txt", + data: b"facade solid header encrypted two two two\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + ], + rar15_options_with_features(ArchiveVersion::Rar30, features), + ) + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade solid header encrypted one one one\n" + ); + assert_eq!( + extracted[1].data, + b"facade solid header encrypted two two two\n" + ); + } + + #[test] + fn direct_writer_creates_rar50_stored_archive() { + let bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entries( + [ + rar50_entry(b"rar5-store.txt", b"facade rar5 stored payload\n") + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3), + ] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert_eq!(archive.family(), ArchiveFamily::Rar50Plus); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 stored payload\n"); + } + + #[test] + fn direct_writer_creates_rar50_compressed_archive() { + let bytes = rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50)) + .entries( + [rar50_entry( + b"rar5-compressed.txt", + b"facade rar5 compressed payload\nfacade rar5 compressed payload\n", + ) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let file = raw.files().next().unwrap(); + assert_eq!(file.decoded_compression_info().unwrap().method, 3); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar5 compressed payload\nfacade rar5 compressed payload\n" + ); + } + + #[test] + fn direct_writer_creates_rar50_solid_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let first = b"facade rar50 solid shared phrase alpha beta gamma\n".repeat(16); + let second = b"facade rar50 solid shared phrase alpha beta gamma\nsecond\n".repeat(8); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-solid-one.txt", &first) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3), + rar50_entry(b"rar5-solid-two.txt", &second) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3), + ] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let files: Vec<_> = raw.files().collect(); + assert!(raw.main.is_solid()); + assert!(!files[0].decoded_compression_info().unwrap().solid); + assert!(files[1].decoded_compression_info().unwrap().solid); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); + } + + #[test] + fn direct_writer_creates_rar50_delta_filtered_compressed_archive() { + let payload: Vec = (0..180) + .map(|index| (index * 11 + index / 5) as u8) + .collect(); + let bytes = rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50)) + .entries( + [rar50_entry(b"rar5-delta-filtered.bin", &payload) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .filter_policy(rar50::FilterPolicy::explicit(rar50::FilterKind::Delta { + channels: 3, + })) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_e8_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0facade rar5 e8 filter payload".to_vec(); + let bytes = rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50)) + .entries( + [rar50_entry(b"rar5-e8-filtered.bin", &payload) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .filter_policy(rar50::FilterPolicy::explicit(rar50::FilterKind::E8)) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_arm_filtered_compressed_archive() { + let payload = [0x04, 0x00, 0x00, 0xeb, b'A', b'R', b'M', b'!']; + let bytes = rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50)) + .entries( + [rar50_entry(b"rar5-arm-filtered.bin", &payload) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .filter_policy(rar50::FilterPolicy::explicit(rar50::FilterKind::Arm)) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_auto_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0facade rar5 auto filter payload\n".repeat(16); + let bytes = rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50)) + .entries( + [rar50_entry(b"rar5-auto-filtered.bin", &payload) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .filter_policy(rar50::FilterPolicy::Auto) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_stored_archive_with_comment_service() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [ + rar50_entry(b"rar5-commented.txt", b"facade rar5 comment payload\n") + .with_attributes(0x20) + .with_host_os(3), + ] + .to_vec(), + ) + .archive_comment(Some(b"facade rar5 comment\n")) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let services: Vec<_> = raw.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert_eq!( + collect_rar50_file(raw, services[0]).unwrap().data, + b"facade rar5 comment\n" + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 comment payload\n"); + } + + #[test] + fn direct_writer_creates_rar50_stored_file_comment_service() { + let bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entry( + rar50_entry( + b"rar5-file-commented.txt", + b"facade rar5 file comment payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_service(rar50::ServiceEntry::new( + b"CMT".to_vec(), + b"facade rar5 file comment\n".to_vec(), + )), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let services: Vec<_> = raw.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert_eq!( + collect_rar50_file(raw, services[0]).unwrap().data, + b"facade rar5 file comment\n" + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 file comment payload\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_stored_file_comment_service() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entry( + rar50_entry( + b"rar5-encrypted-file-commented.txt", + b"facade encrypted rar5 file comment payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()) + .with_service( + rar50::ServiceEntry::new( + b"CMT".to_vec(), + b"facade encrypted rar5 file comment\n".to_vec(), + ) + .with_password(b"password".to_vec()), + ), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let service = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!(service.data, b"facade encrypted rar5 file comment\n"); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade encrypted rar5 file comment payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_stored_file_comment_service() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entry( + rar50_entry( + b"rar5-header-file-commented.txt", + b"facade header encrypted rar5 file comment payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()) + .with_service( + rar50::ServiceEntry::new( + b"CMT".to_vec(), + b"facade header encrypted rar5 file comment\n".to_vec(), + ) + .with_password(b"password".to_vec()), + ), + ) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let service = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!(service.data, b"facade header encrypted rar5 file comment\n"); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade header encrypted rar5 file comment payload\n" + ); + } + + #[test] + fn direct_writer_creates_rar50_stored_archive_with_quick_open_service() { + let mut features = FeatureSet::store_only(); + features.quick_open = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [ + rar50_entry(b"rar5-qo.txt", b"facade rar5 quick-open payload\n") + .with_attributes(0x20) + .with_host_os(3), + ] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.locator().unwrap().quick_open_offset.unwrap() > 0); + let services: Vec<_> = raw.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"QO"); + assert!(!collect_rar50_file(raw, services[0]) + .unwrap() + .data + .is_empty()); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 quick-open payload\n"); + } + + #[test] + fn rar50_quick_open_wrapper_checksums_the_block_size_vint() { + // The wrapper is CRC32 || BlockSize || body and the checksum covers + // BlockSize too, which is easy to miss because the length is written + // after the checksum it belongs to. Checksumming the body alone is + // invisible from inside rars: nothing here reads a quick-open index + // back, and a reference reader that rejects the wrapper just walks the + // block chain instead and calls the archive fine. + let mut features = FeatureSet::store_only(); + features.quick_open = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [ + rar50_entry(b"a.txt", b"one\n").with_host_os(3), + rar50_entry(b"b.txt", b"two\n").with_host_os(3), + ] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"QO"); + let payload = collect_rar50_file(raw, service).unwrap().data; + + let mut pos = 0usize; + let mut wrappers = 0usize; + while pos + 4 < payload.len() { + let stored = u32::from_le_bytes(payload[pos..pos + 4].try_into().unwrap()); + let (block_size, size_len) = { + let (mut value, mut shift, mut len) = (0u64, 0u32, 0usize); + loop { + let byte = payload[pos + 4 + len]; + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + len += 1; + if byte & 0x80 == 0 { + break (value, len); + } + } + }; + let framed_start = pos + 4; + let framed_end = framed_start + size_len + block_size as usize; + let framed = &payload[framed_start..framed_end]; + + assert_eq!( + crate::crc32::crc32(framed), + stored, + "wrapper {wrappers}: checksum must span the BlockSize vint and the body" + ); + assert_ne!( + crate::crc32::crc32(&framed[size_len..]), + stored, + "wrapper {wrappers}: checksum must not be over the body alone" + ); + + pos = framed_end; + wrappers += 1; + } + assert_eq!(wrappers, 2); + } + + #[test] + fn direct_writer_creates_rar50_stored_archive_with_file_services() { + let bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entry( + rar50_entry(b"rar5-services.txt", b"facade rar5 service payload\n") + .with_attributes(0x20) + .with_host_os(3) + .with_service(rar50::ServiceEntry::new( + b"ACL".to_vec(), + b"facade acl".to_vec(), + )) + .with_service(rar50::ServiceEntry::new( + b"STM".to_vec(), + b"facade stream".to_vec(), + )), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let services: Vec<_> = raw.services().collect(); + assert_eq!(services.len(), 2); + assert_eq!(services[0].name, b"ACL"); + assert_eq!(services[1].name, b"STM"); + assert_eq!( + collect_rar50_file(raw, services[0]).unwrap().data, + b"facade acl" + ); + assert_eq!( + collect_rar50_file(raw, services[1]).unwrap().data, + b"facade stream" + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 service payload\n"); + } + + #[test] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_stored_archive_with_recovery_service() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [ + rar50_entry(b"rar5-recovery.txt", b"facade rar5 recovery payload\n") + .with_attributes(0x20) + .with_host_os(3), + ] + .to_vec(), + ) + .recovery_percent(Some(9)) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.has_recovery_record()); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + let recovery = service.recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 9); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 recovery payload\n"); + } + + #[test] + #[cfg(feature = "recovery")] + fn archive_facade_repairs_rar50_inline_recovery_damage() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 repair payload\n".repeat(64); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry(b"rar5-repair.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .recovery_percent(Some(20)) + .finish() + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let data_range = archive + .as_rar50() + .unwrap() + .files() + .next() + .unwrap() + .block + .data_range + .clone(); + let mut damaged = bytes.clone(); + damaged[data_range.start + 4..data_range.start + 80].fill(0xa5); + let damaged_archive = ArchiveReader::read(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive, None).is_err()); + + let mut repaired = Vec::new(); + damaged_archive.repair_recovery_to(&mut repaired).unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = ArchiveReader::read(&repaired).unwrap(); + assert_eq!( + collect_extract(&repaired_archive, None).unwrap()[0].data, + payload + ); + } + + #[test] + #[cfg(feature = "recovery")] + fn archive_facade_reports_rar13_family_for_unsupported_recovery_repair() { + let bytes = rar13::write_stored_archive( + &[rar13::StoredEntry { + name: b"old.txt", + data: b"old rar payload", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }], + rar13_options(ArchiveVersion::Rar13), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let mut repaired = Vec::new(); + let err = archive.repair_recovery_to(&mut repaired).unwrap_err(); + + assert_eq!( + err, + Error::UnsupportedFamilyFeature { + family: ArchiveFamily::Rar13, + feature: "recovery repair for RAR 1.3/1.4 archives" + } + ); + } + + #[test] + #[cfg(feature = "recovery")] + fn archive_facade_repairs_rar15_40_recovery_as_full_archive_bytes() { + let bytes = std::fs::read(rar15_40_fixture("rar250_protect_head_rr5.rar")).unwrap(); + let mut damaged = bytes.clone(); + damaged[512 + 16..512 + 80].fill(0xa5); + let damaged_archive = ArchiveReader::read(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive, None).is_err()); + + let mut repaired = Vec::new(); + damaged_archive.repair_recovery_to(&mut repaired).unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = ArchiveReader::read(&repaired).unwrap(); + assert_eq!( + collect_extract(&repaired_archive, None).unwrap()[0].name, + b"BIG.BIN" + ); + } + + #[test] + #[cfg(feature = "recovery")] + fn archive_facade_repairs_rar3_newsub_recovery_as_full_archive_bytes() { + let bytes = std::fs::read(rar15_40_fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let mut damaged = bytes.clone(); + damaged[512 + 16..512 + 80].fill(0xa5); + let damaged_archive = ArchiveReader::read(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive, None).is_err()); + + let mut repaired = Vec::new(); + damaged_archive.repair_recovery_to(&mut repaired).unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = ArchiveReader::read(&repaired).unwrap(); + assert_eq!( + collect_extract(&repaired_archive, None).unwrap()[0].name, + b"bigtext_64k.bin" + ); + } + + #[test] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_compressed_archive_with_recovery_service() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 compressed recovery payload repeated repeated\n".repeat(8); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [rar50_entry(b"rar5-compressed-recovery.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .recovery_percent(Some(9)) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.has_recovery_record()); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 9); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar70_stored_archive_with_metadata() { + let bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar70).with_compression_level(0)) + .entries( + [ + rar50_entry(b"rar7-metadata.txt", b"facade rar7 metadata payload\n") + .with_attributes(0x20) + .with_host_os(3), + ] + .to_vec(), + ) + .archive_metadata(Some(rar50::ArchiveMetadataEntry { + name: Some(b"facade-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let metadata = archive.as_rar50().unwrap().main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"facade-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"facade rar7 metadata payload\n"); + } + + #[test] + fn direct_writer_creates_rar70_compressed_archive_with_metadata() { + let payload = b"facade rar7 compressed metadata payload repeated\n".repeat(8); + let bytes = rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar70)) + .entries( + [rar50_entry(b"rar7-compressed-metadata.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .archive_metadata(Some(rar50::ArchiveMetadataEntry { + name: Some(b"facade-compressed-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let metadata = archive.as_rar50().unwrap().main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"facade-compressed-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_compressed_archive_with_comment() { + let payload = b"facade rar5 compressed archive comment payload repeated\n".repeat(8); + let features = FeatureSet::store_only(); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [rar50_entry(b"rar5-compressed-comment.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .archive_comment(Some(b"facade compressed comment\n")) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let comment = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!(comment.data, b"facade compressed comment\n"); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar70_encrypted_stored_archive_with_metadata() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar70, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar7-encrypted-metadata.txt", + b"facade rar7 encrypted metadata payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .archive_metadata(Some(rar50::ArchiveMetadataEntry { + name: Some(b"facade-encrypted-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let metadata = archive.as_rar50().unwrap().main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"facade-encrypted-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar7 encrypted metadata payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar70_encrypted_compressed_archive_with_metadata() { + let features = FeatureSet::store_only(); + let payload = b"facade rar7 encrypted compressed metadata payload repeated\n".repeat(8); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar70, features)) + .entries( + [ + rar50_entry(b"rar7-encrypted-compressed-metadata.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .archive_metadata(Some(rar50::ArchiveMetadataEntry { + name: Some(b"facade-encrypted-compressed-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let metadata = archive.as_rar50().unwrap().main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"facade-encrypted-compressed-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar70_header_encrypted_stored_archive_with_metadata() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar70, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar7-header-metadata.txt", + b"facade rar7 header encrypted metadata payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .archive_metadata(Some(rar50::ArchiveMetadataEntry { + name: Some(b"facade-header-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let metadata = archive.as_rar50().unwrap().main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"facade-header-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar7 header encrypted metadata payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar70_header_encrypted_compressed_archive_with_metadata() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let payload = + b"facade rar7 header encrypted compressed metadata payload repeated\n".repeat(8); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar70, features)) + .entries( + [ + rar50_entry(b"rar7-header-compressed-metadata.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .archive_metadata(Some(rar50::ArchiveMetadataEntry { + name: Some(b"facade-header-compressed-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let metadata = archive.as_rar50().unwrap().main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"facade-header-compressed-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_stored_archive() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar5-secret.txt", + b"facade rar5 encrypted stored payload\n", + ) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + assert!(matches!( + collect_extract(&archive, None), + Err(Error::AtEntry { source, .. }) if matches!(*source, Error::NeedPassword) + )); + assert!(matches!( + collect_extract(&archive, Some(b"wrong")), + Err(Error::AtEntry { source, .. }) + if matches!(*source, Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 encrypted stored payload\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_compressed_archive() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 encrypted compressed\n".repeat(16); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [rar50_entry(b"rar5-secret-compressed.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let file = raw.files().next().unwrap(); + assert!(file.encrypted); + assert_eq!(file.decoded_compression_info().unwrap().method, 3); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_solid_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let first = b"facade rar50 encrypted solid shared phrase alpha beta gamma\n".repeat(12); + let second = + b"facade rar50 encrypted solid shared phrase alpha beta gamma\nsecond\n".repeat(6); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-encrypted-solid-one.txt", &first) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + rar50_entry(b"rar5-encrypted-solid-two.txt", &second) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + let files: Vec<_> = raw.files().collect(); + assert!(raw.main.is_solid()); + assert!(files.iter().all(|file| file.encrypted)); + assert!(!files[0].decoded_compression_info().unwrap().solid); + assert!(files[1].decoded_compression_info().unwrap().solid); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries([rar50_entry(b"rar5-header-secret-compressed.txt", b"facade rar5 header encrypted compressed\nfacade rar5 header encrypted compressed\n").with_attributes(0x20).with_host_os(3).with_password(b"password".to_vec())].to_vec()) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let file = raw.files().next().unwrap(); + assert!(file.encrypted); + assert_eq!(file.decoded_compression_info().unwrap().method, 3); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar5 header encrypted compressed\nfacade rar5 header encrypted compressed\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_solid_compressed_archive() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let first = + b"facade rar50 header encrypted solid shared phrase alpha beta gamma\n".repeat(12); + let second = + b"facade rar50 header encrypted solid shared phrase alpha beta gamma\nsecond\n" + .repeat(6); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-header-solid-one.txt", &first) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + rar50_entry(b"rar5-header-solid-two.txt", &second) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let files: Vec<_> = raw.files().collect(); + assert!(raw.main.is_solid()); + assert!(files.iter().all(|file| file.encrypted)); + assert!(!files[0].decoded_compression_info().unwrap().solid); + assert!(files[1].decoded_compression_info().unwrap().solid); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_stored_archive_with_comment() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar5-secret.txt", + b"facade rar5 encrypted stored payload\n", + ) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .encrypted_archive_comment(b"facade encrypted comment\n", b"password") + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let comment = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!(comment.data, b"facade encrypted comment\n"); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"facade rar5 encrypted stored payload\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_compressed_archive_with_comment() { + let payload = b"facade rar5 encrypted compressed comment payload\n".repeat(8); + let features = FeatureSet::store_only(); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-encrypted-compressed-comment.txt", &payload) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .encrypted_archive_comment(b"facade encrypted compressed comment\n", b"password") + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let comment = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!(comment.data, b"facade encrypted compressed comment\n"); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_stored_archive() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar5-header-secret.txt", + b"facade rar5 header encrypted stored payload\n", + ) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + assert!(matches!( + ArchiveReader::read_with_options(&bytes, ArchiveReadOptions::with_password(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar5 header encrypted stored payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_stored_archive_with_comment() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar5-header-comment-secret.txt", + b"facade rar5 header encrypted comment payload\n", + ) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .encrypted_archive_comment(b"facade header encrypted comment\n", b"password") + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let comment = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!(comment.data, b"facade header encrypted comment\n"); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar5 header encrypted comment payload\n" + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_compressed_archive_with_comment() { + let payload = b"facade rar5 header encrypted compressed comment payload\n".repeat(8); + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-header-compressed-comment-secret.txt", &payload) + .with_mtime(Some(0)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .encrypted_archive_comment( + b"facade header encrypted compressed comment\n", + b"password", + ) + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + let comment = collect_rar50_file(raw, raw.services().next().unwrap()).unwrap(); + assert_eq!( + comment.data, + b"facade header encrypted compressed comment\n" + ); + let extracted = collect_extract(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "recovery")] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_stored_archive_with_recovery() { + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar5-encrypted-recovery.txt", + b"facade rar5 encrypted recovery payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .recovery_percent(Some(6)) + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.has_recovery_record()); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 6); + let recovery_data = collect_rar50_file(raw, service).unwrap().data; + assert!(recovery_data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(recovery_data[0x0c..0x10].try_into().unwrap()) as usize, + recovery_data.len() + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar5 encrypted recovery payload\n" + ); + } + + #[test] + #[cfg(feature = "recovery")] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_compressed_archive_with_recovery() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 encrypted compressed recovery payload repeated\n".repeat(8); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-encrypted-compressed-recovery.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .recovery_percent(Some(6)) + .finish() + .unwrap(); + + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.has_recovery_record()); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 6); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "recovery")] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_stored_archive_with_recovery() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = rar50::Rar50Writer::new( + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries( + [rar50_entry( + b"rar5-header-recovery.txt", + b"facade rar5 header encrypted recovery payload\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())] + .to_vec(), + ) + .recovery_percent(Some(4)) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.has_recovery_record()); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert!(!service.encrypted); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 4); + let recovery_data = collect_rar50_file(raw, service).unwrap().data; + assert!(recovery_data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(recovery_data[0x0c..0x10].try_into().unwrap()) as usize, + recovery_data.len() + ); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!( + extracted[0].data, + b"facade rar5 header encrypted recovery payload\n" + ); + } + + #[test] + #[cfg(feature = "recovery")] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_compressed_archive_with_recovery() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let payload = + b"facade rar5 header encrypted compressed recovery payload repeated\n".repeat(8); + let bytes = + rar50::Rar50Writer::new(rar50_options_with_features(ArchiveVersion::Rar50, features)) + .entries( + [ + rar50_entry(b"rar5-header-compressed-recovery.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ] + .to_vec(), + ) + .recovery_percent(Some(4)) + .finish() + .unwrap(); + + assert!(matches!( + ArchiveReader::read(&bytes), + Err(Error::NeedPassword) + )); + let archive = ArchiveReader::read_with_options( + &bytes, + ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.has_recovery_record()); + let service = raw.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert!(!service.encrypted); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 4); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_stored_volumes() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 encrypted split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-secret50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())], + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + 16, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-secret50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_encrypted_stored_volumes_with_recovery() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 encrypted recovery split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-secret50-rr.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())], + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + 16, + Some(8), + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert_rar50_volume_recovery_records(&archives, 8); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-secret50-rr.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_header_encrypted_stored_volumes_with_recovery() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let payload = b"facade rar5 header encrypted recovery split payload\n".repeat(4); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-header-secret50-rr.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())], + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + 16, + Some(8), + ); + assert!(matches!( + rar50::Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert_rar50_volume_recovery_records(&archives, 8); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-header-secret50-rr.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_stored_volumes() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let payload = b"facade rar5 header encrypted split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-header-secret50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())], + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + 16, + None, + ); + assert!(matches!( + rar50::Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-header-secret50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_compressed_volumes() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 encrypted compressed split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-secret-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())], + rar50_options_with_features(ArchiveVersion::Rar50, features), + 32, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-secret-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_encrypted_compressed_volumes_with_recovery() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 encrypted compressed recovery split payload\n".repeat(12); + let entries = [rar50_entry(b"split-secret-compressed50-rr.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let parts = write_rar50_volume_set( + &entries, + rar50_options_with_features(ArchiveVersion::Rar50, features), + 32, + Some(8), + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert_rar50_volume_recovery_records(&archives, 8); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-secret-compressed50-rr.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_header_encrypted_compressed_volumes_with_recovery() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let payload = + b"facade rar5 header encrypted compressed recovery split payload\n".repeat(12); + let entries = [ + rar50_entry(b"split-header-secret-compressed50-rr.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ]; + let parts = write_rar50_volume_set( + &entries, + rar50_options_with_features(ArchiveVersion::Rar50, features), + 32, + Some(8), + ); + assert!(matches!( + rar50::Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert_rar50_volume_recovery_records(&archives, 8); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!( + extracted[0].name, + b"split-header-secret-compressed50-rr.txt" + ); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_encrypted_solid_compressed_volumes() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let payload = b"facade rar5 encrypted solid compressed split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[ + rar50_entry(b"split-solid-secret-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ], + rar50_options_with_features(ArchiveVersion::Rar50, features), + 32, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + + let extracted = collect_rar50_volumes(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-solid-secret-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_compressed_volumes() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let payload: Vec = (0..512).map(|index| (index * 37 + 11) as u8).collect(); + let parts = write_rar50_volume_set( + &[ + rar50_entry(b"split-header-secret-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ], + rar50_options_with_features(ArchiveVersion::Rar50, features), + 64, + None, + ); + assert!(matches!( + rar50::Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + assert_eq!(extracted[0].name, b"split-header-secret-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn direct_writer_creates_rar50_header_encrypted_solid_compressed_volumes() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let payload = b"facade rar5 header encrypted solid compressed split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[ + rar50_entry(b"split-header-solid-secret-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ], + rar50_options_with_features(ArchiveVersion::Rar50, features), + 32, + None, + ); + assert!(matches!( + rar50::Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + assert_eq!( + extracted[0].name, + b"split-header-solid-secret-compressed50.txt" + ); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_stored_volumes() { + let payload = b"facade rar5 stored split payload\n".repeat(20); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)], + rar50_options(ArchiveVersion::Rar50).with_compression_level(0), + 80, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"split50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_stored_volumes_with_recovery() { + let features = FeatureSet::store_only(); + let payload = b"facade rar5 stored recovery split payload\n".repeat(20); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split50-rr.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)], + rar50_options_with_features(ArchiveVersion::Rar50, features).with_compression_level(0), + 80, + Some(8), + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + assert_rar50_volume_recovery_records(&archives, 8); + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"split50-rr.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_compressed_volumes() { + let payload: Vec = (0..512).map(|index| (index * 53 + 17) as u8).collect(); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)], + rar50_options(ArchiveVersion::Rar50), + 64, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"split-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + #[cfg(feature = "recovery")] + fn direct_writer_creates_rar50_compressed_volumes_with_recovery() { + let features = FeatureSet::store_only(); + let payload: Vec = (0..512).map(|index| (index * 53 + 17) as u8).collect(); + let entries = [rar50_entry(b"split-compressed50-rr.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)]; + let parts = write_rar50_volume_set( + &entries, + rar50_options_with_features(ArchiveVersion::Rar50, features), + 64, + Some(8), + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + assert_rar50_volume_recovery_records(&archives, 8); + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"split-compressed50-rr.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_solid_compressed_volumes() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let payload = b"facade rar5 solid compressed split payload\n".repeat(12); + let parts = write_rar50_volume_set( + &[rar50_entry(b"split-solid-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)], + rar50_options_with_features(ArchiveVersion::Rar50, features), + 32, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"split-solid-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn direct_writer_creates_rar50_multi_file_solid_compressed_volumes() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let mut first = b"facade rar5 multi-file solid split shared phrase\n" + .repeat(8) + .to_vec(); + first.extend_from_slice(&deterministic_noise(2048)); + let mut second = b"facade rar5 multi-file solid split shared phrase\nsecond\n" + .repeat(8) + .to_vec(); + second.extend_from_slice(&deterministic_noise(2048)); + let entries = [ + rar50_entry(b"solid-volume-one.txt", &first) + .with_attributes(0x20) + .with_host_os(3), + rar50_entry(b"solid-volume-two.txt", &second) + .with_attributes(0x20) + .with_host_os(3), + ]; + let parts = write_rar50_volume_set( + &entries, + rar50_options_with_features(ArchiveVersion::Rar50, features), + 512, + None, + ); + let archives: Vec<_> = parts + .iter() + .map(|part| rar50::Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let extracted = collect_rar50_volumes(&archives, None).unwrap(); + + assert_eq!(extracted[0].name, b"solid-volume-one.txt"); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].name, b"solid-volume-two.txt"); + assert_eq!(extracted[1].data, second); + } + + #[test] + fn archive_as_rar13_returns_some_only_for_rar13_family() { + let bytes = rar13::write_stored_archive( + &[rar13::StoredEntry { + name: b"old.txt", + data: b"r13 downcast", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }], + rar13_options(ArchiveVersion::Rar14), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + let raw = archive.as_rar13().unwrap(); + assert_eq!(raw.entries[0].name, b"old.txt"); + assert!(archive.as_rar15_40().is_none()); + assert!(archive.as_rar50().is_none()); + + // Other-family archives should refuse the rar13 downcast. + let rar15_bytes = rar15_40::write_stored_archive( + &[rar15_40::StoredEntry { + name: b"mid.txt", + data: b"r15 downcast", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + rar15_options(ArchiveVersion::Rar15), + ) + .unwrap(); + let rar15_archive = ArchiveReader::read(&rar15_bytes).unwrap(); + assert!(rar15_archive.as_rar13().is_none()); + + let rar50_bytes = + rar50::Rar50Writer::new(rar50_options(ArchiveVersion::Rar50).with_compression_level(0)) + .entries( + [rar50_entry(b"new.txt", b"r50 downcast") + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .finish() + .unwrap(); + let rar50_archive = ArchiveReader::read(&rar50_bytes).unwrap(); + assert!(rar50_archive.as_rar13().is_none()); + } + + #[test] + #[cfg(feature = "recovery")] + fn archive_facade_repair_recovery_returns_full_repaired_archive_bytes() { + let bytes = std::fs::read(rar15_40_fixture("rar250_protect_head_rr5.rar")).unwrap(); + let mut damaged = bytes.clone(); + damaged[512 + 16..512 + 80].fill(0xa5); + let damaged_archive = ArchiveReader::read(&damaged).unwrap(); + + let repaired = damaged_archive.repair_recovery().unwrap(); + assert_eq!(repaired, bytes); + } + + #[test] + #[cfg(feature = "recovery")] + fn archive_facade_repair_recovery_rejects_rar13_archives() { + let bytes = rar13::write_stored_archive( + &[rar13::StoredEntry { + name: b"old.txt", + data: b"old data", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }], + rar13_options(ArchiveVersion::Rar14), + ) + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + assert_eq!( + archive.repair_recovery(), + Err(Error::UnsupportedFamilyFeature { + family: ArchiveFamily::Rar13, + feature: "recovery repair for RAR 1.3/1.4 archives", + }) + ); + } + + #[test] + fn archive_reader_read_path_dispatches_to_default_options() { + // Existing tests cover read_path_with_options; this ensures the + // zero-arg convenience wrapper actually delegates to it. + let archive = + ArchiveReader::read_path(rar15_40_fixture("rar250_protect_head_rr5.rar")).unwrap(); + assert_eq!(archive.family(), ArchiveFamily::Rar15To40); + assert!(archive.as_rar15_40().unwrap().main.has_recovery_record()); + } + + #[test] + fn archive_member_can_be_read_by_index() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"first.txt".to_vec(), b"first".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"second.txt".to_vec(), b"second".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + + assert_eq!(archive.read_member_at(1, None).unwrap().unwrap(), b"second"); + assert_eq!(archive.read_member_at(2, None).unwrap(), None); + } + + #[test] + fn volume_facade_rejects_empty_and_every_mixed_family_ordering() { + let never_open = |_: &ExtractedEntryMeta| -> Result> { + panic!("invalid volume set must be rejected before opening output") + }; + let empty = Error::InvalidHeader("volume set is empty"); + assert_eq!(volume_members(&[]).unwrap_err(), empty); + assert_eq!(read_volume_members_at(&[], &[], None).unwrap_err(), empty); + assert_eq!( + extract_volumes_to(&[], None, never_open).unwrap_err(), + empty + ); + let cancellation = ReadCancellation::new(); + cancellation.cancel(); + assert_eq!( + extract_volumes_to_with_options( + &[], + ArchiveReadOptions::new().with_cancellation(&cancellation), + never_open, + ) + .unwrap_err(), + Error::Cancelled + ); + + let archives: Vec<_> = [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] + .into_iter() + .map(|version| { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + }) + .collect(); + for first in 0..archives.len() { + for second in 0..archives.len() { + if first == second { + continue; + } + let mixed = [archives[first].clone(), archives[second].clone()]; + let error = Error::InvalidHeader("mixed archive families in volume set"); + assert_eq!(volume_members(&mixed).unwrap_err(), error); + assert_eq!( + extract_volumes_to(&mixed, None, never_open).unwrap_err(), + error + ); + } + } + } + + #[test] + fn volume_index_reads_skip_directories_and_absent_indices_in_each_family() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archives = [ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap()]; + assert_eq!(read_volume_member_at(&archives, 0, None).unwrap(), None); + assert_eq!( + read_volume_member_at(&archives, 1, None).unwrap(), + Some(b"payload".to_vec()) + ); + assert_eq!(read_volume_member_at(&archives, 2, None).unwrap(), None); + } + } + + #[test] + fn volume_members_and_index_reads_fold_split_fragments() { + let payload = vec![7; 200_000]; + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .volume_size(Some(64 * 1024)); + builder + .add_bytes(b"big.bin".to_vec(), payload.clone(), None, None) + .unwrap(); + let archives: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|part| ArchiveReader::read_owned(part).unwrap()) + .collect(); + + let members = volume_members(&archives).unwrap(); + assert_eq!( + volume_members(&archives[1..]).unwrap_err(), + Error::InvalidHeader("volume set starts with a continuation") + ); + assert_eq!(members.len(), 1); + assert_eq!(members[0].meta.name, b"big.bin"); + assert_eq!( + read_volume_member_at(&archives, 0, None).unwrap().unwrap(), + payload + ); + assert_eq!( + read_volume_members_at(&archives, &[0, 0, usize::MAX], None).unwrap(), + vec![Some(payload.clone()), Some(payload), None] + ); + } +} diff --git a/arca-rar/vendor/rars/src/member.rs b/arca-rar/vendor/rars/src/member.rs new file mode 100644 index 00000000..2c85476e --- /dev/null +++ b/arca-rar/vendor/rars/src/member.rs @@ -0,0 +1,276 @@ +//! Borrowed member traversal without copying names or format-specific records. + +use crate::{rar13, rar15_40, rar50, Archive, ArchiveMember, Result}; +use std::borrow::Cow; +use std::collections::HashMap; + +/// A member header borrowed from its archive, including all format metadata. +/// +/// Use [`Self::to_owned`] when the metadata must outlive the archive borrow. +#[derive(Debug, Clone, Copy)] +#[non_exhaustive] +pub enum ArchiveMemberRef<'a> { + /// RAR 1.3/1.4 member. + Rar13(&'a rar13::Entry), + /// RAR 1.5 through 4.x member. + Rar15To40(&'a rar15_40::FileHeader), + /// RAR 5.0 or later member. + Rar50Plus(&'a rar50::FileHeader), +} + +impl<'a> ArchiveMemberRef<'a> { + /// Exact name bytes, without copying or interpreting legacy encodings. + pub fn name_bytes(self) -> &'a [u8] { + match self { + Self::Rar13(entry) => &entry.name, + Self::Rar15To40(file) => &file.name, + Self::Rar50Plus(file) => &file.name, + } + } + + /// Whether the member is a directory. + pub fn is_directory(self) -> bool { + match self { + Self::Rar13(entry) => entry.is_directory(), + Self::Rar15To40(file) => file.is_directory(), + Self::Rar50Plus(file) => file.is_directory(), + } + } + + /// Whether the member carries a RAR5 redirection record. + pub fn is_redirection(self) -> bool { + matches!(self, Self::Rar50Plus(file) if file.is_redirection()) + } + + /// Whether this header continues a member from an earlier volume. + pub fn is_split_before(self) -> bool { + match self { + Self::Rar13(entry) => entry.is_split_before(), + Self::Rar15To40(file) => file.is_split_before(), + Self::Rar50Plus(file) => file.is_split_before(), + } + } + + /// Whether this member declares a dependency on preceding decoded history. + /// RAR 1.3 has no member flag; consult its archive main header. + pub fn is_solid(self) -> bool { + match self { + Self::Rar13(_) => false, + Self::Rar15To40(file) => file.is_solid(), + Self::Rar50Plus(file) => file.compression_info & 0x40 != 0, + } + } + + /// Whether the format supplied Unicode rather than unspecified legacy bytes. + pub fn name_is_unicode(self) -> bool { + match self { + Self::Rar13(_) => false, + Self::Rar15To40(file) => file.unicode_name.is_some(), + Self::Rar50Plus(_) => true, + } + } + + /// Display/destination name interpretation; stored identity remains unchanged. + pub fn decoded_name( + self, + encoding: Option, + ) -> Result> { + crate::filename::decoded_name(self.name_bytes(), self.name_is_unicode(), encoding) + } + + /// Copies this member's common and format-specific metadata. + pub fn to_owned(&self) -> ArchiveMember { + match *self { + Self::Rar13(entry) => crate::rar13_member(entry), + Self::Rar15To40(file) => crate::rar15_40_member(file), + Self::Rar50Plus(file) => crate::rar50_member(file), + } + } +} + +/// Lazy borrowed traversal returned by [`Archive::member_refs`]. +#[derive(Debug, Clone)] +pub struct ArchiveMemberRefs<'a> { + inner: MemberRefsInner<'a>, +} + +#[derive(Debug, Clone)] +enum MemberRefsInner<'a> { + Rar13(std::slice::Iter<'a, rar13::Entry>), + Rar15To40(std::slice::Iter<'a, rar15_40::Block>), + Rar50Plus(std::slice::Iter<'a, rar50::Block>), +} + +impl<'a> ArchiveMemberRefs<'a> { + pub(crate) fn new(archive: &'a Archive) -> Self { + let inner = match archive { + Archive::Rar13(archive) => MemberRefsInner::Rar13(archive.entries.iter()), + Archive::Rar15To40(archive) => MemberRefsInner::Rar15To40(archive.blocks.iter()), + Archive::Rar50Plus(archive) => MemberRefsInner::Rar50Plus(archive.blocks.iter()), + }; + Self { inner } + } +} + +impl<'a> Iterator for ArchiveMemberRefs<'a> { + type Item = ArchiveMemberRef<'a>; + + fn next(&mut self) -> Option { + match &mut self.inner { + MemberRefsInner::Rar13(entries) => entries.next().map(ArchiveMemberRef::Rar13), + MemberRefsInner::Rar15To40(blocks) => blocks.find_map(|block| match block { + rar15_40::Block::File(file) => Some(ArchiveMemberRef::Rar15To40(file)), + _ => None, + }), + MemberRefsInner::Rar50Plus(blocks) => blocks.find_map(|block| match block { + rar50::Block::File(file) => Some(ArchiveMemberRef::Rar50Plus(file)), + _ => None, + }), + } + } +} + +/// An optional index borrowing immutable archive headers and name bytes. +/// +/// Construction traverses headers once and allocates caller-owned index storage. +/// Indexed metadata lookup does not allocate or traverse preceding members. +/// Duplicate entries retain their archive-order indices. Name lookup selects the +/// last entry, while payload lookup excludes directories and redirections. +/// Payload decoding still uses the archive's extraction policies. +#[derive(Debug)] +pub struct ArchiveIndex<'a> { + members: Vec>, + names: HashMap<&'a [u8], NameIndices>, +} + +#[derive(Debug, Default)] +struct NameIndices { + last: usize, + payload: Option, +} + +impl<'a> ArchiveIndex<'a> { + pub(crate) fn new(archive: &'a Archive) -> Self { + let members: Vec<_> = archive.member_refs().collect(); + let mut names: HashMap<&[u8], NameIndices> = HashMap::new(); + for (index, member) in members.iter().enumerate() { + let indices = names.entry(member.name_bytes()).or_default(); + indices.last = index; + if !member.is_directory() && !member.is_redirection() { + indices.payload = Some(index); + } + } + Self { members, names } + } + + /// Number of members, including directories and redirections. + pub fn len(&self) -> usize { + self.members.len() + } + + /// Whether the archive contains no members. + pub fn is_empty(&self) -> bool { + self.members.is_empty() + } + + /// Borrows metadata for an archive-order index without copying it. + pub fn get(&self, index: usize) -> Option> { + self.members.get(index).copied() + } + + /// Archive-order index of the last entry with this exact stored name. + pub fn index_of(&self, name: &[u8]) -> Option { + self.names.get(name).map(|indices| indices.last) + } + + /// Last payload index, matching [`Archive::read_member`] selection. + /// A later directory or redirection with the same name does not hide a file. + pub fn payload_index_of(&self, name: &[u8]) -> Option { + self.names.get(name).and_then(|indices| indices.payload) + } + + /// All borrowed member headers in archive order. + pub fn members(&self) -> &[ArchiveMemberRef<'a>] { + &self.members + } +} + +#[cfg(all(test, feature = "write"))] +mod tests { + use crate::{ArchiveReader, ArchiveVersion, Builder}; + + #[test] + fn borrowed_members_preserve_metadata_and_archive_order_across_families() { + for version in ArchiveVersion::ALL { + let mut builder = Builder::new(version).store(true); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"dir/file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let members: Vec<_> = archive.member_refs().collect(); + assert_eq!(members.len(), 2, "{version}"); + assert!(members[0].is_directory()); + assert!(!members[1].is_directory()); + assert_eq!(members[1].name_bytes(), b"dir/file"); + for (borrowed, owned) in members.iter().zip(archive.members()) { + assert_eq!(borrowed.to_owned(), owned); + assert_eq!(borrowed.is_solid(), owned.is_solid()); + assert_eq!(borrowed.name_is_unicode(), owned.name_is_unicode()); + assert_eq!(borrowed.is_redirection(), owned.meta.is_redirection); + assert_eq!( + borrowed.decoded_name(None).unwrap().as_ref(), + owned.meta.name + ); + assert!(std::ptr::eq( + borrowed.name_bytes().as_ptr(), + archive + .member_refs() + .find(|m| m.name_bytes() == borrowed.name_bytes()) + .unwrap() + .name_bytes() + .as_ptr(), + )); + } + let mut owned = archive.members(); + assert_eq!(owned.nth(1).unwrap().meta.name, b"dir/file"); + assert_eq!(owned.next(), None); + assert_eq!( + archive.read_member_at(1, None).unwrap().unwrap(), + b"payload" + ); + } + } + + #[test] + fn member_index_preserves_duplicates_and_distinguishes_payload_lookup() { + for version in ArchiveVersion::ALL { + let mut builder = Builder::new(version) + .store(true) + .allow_duplicate_names(true); + for data in [b"first".as_slice(), b"second"] { + builder + .add_bytes(b"same".to_vec(), data.to_vec(), None, None) + .unwrap(); + } + builder.add_directory(b"same".to_vec(), None, None).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let index = archive.index(); + assert_eq!(index.len(), 3, "{version}"); + assert!(!index.is_empty()); + assert_eq!(index.index_of(b"same"), Some(2)); + assert_eq!(index.payload_index_of(b"same"), Some(1)); + assert_eq!(index.index_of(b"missing"), None); + assert_eq!(index.payload_index_of(b"missing"), None); + assert_eq!(index.get(0).unwrap().to_owned().meta.unpacked_size, 5); + assert_eq!(index.get(1).unwrap().to_owned().meta.unpacked_size, 6); + assert!(index.get(2).unwrap().is_directory()); + assert!(index.get(3).is_none()); + assert_eq!(index.members().len(), 3); + assert_eq!( + archive.read_member(b"same", None).unwrap().unwrap(), + b"second" + ); + } + } +} diff --git a/arca-rar/vendor/rars/src/member_read.rs b/arca-rar/vendor/rars/src/member_read.rs new file mode 100644 index 00000000..409a7069 --- /dev/null +++ b/arca-rar/vendor/rars/src/member_read.rs @@ -0,0 +1,206 @@ +//! Several selected payloads decoded through one extraction session. + +use crate::{Archive, ArchiveReadOptions, ExtractionDecision, Result, SharedBuffer}; +use std::collections::BTreeMap; +use std::io::Write; +use std::sync::{Arc, Mutex}; + +type Bytes = Arc>>>; + +struct SelectedBuffer { + first_position: usize, + duplicates: Vec, + bytes: Bytes, +} + +struct ReadBatch { + buffers: BTreeMap, + length: usize, +} + +impl ReadBatch { + fn new(indices: &[usize]) -> Self { + let mut buffers = BTreeMap::::new(); + for (position, &index) in indices.iter().enumerate() { + buffers + .entry(index) + .and_modify(|buffer| buffer.duplicates.push(position)) + .or_insert_with(|| SelectedBuffer { + first_position: position, + duplicates: Vec::new(), + bytes: Arc::new(Mutex::new(None)), + }); + } + Self { + buffers, + length: indices.len(), + } + } + + fn contains(&self, index: usize) -> bool { + self.buffers.contains_key(&index) + } + + fn writer(&self, index: usize) -> Option> { + let buffer = self.buffers.get(&index)?; + *buffer + .bytes + .lock() + .expect("member buffer mutex is not poisoned") = Some(Vec::new()); + Some(Box::new(SharedBuffer(Arc::clone(&buffer.bytes)))) + } + + fn finish(self) -> Vec>> { + let mut result = vec![None; self.length]; + for buffer in self.buffers.into_values() { + if let Some(bytes) = buffer + .bytes + .lock() + .expect("member buffer mutex is not poisoned") + .take() + { + for position in buffer.duplicates { + result[position] = Some(bytes.clone()); + } + // Unique selections transfer their output without a payload copy. + result[buffer.first_position] = Some(bytes); + } + } + result + } +} + +impl Archive { + /// Whether archive or member headers declare solid decoding dependencies. + /// This uses the same conservative policy as controlled extraction. + pub fn is_solid(&self) -> bool { + let main_solid = match self { + Self::Rar13(archive) => archive.main.is_solid(), + Self::Rar15To40(archive) => archive.main.is_solid(), + Self::Rar50Plus(archive) => archive.main.is_solid(), + }; + main_solid || self.member_refs().any(|member| member.is_solid()) + } + + /// Reads several members by archive-order index using one decoder session. + /// See [`Self::read_members_at_with_options`] for selection and budget rules. + pub fn read_members_at( + &self, + indices: &[usize], + password: Option<&[u8]>, + ) -> Result>>> { + self.read_members_at_with_options( + indices, + ArchiveReadOptions::with_optional_password(password), + ) + } + + /// Reads selected payloads in one pass, avoiding repeated solid-prefix decoding. + /// + /// Results follow the requested order, including duplicate indices. Missing + /// members, directories and redirections return `None`. Each selected payload + /// is decoded once; repeated results copy its verified bytes. Independent + /// unselected payloads are skipped. Solid predecessors are decoded and + /// verified once, even when their output is discarded. + /// + /// Output budgets are shared across this call and include all decoded bytes; + /// returning another copy of a selected payload does not decode it again or + /// consume logical output quota again. Result buffers are caller output, not + /// decoder workspace. No buffers are returned if any selected payload or + /// required predecessor fails. Cancellation is checked for empty selections. + pub fn read_members_at_with_options( + &self, + indices: &[usize], + options: ArchiveReadOptions<'_>, + ) -> Result>>> { + options.check_cancelled()?; + let batch = ReadBatch::new(indices); + if indices.is_empty() { + return Ok(batch.finish()); + } + let last = self + .member_refs() + .enumerate() + .filter(|(index, member)| { + batch.contains(*index) && !member.is_directory() && !member.is_redirection() + }) + .map(|(index, _)| index) + .last(); + let Some(last) = last else { + return Ok(batch.finish()); + }; + let solid = self.is_solid(); + let mut current = 0; + self.extract_with_control(options, |member| { + let index = current; + current += 1; + if index > last { + return Ok(ExtractionDecision::Stop); + } + if member.meta.is_directory || member.meta.is_redirection { + return Ok(ExtractionDecision::Skip); + } + if let Some(writer) = batch.writer(index) { + return Ok(ExtractionDecision::Extract(writer)); + } + Ok(if solid { + ExtractionDecision::Extract(Box::new(std::io::sink())) + } else { + ExtractionDecision::Skip + }) + })?; + Ok(batch.finish()) + } +} + +// Redirections have logical indices but do not open volume extraction writers. +// Continuation headers share the logical index of their first fragment. +pub(crate) fn volume_output_indices(archives: &[Archive]) -> impl Iterator + '_ { + archives + .iter() + .flat_map(Archive::member_refs) + .filter(|member| !member.is_split_before()) + .enumerate() + .filter_map(|(index, member)| (!member.is_redirection()).then_some(index)) +} + +/// Reads several logical volume members in one traversal of the volume set. +/// See [`read_volume_members_at_with_options`] for budget and selection rules. +pub fn read_volume_members_at( + archives: &[Archive], + indices: &[usize], + password: Option<&[u8]>, +) -> Result>>> { + read_volume_members_at_with_options( + archives, + indices, + ArchiveReadOptions::with_optional_password(password), + ) +} + +/// Reads selected logical volume members, preserving order and duplicate indices. +/// +/// Missing members, directories and redirections return `None`. The current +/// volume traversal decodes and verifies the whole set, including unselected +/// payloads and empty selections; discarded output counts against shared quotas. +/// Each payload is decoded once. Copies returned for repeated indices do not +/// consume additional logical output quota. No result buffers escape on failure. +pub fn read_volume_members_at_with_options( + archives: &[Archive], + indices: &[usize], + options: ArchiveReadOptions<'_>, +) -> Result>>> { + options.check_cancelled()?; + let batch = ReadBatch::new(indices); + let mut output_indices = volume_output_indices(archives); + crate::extract_volumes_to_with_options(archives, options, |meta| { + let index = output_indices.next(); + if !meta.is_directory { + if let Some(writer) = index.and_then(|index| batch.writer(index)) { + return Ok(writer); + } + } + Ok(Box::new(std::io::sink())) + })?; + Ok(batch.finish()) +} diff --git a/arca-rar/vendor/rars/src/output_limit.rs b/arca-rar/vendor/rars/src/output_limit.rs new file mode 100644 index 00000000..6a0dd71c --- /dev/null +++ b/arca-rar/vendor/rars/src/output_limit.rs @@ -0,0 +1,483 @@ +//! Output admission and a writer guard whose error survives I/O adapters. +#[cfg(test)] +use crate::ArchiveReadOptions; +use crate::{Error, Result}; +use std::io::{self, Write}; + +pub(crate) fn check(limit: Option, required: u64, name: &[u8]) -> Result<()> { + if let Some(limit) = limit.filter(|limit| required > *limit) { + return Err(error(limit, required, name)); + } + Ok(()) +} + +fn error(limit: u64, required: u64, name: &[u8]) -> Error { + Error::MemberOutputLimitExceeded { limit, required }.at_entry(name.to_vec(), "limiting output") +} + +/// Owned by one sequential extraction operation, including all its volumes. +/// Declarations admit work; only bytes accepted by the logical output sink charge it. +pub(crate) struct OutputBudget { + pub(crate) control: crate::read_control::ReadControl, + member_limit: Option, + total_limit: Option, + used: u64, +} + +impl OutputBudget { + pub(crate) fn new(options: crate::ArchiveReadOptions<'_>) -> Self { + Self { + control: crate::read_control::ReadControl::new(options.cancellation), + member_limit: options.max_member_output_bytes, + total_limit: options.max_total_output_bytes, + used: 0, + } + } + + pub(crate) fn is_limited(&self) -> bool { + self.member_limit.is_some() || self.total_limit.is_some() + } + + pub(crate) fn check(&self, required: u64, name: &[u8]) -> Result<()> { + self.control.check()?; + check(self.member_limit, required, name)?; + if let Some(limit) = self.total_limit { + if required > limit - self.used { + return Err(self + .total_error(limit, required) + .at_entry(name.to_vec(), "limiting output")); + } + } + Ok(()) + } + + fn total_error(&self, limit: u64, additional: u64) -> Error { + Error::TotalOutputLimitExceeded { + limit, + required: self.used.saturating_add(additional), + used: self.used, + } + } + + pub(crate) fn run( + &mut self, + name: &[u8], + writer: W, + work: impl FnOnce(&mut LimitedWriter<'_, W>) -> Result, + ) -> Result { + self.control.check()?; + let mut guarded = LimitedWriter { + writer, + budget: self, + written: 0, + exceeded: None, + sink_error: None, + }; + let result = work(&mut guarded); + // Older decoders/error adapters may wrap or replace the sentinel. Keep + // the refusal out of band so it cannot become bad-password/checksum/I/O. + // Also preserve destination failures that legacy codecs turn into + // generic decode errors, including the original I/O source. + match (guarded.exceeded, guarded.sink_error) { + (Some(error), _) => Err(error.at_entry(name.to_vec(), "limiting output")), + (None, Some(error)) => Err(error.at_entry(name.to_vec(), "extracting")), + (None, None) => guarded + .budget + .control + .finish(result) + .and_then(|value| guarded.budget.control.check().map(|()| value)) + .map_err(|error| { + if error.kind() == crate::ErrorKind::Cancelled + && error.entry_context().is_none() + { + error.at_entry(name.to_vec(), "extracting") + } else { + error + } + }), + } + } +} + +#[cfg(test)] +pub(crate) fn run( + limit: Option, + name: &[u8], + writer: W, + work: impl FnOnce(&mut LimitedWriter<'_, W>) -> Result, +) -> Result { + let mut options = ArchiveReadOptions::new(); + options.max_member_output_bytes = limit; + OutputBudget::new(options).run(name, writer, work) +} + +pub(crate) struct LimitedWriter<'a, W> { + writer: W, + budget: &'a mut OutputBudget, + written: u64, + exceeded: Option, + sink_error: Option, +} + +impl LimitedWriter<'_, W> { + fn remember_sink_result(&mut self, result: io::Result) -> io::Result { + result.map_err(|error| { + // write_all retries Interrupted; it is not a terminal sink failure. + if error.kind() == io::ErrorKind::Interrupted { + return error; + } + let kind = error.kind(); + let error = Error::from(error); + self.sink_error.get_or_insert_with(|| error.clone()); + io::Error::new(kind, error) + }) + } + + fn write_sink(&mut self, bytes: &[u8]) -> io::Result { + let result = match self.writer.write(bytes) { + Ok(0) if !bytes.is_empty() => Err(io::ErrorKind::WriteZero.into()), + result => result, + }; + self.remember_sink_result(result) + } +} + +impl Write for LimitedWriter<'_, W> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.budget.control.check().map_err(io::Error::other)?; + if !self.budget.is_limited() { + let len = if self.budget.control.is_enabled() { + bytes.len().min(64 * 1024) + } else { + bytes.len() + }; + return self.write_sink(&bytes[..len]); + } + if self.exceeded.is_none() { + let count = bytes.len() as u64; + if let Some(limit) = self.budget.member_limit { + if count > limit - self.written { + self.exceeded = Some(Error::MemberOutputLimitExceeded { + limit, + required: self.written.saturating_add(count), + }); + } + } + if self.exceeded.is_none() { + if let Some(limit) = self.budget.total_limit { + if count > limit - self.budget.used { + self.exceeded = Some(self.budget.total_error(limit, count)); + } + } + } + } + if self.exceeded.is_some() { + return Err(io::Error::other("output limit exceeded")); + } + let len = if self.budget.control.is_enabled() { + bytes.len().min(64 * 1024) + } else { + bytes.len() + }; + let n = self.write_sink(&bytes[..len])?; + if self.budget.member_limit.is_some() { + self.written += n as u64; + } + if self.budget.total_limit.is_some() { + self.budget.used += n as u64; + } + Ok(n) + } + fn flush(&mut self) -> io::Result<()> { + self.budget.control.check().map_err(io::Error::other)?; + let result = self.writer.flush(); + self.remember_sink_result(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interrupted_sink_writes_retry_without_latching_a_failure() { + struct InterruptedOnce(bool); + impl Write for InterruptedOnce { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if std::mem::replace(&mut self.0, false) { + Err(io::ErrorKind::Interrupted.into()) + } else { + Ok(bytes.len()) + } + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let mut budget = + OutputBudget::new(ArchiveReadOptions::new().with_max_total_output_bytes(3)); + budget + .run(b"member", InterruptedOnce(true), |w| { + w.write_all(b"abc").map_err(Error::from) + }) + .unwrap(); + assert_eq!(budget.used, 3); + } + + #[test] + fn zero_writes_and_flush_errors_survive_lossy_codec_adapters() { + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> io::Result { + Ok(0) + } + fn flush(&mut self) -> io::Result<()> { + Err(io::ErrorKind::PermissionDenied.into()) + } + } + for flush in [false, true] { + let mut budget = OutputBudget::new(ArchiveReadOptions::new()); + let error = budget + .run(b"member", Broken, |w| { + assert_eq!(w.write(b"").unwrap(), 0); + if flush { + assert!(w.flush().is_err()); + } else { + assert!(w.write_all(b"a").is_err()); + } + Err::<(), _>(Error::InvalidHeader("lossy adapter")) + }) + .unwrap_err(); + let expected = if flush { + io::ErrorKind::PermissionDenied + } else { + io::ErrorKind::WriteZero + }; + assert!(matches!(error.root_cause(), Error::Io(e) if e.kind == expected)); + assert_eq!(error.entry_context().unwrap().0, b"member"); + } + } + + #[test] + fn cancellation_survives_adapters_without_refunding_or_hiding_quota_refusal() { + let token = crate::ReadCancellation::new(); + let mut budget = OutputBudget::new( + ArchiveReadOptions::new() + .with_cancellation(&token) + .with_max_total_output_bytes(4), + ); + let err = budget + .run(b"member", io::sink(), |w| { + w.write_all(b"ab")?; + token.cancel(); + assert!(w.write_all(b"c").is_err()); + Err::<(), _>(Error::WrongPasswordOrCorruptData) + }) + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::Cancelled); + assert_eq!(err.entry_context().unwrap().0, b"member"); + assert_eq!(budget.used, 2); + + let token = crate::ReadCancellation::new(); + let mut budget = OutputBudget::new( + ArchiveReadOptions::new() + .with_cancellation(&token) + .with_max_total_output_bytes(0), + ); + let err = budget + .run(b"member", io::sink(), |w| { + assert!(w.write_all(b"a").is_err()); + token.cancel(); + assert!(w.write_all(b"b").is_err()); + Err::<(), _>(Error::WrongPasswordOrCorruptData) + }) + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(budget.used, 0); + } + #[test] + fn total_accounting_survives_short_writes_and_later_errors() { + struct Short; + impl Write for Short { + fn write(&mut self, b: &[u8]) -> io::Result { + Ok(b.len().min(1)) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let mut budget = + OutputBudget::new(ArchiveReadOptions::new().with_max_total_output_bytes(6)); + budget.check(6, b"first").unwrap(); + // Admission is not a charge, and short writes do not count offered bytes twice. + assert_eq!(budget.used, 0); + let err = budget + .run(b"first", Short, |w| { + w.write_all(b"abc")?; + Err::<(), _>(Error::Crc32Mismatch { + expected: 1, + actual: 2, + }) + }) + .unwrap_err(); + assert!(matches!(err, Error::Crc32Mismatch { .. })); + assert_eq!(budget.used, 3); + budget.check(3, b"second").unwrap(); + let err = budget + .run(b"second", io::sink(), |w| { + w.write_all(b"d")?; + assert!(w.write_all(b"efg").is_err()); + assert!(w.write_all(b"x").is_err()); + Err::<(), _>(Error::WrongPasswordOrCorruptData) + }) + .unwrap_err(); + let err = Error::InVolume { + number: 2, + source: Box::new(err), + }; + assert_eq!(budget.used, 4); + assert_eq!(err.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(err.entry_context().unwrap().0, b"second"); + assert!(matches!( + err.root_cause(), + Error::TotalOutputLimitExceeded { + limit: 6, + used: 4, + required: 7 + } + )); + } + + #[test] + fn total_admission_and_runtime_use_overflow_safe_arithmetic() { + let mut budget = + OutputBudget::new(ArchiveReadOptions::new().with_max_total_output_bytes(u64::MAX)); + budget.used = u64::MAX - 1; + budget.check(1, b"member").unwrap(); + let err = budget.check(2, b"member").unwrap_err(); + assert!( + matches!(err.root_cause(), Error::TotalOutputLimitExceeded { limit:u64::MAX, required:u64::MAX, used } if *used == u64::MAX - 1) + ); + budget + .run(b"member", io::sink(), |w| { + w.write_all(b"x")?; + Ok(()) + }) + .unwrap(); + budget.check(0, b"empty").unwrap(); + let err = budget + .run(b"member", io::sink(), |w| { + w.write_all(b"x")?; + Ok(()) + }) + .unwrap_err(); + assert!(matches!( + err.root_cause(), + Error::TotalOutputLimitExceeded { used: u64::MAX, .. } + )); + } + + #[test] + fn member_precedence_and_total_only_refusal() { + for (member, expected_member) in [(2, true), (3, false)] { + let mut budget = OutputBudget::new( + ArchiveReadOptions::new() + .with_max_member_output_bytes(member) + .with_max_total_output_bytes(2), + ); + for err in [ + budget.check(3, b"member").unwrap_err(), + budget + .run(b"member", io::sink(), |w| { + w.write_all(b"abc")?; + Ok(()) + }) + .unwrap_err(), + ] { + assert_eq!( + matches!(err.root_cause(), Error::MemberOutputLimitExceeded { .. }), + expected_member + ); + assert_eq!(err.kind(), crate::ErrorKind::ResourceLimit); + } + assert_eq!(budget.used, 0); + } + } + + #[test] + fn total_guard_keeps_real_sink_errors_and_empty_output() { + let mut budget = + OutputBudget::new(ArchiveReadOptions::new().with_max_total_output_bytes(3)); + let mut empty = [].as_mut_slice(); + let err = budget + .run(b"member", &mut empty, |w| { + w.write_all(b"a")?; + Ok(()) + }) + .unwrap_err(); + assert!(matches!(err.root_cause(), Error::Io(e) if e.kind == io::ErrorKind::WriteZero)); + assert_eq!(budget.used, 0); + let mut zero = OutputBudget::new(ArchiveReadOptions::new().with_max_total_output_bytes(0)); + zero.run(b"empty", io::sink(), |w| { + assert_eq!(w.write(b"")?, 0); + Ok(()) + }) + .unwrap(); + assert_eq!(zero.used, 0); + } + #[test] + fn refusal_is_latched_and_restored_after_error_remapping() { + let mut out = Vec::new(); + let err = run(Some(3), b"member", &mut out, |w| { + w.write_all(b"ab")?; + assert!(w.write_all(b"cd").is_err()); + assert!(w.write_all(b"x").is_err()); + Err::<(), _>(Error::WrongPasswordOrCorruptData) + }) + .unwrap_err(); + assert_eq!(out, b"ab"); + assert!(matches!( + err.root_cause(), + Error::MemberOutputLimitExceeded { + limit: 3, + required: 4 + } + )); + assert_eq!(err.kind(), crate::ErrorKind::ResourceLimit); + } + #[test] + fn partial_writes_count_only_accepted_bytes_and_keep_sink_errors() { + struct Short(Vec); + impl Write for Short { + fn write(&mut self, b: &[u8]) -> io::Result { + if self.0.len() == 3 { + return Err(io::ErrorKind::PermissionDenied.into()); + } + let n = b.len().min(1); + self.0.extend_from_slice(&b[..n]); + Ok(n) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let mut out = Short(Vec::new()); + let err = run(Some(4), b"member", &mut out, |w| { + w.write_all(b"abcd")?; + Ok(()) + }) + .unwrap_err(); + assert_eq!(out.0, b"abc"); + assert!( + matches!(err.root_cause(), Error::Io(e) if e.kind == io::ErrorKind::PermissionDenied) + ); + let mut out = Vec::new(); + run(Some(3), b"member", &mut out, |w| { + w.write_all(b"abc")?; + Ok(()) + }) + .unwrap(); + assert_eq!(out, b"abc"); + check(Some(u64::MAX), u64::MAX, b"member").unwrap(); + } +} diff --git a/arca-rar/vendor/rars/src/parallel.rs b/arca-rar/vendor/rars/src/parallel.rs new file mode 100644 index 00000000..760c2fa6 --- /dev/null +++ b/arca-rar/vendor/rars/src/parallel.rs @@ -0,0 +1,376 @@ +//! Fan-out helpers, and the one place that knows whether there is anything to +//! fan out to. +//! +//! Without the `parallel` feature these helpers run sequentially. +//! Bare WebAssembly also has no threads: `wasm32-unknown-unknown` has no way to +//! start one, so rayon's pool panics the first time it is touched rather than +//! degrading. The `sequential` module below is the same three functions done in +//! sequence, and rayon is not compiled in at all for that target. + +#[cfg(any( + not(feature = "parallel"), + all(target_arch = "wasm32", target_os = "unknown") +))] +pub(crate) use sequential::*; +#[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] +pub(crate) use threaded::*; + +#[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] +mod threaded { + use rayon::prelude::*; + + /// Process preallocated slots without allocating a separate result vector. + /// Returns only after every admitted callback has finished. + pub(crate) fn for_each_mut(items: &mut [T], apply: F) + where + T: Send, + F: Fn(usize, &mut T) + Sync + Send, + { + items + .par_iter_mut() + .enumerate() + .for_each(|(index, item)| apply(index, item)); + } + + /// Maps `items` in parallel and returns, in order, the outputs before the + /// first failure together with that failure, so a caller can still publish + /// the successful prefix. + pub(crate) fn map_prefix(items: Vec, map: F) -> (Vec, Option) + where + T: Send, + O: Send, + E: Send, + F: Fn(T) -> Result + Sync + Send, + { + let mut outputs = Vec::with_capacity(items.len()); + for result in items.into_par_iter().map(map).collect::>() { + match result { + Ok(output) => outputs.push(output), + Err(error) => return (outputs, Some(error)), + } + } + (outputs, None) + } + + /// Maps `items` in parallel but keeps only a window of results alive, + /// handing each to `consume` in order before starting the next window. + /// + /// Mapping everything up front is simpler, but it holds every result at + /// once; when the results are compressed file payloads that is the + /// difference between one copy of the archive in memory and two. + #[cfg(any(test, feature = "write"))] + pub(crate) fn map_slice_windowed<'a, T, O, E, F, C>( + items: &'a [T], + window: usize, + map: F, + mut consume: C, + ) -> Result<(), E> + where + T: Sync + 'a, + O: Send, + E: Send, + F: Fn(&'a T) -> Result + Sync + Send, + C: FnMut(&'a T, O) -> Result<(), E>, + { + let window = window.max(1); + for chunk in items.chunks(window) { + let mapped: Vec = chunk.par_iter().map(&map).collect::>()?; + for (item, output) in chunk.iter().zip(mapped) { + consume(item, output)?; + } + } + Ok(()) + } + + /// How many members to keep in flight at once. + pub(crate) fn default_window() -> usize { + threads() + } + + /// How many members can be worked on at the same time. + pub(crate) fn threads() -> usize { + rayon::current_num_threads().max(1) + } +} + +#[cfg(any( + test, + not(feature = "parallel"), + all(target_arch = "wasm32", target_os = "unknown") +))] +mod sequential { + /// Process preallocated slots without allocating a separate result vector. + /// Returns only after every admitted callback has finished. + pub(crate) fn for_each_mut(items: &mut [T], apply: F) + where + T: Send, + F: Fn(usize, &mut T) + Sync + Send, + { + items + .iter_mut() + .enumerate() + .for_each(|(index, item)| apply(index, item)); + } + + pub(crate) fn map_prefix(items: Vec, map: F) -> (Vec, Option) + where + T: Send, + O: Send, + E: Send, + F: Fn(T) -> Result + Sync + Send, + { + let mut outputs = Vec::with_capacity(items.len()); + for item in items { + match map(item) { + Ok(output) => outputs.push(output), + Err(error) => return (outputs, Some(error)), + } + } + (outputs, None) + } + + /// One at a time, so the window only decides how much output is alive at + /// once rather than how much work runs at once. + #[cfg(any(test, feature = "write"))] + pub(crate) fn map_slice_windowed<'a, T, O, E, F, C>( + items: &'a [T], + _window: usize, + map: F, + mut consume: C, + ) -> Result<(), E> + where + T: Sync + 'a, + O: Send, + E: Send, + F: Fn(&'a T) -> Result + Sync + Send, + C: FnMut(&'a T, O) -> Result<(), E>, + { + for item in items { + let output = map(item)?; + consume(item, output)?; + } + Ok(()) + } + + pub(crate) fn default_window() -> usize { + 1 + } + + #[cfg(any(test, feature = "write"))] + pub(crate) fn threads() -> usize { + 1 + } +} + +#[cfg(test)] +mod tests { + use super::sequential; + + #[test] + fn sequential_slots_and_collect_keep_order_and_stop_on_error() { + let mut slots = [7, 7, 7]; + sequential::for_each_mut(&mut slots, |index, slot| *slot += index); + assert_eq!(slots, [7, 8, 9]); + sequential::for_each_mut::(&mut [], |_, _| panic!("empty callback")); + let mapped = sequential::map_prefix(vec![3, 1, 2], |x| Ok::<_, ()>(x * 2)); + assert_eq!(mapped, (vec![6, 2, 4], None)); + let empty = sequential::map_prefix(Vec::::new(), |_| -> Result { + panic!("empty map") + }); + assert_eq!(empty, (vec![], None)); + let visited = std::sync::Mutex::new(Vec::new()); + let mapped = sequential::map_prefix(vec![3, 1, 2], |x| { + visited.lock().unwrap().push(x); + if x == 1 { + Err(11) + } else { + Ok(x) + } + }); + assert_eq!(mapped, (vec![3], Some(11))); + assert_eq!(*visited.lock().unwrap(), [3, 1]); + assert_eq!(sequential::default_window(), 1); + assert_eq!(sequential::threads(), 1); + } + + #[test] + fn sequential_windows_publish_in_order_and_preserve_both_failure_kinds() { + let input = [3, 1, 2]; + for window in [0, 1, 8] { + let mut emitted = vec![]; + let result: Result<(), ()> = sequential::map_slice_windowed( + &input, + window, + |x| Ok(x * 2), + |x, y| { + emitted.push((*x, y)); + Ok(()) + }, + ); + assert_eq!(result, Ok(())); + assert_eq!(emitted, [(3, 6), (1, 2), (2, 4)]); + } + let mut emitted = vec![]; + let result = sequential::map_slice_windowed( + &input, + 2, + |x| if *x == 1 { Err(11) } else { Ok(x * 2) }, + |x, y| { + emitted.push((*x, y)); + Ok(()) + }, + ); + assert_eq!(result, Err(11)); + assert_eq!(emitted, [(3, 6)]); + let mut emitted = vec![]; + let visited = std::sync::Mutex::new(Vec::new()); + let result = sequential::map_slice_windowed( + &input, + 2, + |x| { + visited.lock().unwrap().push(*x); + Ok(x * 2) + }, + |x, y| { + emitted.push((*x, y)); + if *x == 1 { + Err(12) + } else { + Ok(()) + } + }, + ); + assert_eq!(result, Err(12)); + assert_eq!(emitted, [(3, 6), (1, 2)]); + assert_eq!(*visited.lock().unwrap(), [3, 1]); + let empty: Result<(), ()> = sequential::map_slice_windowed::( + &[], + 0, + |_| panic!("empty map"), + |_, _| panic!("empty consume"), + ); + assert_eq!(empty, Ok(())); + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn threaded_windows_preserve_order_and_wait_for_admitted_callbacks() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Mutex; + let input = [3, 1, 2, 4]; + let active = AtomicUsize::new(0); + struct Active<'a>(&'a AtomicUsize); + impl Drop for Active<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } + } + for window in [0, 1, 2, 8] { + let mut emitted = Vec::new(); + let result: Result<(), ()> = super::threaded::map_slice_windowed( + &input, + window, + |x| { + active.fetch_add(1, Ordering::Relaxed); + let _active = Active(&active); + Ok(x * 2) + }, + |x, y| { + emitted.push((*x, y)); + Ok(()) + }, + ); + assert_eq!(result, Ok(())); + assert_eq!(emitted, [(3, 6), (1, 2), (2, 4), (4, 8)]); + assert_eq!(active.load(Ordering::Acquire), 0); + } + let mut emitted = Vec::new(); + let result = super::threaded::map_slice_windowed( + &input, + 2, + |x| { + active.fetch_add(1, Ordering::Relaxed); + let _active = Active(&active); + if *x == 2 { + Err(11) + } else { + Ok(x * 2) + } + }, + |x, y| { + emitted.push((*x, y)); + Ok(()) + }, + ); + assert_eq!(result, Err(11)); + // A mapping failure discards the whole failed window. Earlier windows + // are already published, and every admitted mapping has returned. + assert_eq!(emitted, [(3, 6), (1, 2)]); + assert_eq!(active.load(Ordering::Acquire), 0); + + let visited = Mutex::new(Vec::new()); + emitted.clear(); + let result = super::threaded::map_slice_windowed( + &input, + 2, + |x| { + visited.lock().unwrap().push(*x); + Ok(x * 2) + }, + |x, y| { + emitted.push((*x, y)); + if *x == 1 { + Err(12) + } else { + Ok(()) + } + }, + ); + assert_eq!(result, Err(12)); + assert_eq!(emitted, [(3, 6), (1, 2)]); + let mut visited = visited.into_inner().unwrap(); + visited.sort_unstable(); + assert_eq!( + visited, + [1, 3], + "later windows must not start after consumer failure" + ); + + emitted.clear(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _: Result<(), ()> = super::threaded::map_slice_windowed( + &input, + 2, + |x| { + active.fetch_add(1, Ordering::Relaxed); + let _active = Active(&active); + assert_ne!(*x, 2, "injected map panic"); + Ok(x * 2) + }, + |x, y| { + emitted.push((*x, y)); + Ok(()) + }, + ); + })); + assert!(result.is_err()); + assert_eq!(emitted, [(3, 6), (1, 2)]); + assert_eq!(active.load(Ordering::Acquire), 0); + let empty: Result<(), ()> = super::threaded::map_slice_windowed::( + &[], + 0, + |_| panic!("empty map"), + |_, _| panic!("empty consume"), + ); + assert_eq!(empty, Ok(())); + } +} diff --git a/arca-rar/vendor/rars/src/parse_budget.rs b/arca-rar/vendor/rars/src/parse_budget.rs new file mode 100644 index 00000000..4940e56d --- /dev/null +++ b/arca-rar/vendor/rars/src/parse_budget.rs @@ -0,0 +1,154 @@ +//! Header admission for one physical archive parse, before full-body allocation. +use crate::{ArchiveReadOptions, Error, Result}; + +pub(crate) struct ParseBudget { + pub(crate) control: crate::read_control::ReadControl, + count_limit: Option, + byte_limit: Option, + count: u64, + bytes: u64, +} + +impl ParseBudget { + pub(crate) fn new(options: ArchiveReadOptions<'_>) -> Self { + Self { + control: crate::read_control::ReadControl::new(options.cancellation), + count_limit: options.max_header_count, + byte_limit: options.max_header_bytes, + count: 0, + bytes: 0, + } + } + + pub(crate) fn is_limited(&self) -> bool { + self.count_limit.is_some() || self.byte_limit.is_some() + } + + // Encrypted readers can check count before deriving keys/decrypting a prefix. + pub(crate) fn check_count(&self, offset: usize) -> Result<()> { + self.control.check()?; + if let Some(limit) = self.count_limit { + if self.count == limit { + return Err(Error::HeaderCountLimitExceeded { + limit, + required: self.count.saturating_add(1), + } + .at_archive_offset(offset)); + } + } + Ok(()) + } + + pub(crate) fn admit(&mut self, size: usize, offset: usize) -> Result<()> { + self.check_count(offset)?; + if let Some(limit) = self.byte_limit { + if size as u64 > limit - self.bytes { + return Err(Error::HeaderBytesLimitExceeded { + limit, + required: self.bytes.saturating_add(size as u64), + } + .at_archive_offset(offset)); + } + self.bytes += size as u64; + } + if self.count_limit.is_some() { + self.count += 1; + } + Ok(()) + } +} + +#[cfg(all(test, feature = "write"))] +pub(crate) struct PrefixReader { + inner: std::io::Cursor>, + pub reads: Vec, +} + +#[cfg(all(test, feature = "write"))] +impl PrefixReader { + pub(crate) fn new(prefix: Vec) -> Self { + Self { + inner: std::io::Cursor::new(prefix), + reads: Vec::new(), + } + } +} + +#[cfg(all(test, feature = "write"))] +impl std::io::Read for PrefixReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.reads.push(buf.len()); + if self.inner.position() as usize + buf.len() > self.inner.get_ref().len() { + return Err(std::io::ErrorKind::PermissionDenied.into()); + } + std::io::Read::read(&mut self.inner, buf) + } +} + +#[cfg(all(test, feature = "write"))] +impl std::io::Seek for PrefixReader { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + std::io::Seek::seek(&mut self.inner, pos) + } +} + +#[cfg(all(test, feature = "write"))] +mod tests { + use super::*; + + #[test] + fn cumulative_admission_is_inclusive_and_refusal_does_not_charge() { + let mut b = ParseBudget::new( + ArchiveReadOptions::new() + .with_max_header_count(2) + .with_max_header_bytes(10), + ); + b.admit(4, 0).unwrap(); + let e = b.admit(7, 4).unwrap_err(); + assert!(matches!( + e.root_cause(), + Error::HeaderBytesLimitExceeded { + limit: 10, + required: 11 + } + )); + assert!(matches!(e, Error::AtArchiveOffset { offset: 4, .. })); + b.admit(6, 4).unwrap(); + let e = b.admit(1, 10).unwrap_err(); + assert!(matches!( + e.root_cause(), + Error::HeaderCountLimitExceeded { + limit: 2, + required: 3 + } + )); + assert_eq!(e.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!((b.count, b.bytes), (2, 10)); + } + + #[test] + fn overflow_cannot_bypass_limits() { + let mut b = ParseBudget::new( + ArchiveReadOptions::new() + .with_max_header_count(u64::MAX) + .with_max_header_bytes(u64::MAX), + ); + b.bytes = u64::MAX - 1; + b.count = u64::MAX - 1; + assert!(matches!( + b.admit(2, 0).unwrap_err().root_cause(), + Error::HeaderBytesLimitExceeded { + required: u64::MAX, + .. + } + )); + b.admit(1, 0).unwrap(); + assert!(matches!( + b.admit(0, 0).unwrap_err().root_cause(), + Error::HeaderCountLimitExceeded { + required: u64::MAX, + .. + } + )); + } +} diff --git a/arca-rar/vendor/rars/src/pending_archive.rs b/arca-rar/vendor/rars/src/pending_archive.rs new file mode 100644 index 00000000..54e1a533 --- /dev/null +++ b/arca-rar/vendor/rars/src/pending_archive.rs @@ -0,0 +1,58 @@ +//! Temporary archive publication shared by writing and recovery repair. + +use crate::Result; +use std::fs; +use std::path::{Path, PathBuf}; + +pub(crate) struct PendingArchive { + pub(crate) path: Option, + _charge: C, +} + +#[cfg(feature = "recovery")] +impl PendingArchive<()> { + pub(crate) fn create(destination: &Path) -> Result<(Self, fs::File)> { + Self::with_admission(destination, |_| Ok(())) + } +} + +impl PendingArchive { + pub(crate) fn with_admission( + destination: &Path, + admit: impl FnMut(usize) -> Result, + ) -> Result<(Self, fs::File)> { + use crate::atomic64::AtomicU64; + use std::sync::atomic::Ordering; + static NEXT: AtomicU64 = AtomicU64::new(0); + Self::create_with_sequence(destination, admit, || NEXT.fetch_add(1, Ordering::Relaxed)) + } + + pub(crate) fn create_with_sequence( + destination: &Path, + admit: impl FnMut(usize) -> Result, + next_sequence: impl FnMut() -> u64, + ) -> Result<(Self, fs::File)> { + let directory = destination.parent().unwrap_or_else(|| Path::new("")); + let (path, file, charge) = crate::temp_file::create_named_with_sequence( + crate::temp_file::TemporaryKind::Archive, + directory, + admit, + next_sequence, + )?; + Ok(( + Self { + path: Some(path), + _charge: charge, + }, + file, + )) + } +} + +impl Drop for PendingArchive { + fn drop(&mut self) { + if let Some(path) = &self.path { + let _ = fs::remove_file(path); + } + } +} diff --git a/arca-rar/vendor/rars/src/progress.rs b/arca-rar/vendor/rars/src/progress.rs new file mode 100644 index 00000000..41855893 --- /dev/null +++ b/arca-rar/vendor/rars/src/progress.rs @@ -0,0 +1,124 @@ +//! Progress events shared by archive writing and recovery generation. + +/// A high-level archive-writing operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum WriteOperation { + /// Decoding and verifying source payloads for a rewrite; may overlap encoding. + Staging, + /// Compressing or otherwise preparing member payloads. + Compression, + /// Building a RAR 5 recovery record. + Recovery, + /// Writing finished archive bytes to the output. + Emission, +} + +/// Progress reported by archive writers. +/// +/// Callbacks can be invoked concurrently when parallel compression is enabled. +/// Each member starts before its source is consumed and finishes after its packed +/// payload is prepared. Different members may overlap. `Advanced` and +/// `BytesWritten` carry absolute byte counts; entry events must not be added to +/// them. A failed operation does not +/// report `OperationFinished`. Emission can overlap compression and contain recovery sub-operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum WriteProgressEvent<'a> { + /// An operation has started. + OperationStarted { + operation: WriteOperation, + total_bytes: Option, + total_entries: Option, + pass: usize, + }, + /// Work on one archive member has started. + EntryStarted { + operation: WriteOperation, + index: usize, + total_entries: usize, + name: &'a [u8], + input_bytes: u64, + }, + /// Work on one archive member has finished. + EntryFinished { + operation: WriteOperation, + index: usize, + total_entries: usize, + name: &'a [u8], + input_bytes: u64, + }, + /// Absolute progress within the current operation or pass. + Advanced { + operation: WriteOperation, + completed_bytes: u64, + total_bytes: u64, + pass: usize, + }, + /// Absolute archive bytes accepted by the output while its final size is unknown. + BytesWritten { completed_bytes: u64 }, + /// One volume of a multi-volume set has been written out. + VolumeFinished { + volume_number: usize, + total_volumes: Option, + bytes: u64, + }, + /// An operation has finished. + OperationFinished { + operation: WriteOperation, + total_bytes: Option, + total_entries: Option, + pass: usize, + }, +} + +/// Receives archive-writing progress events. +pub trait WriteProgress: Send + Sync { + fn report(&self, event: WriteProgressEvent<'_>); + + /// Returns true when the caller wants the active write operation to stop. + /// Writers retain an observed request for the rest of the write. + /// Cancellation is cooperative: it cannot interrupt a blocked caller I/O + /// operation or an indivisible allocation or cryptographic setup call. + fn is_cancelled(&self) -> bool { + false + } +} + +impl WriteProgress for F +where + F: Fn(WriteProgressEvent<'_>) + Send + Sync, +{ + fn report(&self, event: WriteProgressEvent<'_>) { + self(event); + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ProgressReporter<'a>(pub(crate) &'a dyn WriteProgress); + +impl std::fmt::Debug for ProgressReporter<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ProgressReporter(..)") + } +} + +impl ProgressReporter<'_> { + pub(crate) fn report(self, event: WriteProgressEvent<'_>) { + self.0.report(event); + } + + pub(crate) fn is_cancelled(self) -> bool { + self.0.is_cancelled() + } +} + +#[cfg(test)] +#[test] +fn progress_reporter_debug_does_not_expose_the_callback() { + let callback = |_event: WriteProgressEvent<'_>| {}; + assert_eq!( + format!("{:?}", ProgressReporter(&callback)), + "ProgressReporter(..)" + ); +} diff --git a/arca-rar/vendor/rars/src/rar13.rs b/arca-rar/vendor/rars/src/rar13.rs new file mode 100644 index 00000000..60de3a76 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar13.rs @@ -0,0 +1,1424 @@ +//! RAR 1.3/1.4 format parsing and sequential extraction. + +#[cfg(feature = "write")] +pub(crate) mod write; +#[cfg(feature = "write")] +pub(crate) use write::{ + write_archive_with_sources, write_stored_volumes_with_progress, BorrowedFileEntry, +}; +#[cfg(feature = "write")] +pub use write::{ + write_compressed_archive, write_compressed_archive_with_comment, + write_compressed_archive_with_comment_and_progress, write_compressed_volumes, + write_compressed_volumes_with_progress, write_stored_archive, + write_stored_archive_with_comment, write_stored_volumes, write_streaming_archive_to, + EntrySource, FileEntry, MemberCoding, StoredEntry, StreamingEntry, WriterOptions, + WriterResources, +}; + +use crate::codec::rar13::Reader15State; +use crate::codec::workspace::{Allowance, Budget, Buffer}; +use crate::crypto::rar13::{Rar13Cipher, Rar13DecryptReader}; +use crate::detect::{find_archive_start, ArchiveSignature, RAR13_SIGNATURE, SFX_SCAN_LIMIT}; +use crate::error::{Error, Result}; +use crate::io_util::{read_exact_at, read_u16, read_u32}; +pub(crate) use crate::source::ArchiveSource; +use crate::version::ArchiveFamily; +use crate::volume_extract::ChainedReader; +use std::fs::File; +use std::io::{Read, Write}; +use std::ops::Range; +use std::path::Path; +use std::sync::Arc; + +const MAIN_HEAD_SIZE: u16 = 7; +const FILE_HEAD_BASE_SIZE: usize = 21; +const MHD_VOLUME: u8 = 0x01; +const MHD_COMMENT: u8 = 0x02; +const MHD_SOLID: u8 = 0x08; +const MHD_PACK_COMMENT: u8 = 0x10; +const MHD_AV: u8 = 0x20; +#[cfg(feature = "write")] +const MHD_ALWAYS_SET: u8 = 0x80; +const RAR13_AV_PREFIX: &[u8; 6] = b"\x1ai\x6d\x02\xda\xae"; +const COPY_BUFFER_SIZE: usize = 64 * 1024; +const LHD_SPLIT_BEFORE: u8 = 0x01; +const LHD_SPLIT_AFTER: u8 = 0x02; +const LHD_PASSWORD: u8 = 0x04; +const LHD_COMMENT: u8 = 0x08; +#[cfg(feature = "write")] +const LHD_SOLID: u8 = 0x10; +const METHOD_STORE: u8 = 0; +#[cfg(feature = "write")] +const METHOD_BEST: u8 = 5; +#[cfg(feature = "write")] +const DEFAULT_UNP_VER: u8 = 2; + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct MainHeader { + pub flags: u8, + pub head_size: u16, + pub extra: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct FileHeader { + pub flags: u8, + pub pack_size: u32, + pub unp_size: u32, + pub file_crc: u16, + pub file_time: u32, + pub file_attr: u8, + pub unp_ver: u8, + pub method: u8, + pub head_size: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Entry { + pub header: FileHeader, + pub name: Vec, + pub extra: Vec, + pub packed_range: Range, +} + +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Archive { + pub sfx_offset: usize, + pub main: MainHeader, + pub entries: Vec, + source: ArchiveSource, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct AuthenticityVerification { + pub size: u16, + pub prefix: [u8; 6], + pub cipher_body: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum AuthenticityVerificationStatus { + Absent, + StructurallyPresent, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct ExtractedEntryMeta { + pub name: Vec, + pub file_time: u32, + pub file_attr: u8, + pub is_directory: bool, +} + +impl MainHeader { + pub fn is_volume(&self) -> bool { + self.flags & MHD_VOLUME != 0 + } + + pub fn has_archive_comment(&self) -> bool { + self.flags & MHD_COMMENT != 0 + } + + pub fn has_packed_comment(&self) -> bool { + self.flags & MHD_PACK_COMMENT != 0 + } + + pub fn is_solid(&self) -> bool { + self.flags & MHD_SOLID != 0 + } + + pub fn has_authenticity_verification(&self) -> bool { + self.flags & MHD_AV != 0 + } + + fn parse(input: &[u8]) -> Result { + if input.len() < MAIN_HEAD_SIZE as usize { + return Err(Error::TooShort); + } + if !input.starts_with(RAR13_SIGNATURE) { + return Err(Error::UnsupportedSignature); + } + + let head_size = read_u16(input, 4)?; + let flags = input[6]; + if head_size < MAIN_HEAD_SIZE { + return Err(Error::InvalidHeader( + "RAR 1.3 main header is shorter than 7 bytes", + )); + } + if head_size as usize > input.len() { + return Err(Error::TooShort); + } + + let extra = input[MAIN_HEAD_SIZE as usize..head_size as usize].to_vec(); + + Ok(Self { + flags, + head_size, + extra, + }) + } +} + +// Inspect only the fixed prefix before any name/extra allocation or full read. +fn admit_header( + prefix: &[u8], + remaining: u64, + main: bool, + budget: &mut crate::parse_budget::ParseBudget, + offset: usize, +) -> Result<()> { + budget.control.check()?; + if !budget.is_limited() { + return Ok(()); + } + let base = if main { + MAIN_HEAD_SIZE as usize + } else { + FILE_HEAD_BASE_SIZE + }; + if prefix.len() < base { + return Err(Error::TooShort); + } + if main && !prefix.starts_with(RAR13_SIGNATURE) { + return Err(Error::UnsupportedSignature); + } + let size = read_u16(prefix, if main { 4 } else { 10 })? as usize; + let minimum = if main { + base + } else { + base + prefix[19] as usize + }; + if size < minimum { + return Err(Error::InvalidHeader(if main { + "RAR 1.3 main header is shorter than 7 bytes" + } else { + "RAR 1.3 file header is shorter than its name" + })); + } + if size as u64 > remaining { + return Err(Error::TooShort); + } + budget.admit(size, offset) +} + +impl FileHeader { + fn parse(input: &[u8]) -> Result<(Self, Vec, Vec, usize)> { + if input.len() < FILE_HEAD_BASE_SIZE { + return Err(Error::TooShort); + } + + let pack_size = read_u32(input, 0)?; + let unp_size = read_u32(input, 4)?; + let file_crc = read_u16(input, 8)?; + let head_size = read_u16(input, 10)?; + let file_time = read_u32(input, 12)?; + let file_attr = input[16]; + let flags = input[17]; + let unp_ver = input[18]; + let name_size = input[19] as usize; + let method = input[20]; + let minimum_size = FILE_HEAD_BASE_SIZE + name_size; + + if (head_size as usize) < minimum_size { + return Err(Error::InvalidHeader( + "RAR 1.3 file header is shorter than its name", + )); + } + if input.len() < head_size as usize { + return Err(Error::TooShort); + } + + let name = input[FILE_HEAD_BASE_SIZE..FILE_HEAD_BASE_SIZE + name_size].to_vec(); + let extra = input[minimum_size..head_size as usize].to_vec(); + Ok(( + Self { + flags, + pack_size, + unp_size, + file_crc, + file_time, + file_attr, + unp_ver, + method, + head_size, + }, + name, + extra, + head_size as usize, + )) + } +} + +/// Stored members need no dictionary. Delay model allocation until the first +/// compressed member, retaining the same allowance throughout a solid chain. +struct Decoder15 { + read_control: crate::read_control::ReadControl, + allowance: B, + state: Option>, +} +impl Decoder15 { + fn with_allowance(allowance: &B) -> Self { + Self { + read_control: Default::default(), + allowance: allowance.clone(), + state: None, + } + } + fn reset(&mut self) { + self.state = None; + } + fn decode_member_from_reader( + &mut self, + input: &mut impl Read, + output_size: usize, + solid: bool, + out: &mut impl Write, + ) -> crate::codec::Result<()> { + self.read_control.check_codec()?; + if self.state.is_none() { + self.state = Some(Reader15State::with_allowance(&self.allowance)?); + } + let state = self.state.as_mut().expect("decoder initialized above"); + state.read_control = self.read_control.clone(); + state.decode_member_from_reader(input, output_size, solid, out) + } +} +impl Decoder15 { + fn new() -> Self { + Self::with_allowance(&Allowance::default()) + } +} + +impl Archive { + pub fn parse(input: &[u8]) -> Result { + Self::parse_with_options(input, crate::ArchiveReadOptions::default()) + } + + pub fn parse_with_options( + input: &[u8], + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let data: Arc<[u8]> = Arc::from(input.to_vec().into_boxed_slice()); + Self::parse_shared(data, options) + } + + pub fn parse_owned(input: Vec) -> Result { + Self::parse_owned_with_options(input, crate::ArchiveReadOptions::default()) + } + + pub fn parse_owned_with_options( + input: Vec, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + Self::parse_shared(Arc::from(input.into_boxed_slice()), options) + } + + pub fn parse_path(path: impl AsRef) -> Result { + Self::parse_path_with_options(path, crate::ArchiveReadOptions::default()) + } + + pub fn parse_path_with_options( + path: impl AsRef, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let path = Arc::new(path.as_ref().to_path_buf()); + let mut file = File::open(path.as_ref())?; + let len = file.metadata()?.len(); + let scan_len = len.min(SFX_SCAN_LIMIT as u64) as usize; + let mut scan = vec![0; scan_len]; + file.read_exact(&mut scan)?; + options.check_cancelled()?; + let sig = find_archive_start(&scan, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + if sig.family != ArchiveFamily::Rar13 { + return Err(Error::UnsupportedSignature); + } + Self::parse_seekable(file, len, sig.offset, ArchiveSource::File(path), options) + } + + pub fn parse_path_with_signature( + path: impl AsRef, + signature: ArchiveSignature, + ) -> Result { + Self::parse_path_with_signature_and_options( + path, + signature, + crate::ArchiveReadOptions::default(), + ) + } + + pub fn parse_path_with_signature_and_options( + path: impl AsRef, + signature: ArchiveSignature, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + if signature.family != ArchiveFamily::Rar13 { + return Err(Error::UnsupportedSignature); + } + let path = Arc::new(path.as_ref().to_path_buf()); + let file = File::open(path.as_ref())?; + let len = file.metadata()?.len(); + Self::parse_seekable( + file, + len, + signature.offset, + ArchiveSource::File(path), + options, + ) + } + + fn parse_shared(input: Arc<[u8]>, options: crate::ArchiveReadOptions<'_>) -> Result { + options.check_cancelled()?; + let sig = find_archive_start(&input, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + if sig.family != ArchiveFamily::Rar13 { + return Err(Error::UnsupportedSignature); + } + + let archive = &input[sig.offset..]; + let mut budget = crate::parse_budget::ParseBudget::new(options); + admit_header(archive, archive.len() as u64, true, &mut budget, 0)?; + let main = MainHeader::parse(archive)?; + let mut pos = main.head_size as usize; + let mut entries = Vec::new(); + + while pos < archive.len() { + if archive.len() - pos < FILE_HEAD_BASE_SIZE { + break; + } + + admit_header( + &archive[pos..], + (archive.len() - pos) as u64, + false, + &mut budget, + pos, + )?; + let (header, name, extra, consumed) = FileHeader::parse(&archive[pos..])?; + let data_start = pos + consumed; + let data_end = + data_start + .checked_add(header.pack_size as usize) + .ok_or(Error::InvalidHeader( + "RAR 1.3 file data size overflows usize", + ))?; + if data_end > archive.len() { + return Err(Error::TooShort); + } + + entries.push(Entry { + header, + name, + extra, + packed_range: sig.offset + data_start..sig.offset + data_end, + }); + pos = data_end; + } + + options.check_cancelled()?; + Ok(Self { + sfx_offset: sig.offset, + main, + entries, + source: ArchiveSource::Memory(input), + }) + } + + pub(crate) fn parse_seekable( + file: impl Read + std::io::Seek, + file_len: u64, + sfx_offset: usize, + source: ArchiveSource, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let mut budget = crate::parse_budget::ParseBudget::new(options); + let mut file = budget.control.reader(file); + let main_prefix = read_exact_at(&mut file, sfx_offset, MAIN_HEAD_SIZE as usize)?; + let head_size = read_u16(&main_prefix, 4)? as usize; + admit_header( + &main_prefix, + file_len + .checked_sub(sfx_offset as u64) + .ok_or(Error::TooShort)?, + true, + &mut budget, + 0, + )?; + let main_bytes = read_exact_at(&mut file, sfx_offset, head_size)?; + let main = MainHeader::parse(&main_bytes)?; + let mut pos = main.head_size as usize; + let mut entries = Vec::new(); + + while (sfx_offset + pos) as u64 + FILE_HEAD_BASE_SIZE as u64 <= file_len { + let header_prefix = read_exact_at(&mut file, sfx_offset + pos, FILE_HEAD_BASE_SIZE)?; + let head_size = read_u16(&header_prefix, 10)? as usize; + admit_header( + &header_prefix, + file_len + .checked_sub((sfx_offset + pos) as u64) + .ok_or(Error::TooShort)?, + false, + &mut budget, + pos, + )?; + let header_bytes = read_exact_at(&mut file, sfx_offset + pos, head_size)?; + let (header, name, extra, consumed) = FileHeader::parse(&header_bytes)?; + let data_start = pos + consumed; + let data_end = + data_start + .checked_add(header.pack_size as usize) + .ok_or(Error::InvalidHeader( + "RAR 1.3 file data size overflows usize", + ))?; + if (sfx_offset + data_end) as u64 > file_len { + return Err(Error::TooShort); + } + entries.push(Entry { + header, + name, + extra, + packed_range: sfx_offset + data_start..sfx_offset + data_end, + }); + pos = data_end; + } + + options.check_cancelled()?; + Ok(Self { + sfx_offset, + main, + entries, + source, + }) + } + + fn copy_range_to(&self, range: Range, out: &mut impl Write) -> Result<()> { + self.source.copy_range_to(range, out) + } + + fn range_reader(&self, range: Range) -> Result> { + self.source.range_reader(range) + } + + fn copy_decrypted_range_to( + &self, + range: Range, + mut cipher: Rar13Cipher, + out: &mut impl Write, + ) -> Result<()> { + let mut buffer = [0u8; COPY_BUFFER_SIZE]; + match &self.source { + ArchiveSource::Memory(data) => { + let data = data.get(range).ok_or(Error::TooShort)?; + for chunk in data.chunks(COPY_BUFFER_SIZE) { + buffer[..chunk.len()].copy_from_slice(chunk); + for byte in &mut buffer[..chunk.len()] { + *byte = cipher.decrypt_byte(*byte); + } + out.write_all(&buffer[..chunk.len()])?; + } + } + ArchiveSource::File(_) | ArchiveSource::Reader(_) => { + let mut file = self.source.range_reader(range.clone())?; + let mut remaining = range.len(); + while remaining > 0 { + let to_read = remaining.min(buffer.len()); + file.read_exact(&mut buffer[..to_read])?; + for byte in &mut buffer[..to_read] { + *byte = cipher.decrypt_byte(*byte); + } + out.write_all(&buffer[..to_read])?; + remaining -= to_read; + } + } + } + Ok(()) + } + + /// Streams extracted entries to caller-provided writers. + pub fn extract_to(&self, password: Option<&[u8]>, open: F) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + self.extract_to_with_options( + crate::ArchiveReadOptions::with_optional_password(password), + open, + ) + } + + /// Extracts members with explicit output policy; default wrappers retain their behavior. + pub fn extract_to_with_options( + &self, + options: crate::ArchiveReadOptions<'_>, + mut open: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + self.extract_impl(options, &mut open, None, None) + .map(|_| ()) + } + + pub(crate) fn extract_controlled( + &self, + options: crate::ArchiveReadOptions<'_>, + selector: &mut crate::extraction_control::Selector<'_>, + on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + ) -> Result { + self.extract_impl( + options, + &mut |_| unreachable!("controlled selection supplies writer"), + Some(selector), + on_error, + ) + } + + fn extract_impl( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + selector: Option<&mut crate::extraction_control::Selector<'_>>, + on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + ) -> Result + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + if let Some(limit) = options.max_reader_workspace_bytes { + return self.extract_with_allowance( + options, + open, + selector, + on_error, + &crate::codec::workspace::Allowance::limited(limit), + ); + } + self.extract_with_allowance(options, open, selector, on_error, &Allowance::default()) + } + + fn extract_with_allowance( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + mut selector: Option<&mut crate::extraction_control::Selector<'_>>, + mut on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + allowance: &B, + ) -> Result + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + let password = options.password; + let mut budget = crate::output_limit::OutputBudget::new(options); + let mut unpack15 = Decoder15::with_allowance(allowance); + unpack15.read_control = budget.control.clone(); + let mut extracted_count = 0usize; + let solid = self.main.is_solid(); + for entry in &self.entries { + let selected = crate::extraction_control::select( + &mut selector, + options, + || crate::rar13_member(entry), + solid, + )?; + let mut selected_writer = match selected { + Some(crate::ExtractionDecision::Skip) => continue, + Some(crate::ExtractionDecision::Stop) => { + return Ok(crate::ExtractionOutcome::Stopped) + } + Some(crate::ExtractionDecision::Extract(writer)) => Some(writer), + None => None, + }; + let result = (|| { + options.check_cancelled()?; + if entry.is_split_before() || entry.is_split_after() { + return Err(Error::InvalidHeader( + "RAR 1.3 split entry requires multivolume extraction", + )); + } + let meta = entry.metadata(); + if meta.is_directory { + options.check_cancelled()?; + let _ = match selected_writer.take() { + Some(writer) => writer, + None => open(&meta)?, + }; + options.check_cancelled()?; + extracted_count += 1; + return Ok(()); + } + budget.check(u64::from(entry.header.unp_size), &meta.name)?; + options.check_cancelled()?; + let mut writer = match selected_writer.take() { + Some(writer) => writer, + None => open(&meta)?, + }; + options.check_cancelled()?; + budget.run(&meta.name, &mut writer, |mut writer| { + if entry.is_stored() && !entry.is_encrypted() { + entry + .write_stored_to(self, password, &mut writer) + .map_err(|error| entry.entry_error("extracting", error))?; + } else { + entry + .write_compressed_to( + self, + password, + &mut unpack15, + self.main.is_solid() && extracted_count != 0, + &mut writer, + ) + .map_err(|error| entry.entry_error("extracting", error))?; + } + Ok(()) + })?; + extracted_count += 1; + Ok(()) + })(); + if crate::extraction_control::finish_member( + &mut on_error, + options, + || crate::rar13_member(entry), + solid, + result, + )? { + unpack15.reset(); + unpack15.read_control = budget.control.clone(); + } + } + options.check_cancelled()?; + Ok(crate::ExtractionOutcome::Complete) + } + + #[cfg(feature = "write")] + pub(crate) fn rewrite_preservation_issues(&self) -> Vec { + let mut issues = Vec::new(); + let exact_comment = |extra: &[u8], present: bool| { + if present { + read_u16(extra, 0).is_ok_and(|size| usize::from(size) + 2 == extra.len()) + } else { + extra.is_empty() + } + }; + if self.main.flags & !(MHD_COMMENT | MHD_SOLID | MHD_PACK_COMMENT | MHD_ALWAYS_SET) != 0 + || (self.main.has_packed_comment() && !self.main.has_archive_comment()) + || !exact_comment(&self.main.extra, self.main.has_archive_comment()) + { + issues.push( + "RAR1.3/1.4 main metadata, volumes or authenticity records are unsupported".into(), + ); + } + if self.entries.is_empty() { + issues.push("empty legacy archive (source format cannot be inferred)".into()); + } + for (index, entry) in self.entries.iter().enumerate() { + if entry.header.unp_ver != DEFAULT_UNP_VER + || entry.header.method > METHOD_BEST + || entry.header.flags & !(LHD_PASSWORD | LHD_COMMENT | LHD_SOLID) != 0 + || (!self.main.is_solid() && entry.header.flags & LHD_SOLID != 0) + || !exact_comment(&entry.extra, entry.has_file_comment()) + || (entry.is_directory() + && (entry.header.pack_size != 0 || entry.header.unp_size != 0)) + { + issues.push(format!("RAR1.3/1.4 member {index}: unsupported version, flags, comments or directory payload")); + } + } + if self.entries.last().map(|entry| entry.packed_range.end) != self.source.len().ok() { + issues.push("RAR1.3/1.4 trailing bytes or incomplete archive".into()); + } + issues + } + + pub fn archive_comment(&self) -> Result>> { + self.archive_comment_with_options(crate::ArchiveReadOptions::new()) + } + + /// Decodes the archive comment under the same policy as [`crate::Archive::comment_with_options`]. + pub fn archive_comment_with_options( + &self, + options: crate::ArchiveReadOptions<'_>, + ) -> Result>> { + if let Some(limit) = options.max_reader_workspace_bytes { + return self.archive_comment_with_allowance(options, &Allowance::limited(limit)); + } + self.archive_comment_with_allowance(options, &Allowance::default()) + } + + fn archive_comment_with_allowance( + &self, + options: crate::ArchiveReadOptions<'_>, + allowance: &B, + ) -> Result>> { + options.check_cancelled()?; + let mut budget = crate::output_limit::OutputBudget::new(options); + if !self.main.has_archive_comment() { + return Ok(None); + } + + let length = read_u16(&self.main.extra, 0)? as usize; + if self.main.has_packed_comment() { + if length < 2 { + return Err(Error::InvalidHeader( + "RAR 1.3 packed archive comment is shorter than size field", + )); + } + let unpacked_len = read_u16(&self.main.extra, 2)? as usize; + let packed_len = length - 2; + let packed_start = 4usize; + let packed_end = packed_start + .checked_add(packed_len) + .ok_or(Error::InvalidHeader( + "RAR 1.3 archive comment size overflows", + ))?; + if packed_end > self.main.extra.len() { + return Err(Error::TooShort); + } + + budget.check(unpacked_len as u64, b"CMT")?; + let mut packed = Rar13DecryptReader::new( + &self.main.extra[packed_start..packed_end], + Rar13Cipher::new_comment(), + ); + let mut decoder = Decoder15::with_allowance(allowance); + decoder.read_control = budget.control.clone(); + let mut data = Vec::new(); + budget.run(b"CMT", &mut data, |writer| { + decoder.decode_member_from_reader(&mut packed, unpacked_len, false, writer)?; + Ok(()) + })?; + return Ok(Some(data)); + } + + let comment_start = 2usize; + let comment_end = comment_start + .checked_add(length) + .ok_or(Error::InvalidHeader( + "RAR 1.3 archive comment size overflows", + ))?; + if comment_end > self.main.extra.len() { + return Err(Error::TooShort); + } + budget.check(length as u64, b"CMT")?; + let mut data = Vec::new(); + budget.run(b"CMT", &mut data, |writer| { + writer.write_all(&self.main.extra[comment_start..comment_end])?; + Ok(()) + })?; + Ok(Some(data)) + } + + pub fn authenticity_verification(&self) -> Result> { + if !self.main.has_authenticity_verification() { + return Ok(None); + } + let size = read_u16(&self.main.extra, 0)?; + if size < RAR13_AV_PREFIX.len() as u16 { + return Err(Error::InvalidHeader("RAR 1.3 AV payload is too short")); + } + let payload_end = 2usize + .checked_add(size as usize) + .ok_or(Error::InvalidHeader("RAR 1.3 AV payload size overflows"))?; + if payload_end > self.main.extra.len() { + return Err(Error::TooShort); + } + let prefix_bytes = self + .main + .extra + .get(2..2 + RAR13_AV_PREFIX.len()) + .ok_or(Error::TooShort)?; + let prefix: [u8; 6] = prefix_bytes + .try_into() + .expect("RAR 1.3 AV prefix slice has fixed length"); + if &prefix != RAR13_AV_PREFIX { + return Err(Error::InvalidHeader("RAR 1.3 AV prefix mismatch")); + } + Ok(Some(AuthenticityVerification { + size, + prefix, + cipher_body: self.main.extra[2 + RAR13_AV_PREFIX.len()..payload_end].to_vec(), + })) + } + + pub fn authenticity_verification_status(&self) -> Result { + Ok(if self.authenticity_verification()?.is_some() { + AuthenticityVerificationStatus::StructurallyPresent + } else { + AuthenticityVerificationStatus::Absent + }) + } +} + +impl Entry { + pub fn name_bytes(&self) -> &[u8] { + &self.name + } + + /// Returns the entry name with invalid UTF-8 replaced for display only. + /// + /// Use [`Self::name_bytes`] when exact archive bytes matter. + pub fn name_lossy(&self) -> String { + String::from_utf8_lossy(&self.name).into_owned() + } + + pub fn is_encrypted(&self) -> bool { + self.header.flags & LHD_PASSWORD != 0 + } + + pub fn is_split_before(&self) -> bool { + self.header.flags & LHD_SPLIT_BEFORE != 0 + } + + pub fn is_split_after(&self) -> bool { + self.header.flags & LHD_SPLIT_AFTER != 0 + } + + pub fn is_directory(&self) -> bool { + self.header.file_attr & 0x10 != 0 + } + + pub fn has_file_comment(&self) -> bool { + self.header.flags & LHD_COMMENT != 0 + } + + pub fn file_comment(&self) -> Result>> { + if !self.has_file_comment() { + return Ok(None); + } + let length = read_u16(&self.extra, 0)? as usize; + let comment_start = 2usize; + let comment_end = comment_start + .checked_add(length) + .ok_or(Error::InvalidHeader("RAR 1.3 file comment size overflows"))?; + if comment_end > self.extra.len() { + return Err(Error::TooShort); + } + Ok(Some(self.extra[comment_start..comment_end].to_vec())) + } + + pub fn is_stored(&self) -> bool { + self.header.method == METHOD_STORE + } + + pub fn packed_data<'a>(&self, archive: &'a Archive) -> Result<&'a [u8]> { + match &archive.source { + ArchiveSource::Memory(data) => { + data.get(self.packed_range.clone()).ok_or(Error::TooShort) + } + ArchiveSource::File(_) | ArchiveSource::Reader(_) => Err(Error::InvalidHeader( + "RAR 1.3 file-backed packed data requires owned read", + )), + } + } + + pub fn write_packed_data(&self, archive: &Archive, out: &mut impl Write) -> Result<()> { + archive.copy_range_to(self.packed_range.clone(), out) + } + + pub fn verify_checksum(&self, data: &[u8]) -> Result<()> { + let actual = file_checksum(data); + if actual == self.header.file_crc { + Ok(()) + } else { + Err(Error::CrcMismatch { + expected: self.header.file_crc, + actual, + }) + } + } + + pub fn metadata(&self) -> ExtractedEntryMeta { + ExtractedEntryMeta { + name: self.name.clone(), + file_time: self.header.file_time, + file_attr: self.header.file_attr, + is_directory: self.is_directory(), + } + } + + fn write_stored_to( + &self, + archive: &Archive, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + if self.is_encrypted() { + crate::crypto::require_encryption()?; + let password = password.ok_or(Error::NeedPassword)?; + let mut checksum = Rar13Checksum::new(); + let mut checksum_writer = Rar13ChecksumWriter { + inner: out, + checksum: &mut checksum, + }; + archive.copy_decrypted_range_to( + self.packed_range.clone(), + Rar13Cipher::for_password(password)?, + &mut checksum_writer, + )?; + checksum.verify_stored_length(self.header.unp_size)?; + let actual = checksum.finish(); + return if actual == self.header.file_crc { + Ok(()) + } else { + Err(Error::CrcMismatch { + expected: self.header.file_crc, + actual, + }) + }; + } + let mut checksum = Rar13Checksum::new(); + let mut checksum_writer = Rar13ChecksumWriter { + inner: out, + checksum: &mut checksum, + }; + self.write_packed_data(archive, &mut checksum_writer)?; + checksum.verify_stored_length(self.header.unp_size)?; + let actual = checksum.finish(); + if actual == self.header.file_crc { + Ok(()) + } else { + Err(Error::CrcMismatch { + expected: self.header.file_crc, + actual, + }) + } + } + + fn write_compressed_to( + &self, + archive: &Archive, + password: Option<&[u8]>, + unpack15: &mut Decoder15, + solid: bool, + out: &mut impl Write, + ) -> Result<()> { + if self.is_stored() { + return self.write_stored_to(archive, password, out); + } + let mut checksum = Rar13Checksum::new(); + let mut checksum_writer = Rar13ChecksumWriter { + inner: out, + checksum: &mut checksum, + }; + if self.is_encrypted() { + crate::crypto::require_encryption()?; + let password = password.ok_or(Error::NeedPassword)?; + let packed = archive.range_reader(self.packed_range.clone())?; + let mut packed = Rar13DecryptReader::new(packed, Rar13Cipher::for_password(password)?); + unpack15.decode_member_from_reader( + &mut packed, + self.header.unp_size as usize, + solid, + &mut checksum_writer, + )?; + } else { + let mut packed = archive.range_reader(self.packed_range.clone())?; + unpack15.decode_member_from_reader( + &mut packed, + self.header.unp_size as usize, + solid, + &mut checksum_writer, + )?; + } + let actual = checksum.finish(); + if actual == self.header.file_crc { + Ok(()) + } else { + Err(Error::CrcMismatch { + expected: self.header.file_crc, + actual, + }) + } + } + + pub fn write_to( + &self, + archive: &Archive, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + self.write_compressed_to(archive, password, &mut Decoder15::new(), false, out) + } + + fn entry_error(&self, operation: &'static str, error: Error) -> Error { + if matches!( + error, + Error::NeedPassword | Error::WrongPasswordOrCorruptData + ) { + return error; + } + if self.is_encrypted() + && !matches!( + error.kind(), + crate::ErrorKind::ResourceLimit + | crate::ErrorKind::Cancelled + | crate::ErrorKind::Io + ) + && matches!( + error, + Error::InvalidHeader(_) + | Error::Codec(_) + | Error::CrcMismatch { .. } + | Error::Crc32Mismatch { .. } + | Error::HashMismatch { .. } + ) + { + return Error::WrongPasswordOrCorruptData; + } + error.at_entry(self.name.clone(), operation) + } +} + +/// Streams a multivolume archive set to caller-provided writers. +pub fn extract_volumes_to(volumes: &[Archive], password: Option<&[u8]>, open: F) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + extract_volumes_to_with_options( + volumes, + crate::ArchiveReadOptions::with_optional_password(password), + open, + ) +} + +/// Extracts logical volume members with explicit output policy. +pub fn extract_volumes_to_with_options( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + open: F, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + if let Some(limit) = options.max_reader_workspace_bytes { + return extract_volumes_with_allowance(volumes, options, open, &Allowance::limited(limit)); + } + extract_volumes_with_allowance(volumes, options, open, &Allowance::default()) +} + +fn extract_volumes_with_allowance( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + mut open: F, + allowance: &B, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + options.check_cancelled()?; + let password = options.password; + let mut budget = crate::output_limit::OutputBudget::new(options); + let mut pending: Option> = None; + let mut unpack15 = Decoder15::with_allowance(allowance); + unpack15.read_control = budget.control.clone(); + let mut extracted_count = 0usize; + + for (volume_index, archive) in volumes.iter().enumerate() { + for (entry_index, entry) in archive.entries.iter().enumerate() { + options.check_cancelled()?; + if !entry.is_split_before() && !entry.is_split_after() { + if pending.is_some() { + return Err(Error::InvalidHeader( + "RAR 1.3 split entry is interrupted by a regular entry", + )); + } + let meta = entry.metadata(); + if meta.is_directory { + options.check_cancelled()?; + let _ = open(&meta)?; + options.check_cancelled()?; + extracted_count += 1; + continue; + } + budget.check(u64::from(entry.header.unp_size), &meta.name)?; + options.check_cancelled()?; + let mut writer = open(&meta)?; + options.check_cancelled()?; + budget.run(&meta.name, &mut writer, |mut writer| { + entry + .write_compressed_to( + archive, + password, + &mut unpack15, + archive.main.is_solid() && extracted_count != 0, + &mut writer, + ) + .map_err(|error| entry.entry_error("extracting", error))?; + options.check_cancelled()?; + Ok(()) + })?; + extracted_count += 1; + continue; + } + + match ( + &mut pending, + entry.is_split_before(), + entry.is_split_after(), + ) { + (None, false, true) => { + pending = Some(PendingSplitRefs::with_allowance( + entry, + volume_index, + entry_index, + allowance, + )?); + } + (Some(current), true, true) => { + current.append(entry, volume_index, entry_index)?; + } + (Some(current), true, false) => { + current.append(entry, volume_index, entry_index)?; + let completed = pending.take().expect("pending split"); + let solid = archive.main.is_solid() && extracted_count != 0; + completed + .write_to( + volumes, + entry, + options, + &mut budget, + &mut unpack15, + solid, + &mut open, + ) + .map_err(|error| entry.entry_error("extracting", error))?; + extracted_count += 1; + } + _ => { + return Err(Error::InvalidHeader( + "RAR 1.3 split entry flags are inconsistent", + )); + } + } + } + } + + if pending.is_some() { + return Err(Error::InvalidHeader("RAR 1.3 split entry is incomplete")); + } + + options.check_cancelled()?; + Ok(()) +} + +struct Rar13ChecksumWriter<'a, W: Write + ?Sized> { + inner: &'a mut W, + checksum: &'a mut Rar13Checksum, +} + +impl Write for Rar13ChecksumWriter<'_, W> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.checksum.update(&buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +struct Rar13Checksum { + value: u16, + written: u64, +} + +impl Rar13Checksum { + fn new() -> Self { + Self { + value: 0, + written: 0, + } + } + + fn update(&mut self, input: &[u8]) { + self.written = self.written.saturating_add(input.len() as u64); + for &byte in input { + self.value = self.value.wrapping_add(byte as u16).rotate_left(1); + } + } + + fn verify_stored_length(&self, expected: u32) -> Result<()> { + if self.written != u64::from(expected) { + return Err(Error::InvalidHeader( + "RAR 1.3 stored file has mismatched packed and unpacked sizes", + )); + } + Ok(()) + } + + fn finish(self) -> u16 { + self.value + } +} + +struct PendingSplitRefs { + name: Vec, + fragments: Buffer<(usize, usize), B>, + file_time: u32, + file_attr: u8, + method: u8, + unp_ver: u8, + was_encrypted: bool, +} + +impl PendingSplitRefs { + fn with_allowance( + entry: &Entry, + volume_index: usize, + entry_index: usize, + allowance: &B, + ) -> Result { + Ok(Self { + name: entry.name.clone(), + fragments: Buffer::copied(&[(volume_index, entry_index)], allowance)?, + file_time: entry.header.file_time, + file_attr: entry.header.file_attr, + method: entry.header.method, + unp_ver: entry.header.unp_ver, + was_encrypted: entry.is_encrypted(), + }) + } + + fn append(&mut self, entry: &Entry, volume_index: usize, entry_index: usize) -> Result<()> { + if entry.name != self.name { + return Err(Error::InvalidHeader("RAR 1.3 split entry name changed")); + } + if entry.header.method != self.method { + return Err(Error::InvalidHeader( + "RAR 1.3 split entry compression method changed", + )); + } + if entry.header.unp_ver != self.unp_ver { + return Err(Error::InvalidHeader( + "RAR 1.3 split entry unpack version changed", + )); + } + if entry.is_encrypted() != self.was_encrypted { + return Err(Error::InvalidHeader( + "RAR 1.3 split entry encryption flag changed", + )); + } + self.fragments.try_push((volume_index, entry_index))?; + Ok(()) + } + + // Split decoding needs format state, policy accounting and the output callback. + #[allow(clippy::too_many_arguments)] + fn write_to( + self, + volumes: &[Archive], + final_entry: &Entry, + options: crate::ArchiveReadOptions<'_>, + budget: &mut crate::output_limit::OutputBudget, + unpack15: &mut Decoder15, + solid: bool, + open: &mut F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + let password = options.password; + budget.check(u64::from(final_entry.header.unp_size), &self.name)?; + let mut reader = + self.fragment_reader_with_allowance(volumes, password, &unpack15.allowance)?; + let meta = ExtractedEntryMeta { + name: self.name, + file_time: self.file_time, + file_attr: self.file_attr, + is_directory: false, + }; + options.check_cancelled()?; + let mut writer = open(&meta)?; + options.check_cancelled()?; + budget.run(&meta.name, &mut writer, |mut writer| { + let mut checksum = Rar13Checksum::new(); + let mut checksum_writer = Rar13ChecksumWriter { + inner: &mut writer, + checksum: &mut checksum, + }; + if self.method == METHOD_STORE { + std::io::copy(&mut reader, &mut checksum_writer)?; + checksum.verify_stored_length(final_entry.header.unp_size)?; + } else { + unpack15.decode_member_from_reader( + &mut reader, + final_entry.header.unp_size as usize, + solid, + &mut checksum_writer, + )?; + } + let actual = checksum.finish(); + if actual == final_entry.header.file_crc { + options.check_cancelled()?; + Ok(()) + } else { + Err(Error::CrcMismatch { + expected: final_entry.header.file_crc, + actual, + }) + } + }) + } + + fn fragment_reader_with_allowance<'a>( + &self, + volumes: &'a [Archive], + password: Option<&'a [u8]>, + allowance: &B, + ) -> Result, B>>> { + let mut readers = Buffer::with_capacity(self.fragments.len(), allowance)?; + for &(volume_index, entry_index) in &self.fragments { + let archive = volumes + .get(volume_index) + .ok_or(Error::InvalidHeader("RAR 1.3 split volume is missing"))?; + let entry = archive + .entries + .get(entry_index) + .ok_or(Error::InvalidHeader("RAR 1.3 split entry is missing"))?; + readers.push_admitted(archive.range_reader(entry.packed_range.clone())?); + } + let chained = ChainedReader::with_readers(readers); + if self.was_encrypted { + crate::crypto::require_encryption()?; + let password = password.ok_or(Error::NeedPassword)?; + // RAR 1.402 encrypts the logical packed stream continuously across + // split volumes; restarting the cipher at each part corrupts it. + Ok(PackedReader::Encrypted(Rar13DecryptReader::new( + chained, + Rar13Cipher::for_password(password)?, + ))) + } else { + Ok(PackedReader::Plain(chained)) + } + } +} + +enum PackedReader { + Plain(R), + Encrypted(Rar13DecryptReader), +} +impl Read for PackedReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self { + Self::Plain(reader) => reader.read(out), + Self::Encrypted(reader) => reader.read(out), + } + } +} + +pub fn file_checksum(input: &[u8]) -> u16 { + let mut checksum = Rar13Checksum::new(); + checksum.update(input); + checksum.finish() +} diff --git a/arca-rar/vendor/rars/src/rar13/write.rs b/arca-rar/vendor/rars/src/rar13/write.rs new file mode 100644 index 00000000..ec211eae --- /dev/null +++ b/arca-rar/vendor/rars/src/rar13/write.rs @@ -0,0 +1,3791 @@ +//! RAR 1.3/1.4 archive encoding and assembly. + +use super::*; +use crate::codec::rar13::{ + unpack15_decode, unpack15_encode, unpack15_encode_with_options_and_progress, + EncodeOptions as Rar15EncodeOptions, Unpack15Encoder, +}; +use crate::features::FeatureSet; +pub use crate::streaming::{EntrySource, WriterResources}; +pub use crate::write_plan::MemberCoding; +use crate::write_plan::{PlanShape, WriterOption}; +use crate::write_progress::{ProgressReporter, WorkTracker}; +use crate::write_stream::{MemberBytes, MemberPayload}; +use crate::ArchiveVersion; +use crate::{WriteOperation, WriteProgress, WriteProgressEvent}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct WriterOptions { + pub target: ArchiveVersion, + pub features: FeatureSet, + pub compression_level: Option, +} + +impl WriterOptions { + pub const fn new(target: ArchiveVersion, features: FeatureSet) -> Self { + Self { + target, + features, + compression_level: None, + } + } + + pub const fn with_compression_level(mut self, level: u8) -> Self { + self.compression_level = Some(level); + self + } +} + +impl Default for WriterOptions { + fn default() -> Self { + Self { + target: ArchiveVersion::Rar14, + features: FeatureSet::store_only(), + compression_level: None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StoredEntry<'a> { + pub name: &'a [u8], + pub data: &'a [u8], + pub file_time: u32, + pub file_attr: u8, + pub password: Option<&'a [u8]>, + pub file_comment: Option<&'a [u8]>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileEntry<'a> { + pub name: &'a [u8], + pub data: &'a [u8], + pub file_time: u32, + pub file_attr: u8, + pub password: Option<&'a [u8]>, + pub file_comment: Option<&'a [u8]>, +} + +/// One member of an archive written a member at a time. +/// +/// The bytes are opened when the member is coded and dropped once it is +/// written, so an archive of many files never holds more than one of them. A +/// stored member is copied straight from its source and never lands on the heap +/// at all. +#[derive(Debug, Clone)] +pub struct StreamingEntry { + pub name: Vec, + pub source: EntrySource, + pub file_time: u32, + pub file_attr: u8, + pub password: Option>, + pub file_comment: Option>, +} + +impl StreamingEntry { + pub fn new(name: impl Into>, source: EntrySource) -> Self { + Self { + name: name.into(), + source, + file_time: 0, + file_attr: 0, + password: None, + file_comment: None, + } + } + + pub fn with_file_time(mut self, file_time: u32) -> Self { + self.file_time = file_time; + self + } + + pub fn with_file_attr(mut self, file_attr: u8) -> Self { + self.file_attr = file_attr; + self + } + + pub fn with_password(mut self, password: impl Into>) -> Self { + self.password = Some(password.into()); + self + } + + pub fn with_file_comment(mut self, comment: impl Into>) -> Self { + self.file_comment = Some(comment.into()); + self + } +} + +pub fn write_stored_archive( + entries: &[StoredEntry<'_>], + options: WriterOptions, +) -> Result> { + write_stored_archive_with_comment(entries, options, None) +} + +pub fn write_stored_archive_with_comment( + entries: &[StoredEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, +) -> Result> { + write_stored_archive_with_comment_and_progress(entries, options, archive_comment, None) +} + +pub(crate) fn write_stored_archive_with_comment_and_progress( + entries: &[StoredEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, + progress: Option<&dyn WriteProgress>, +) -> Result> { + let members: Vec<_> = entries.iter().map(Member::from_stored).collect(); + collect_archive( + &members, + options, + MemberCoding::Stored, + archive_comment, + progress, + ) +} + +pub fn write_compressed_archive( + entries: &[FileEntry<'_>], + options: WriterOptions, +) -> Result> { + write_compressed_archive_with_comment(entries, options, None) +} + +/// Builder entries borrow either caller bytes or reopenable sources. This +/// keeps a source-backed archive from materializing every member up front. +pub(crate) struct BorrowedFileEntry<'a> { + pub(crate) file: FileEntry<'a>, + pub(crate) source: Option<&'a EntrySource>, +} + +pub(crate) fn write_archive_with_sources( + entries: &[BorrowedFileEntry<'_>], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + output: &mut dyn Write, +) -> Result<()> { + let members: Vec<_> = entries + .iter() + .map(|entry| { + let mut member = Member::from_file(&entry.file); + if let Some(source) = entry.source { + member.bytes = MemberBytes::NamedSource(source, entry.file.name); + } + member + }) + .collect(); + write_archive_to( + &members, + options, + coding, + archive_comment, + resources, + progress, + output, + ) +} + +pub fn write_compressed_archive_with_comment( + entries: &[FileEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, +) -> Result> { + write_compressed_archive_with_comment_and_progress(entries, options, archive_comment, None) +} + +pub fn write_compressed_archive_with_comment_and_progress( + entries: &[FileEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, + progress: Option<&dyn WriteProgress>, +) -> Result> { + let members: Vec<_> = entries.iter().map(Member::from_file).collect(); + collect_archive( + &members, + options, + MemberCoding::Compressed, + archive_comment, + progress, + ) +} + +/// Writes an archive straight to `output`, holding only the member being coded +/// rather than every input at once. +/// +/// `resources` bounds the working set. RAR 1.3 compresses a member as a unit, +/// so a member larger than the budget is coded on its own rather than refused. +pub fn write_streaming_archive_to( + entries: &[StreamingEntry], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + output: &mut dyn Write, +) -> Result<()> { + // Managed-memory quotas require codec and source allocations to be charged; + // legacy members do not yet provide that accounting. + if resources.max_preparation_bytes().is_some() || resources.max_memory_bytes().is_some() { + return Err(Error::UnsupportedFamilyFeature { + family: options.target.family(), + feature: if resources.max_memory_bytes().is_some() { + "aggregate managed-memory limit" + } else { + "preparation memory quota" + }, + }); + } + + let members: Vec<_> = entries.iter().map(Member::from_streaming).collect(); + write_archive_to( + &members, + options, + coding, + archive_comment, + resources, + progress, + output, + ) +} + +/// One member, however the caller supplied it. +struct Member<'a> { + name: &'a [u8], + bytes: MemberBytes<'a>, + file_time: u32, + file_attr: u8, + password: Option<&'a [u8]>, + file_comment: Option<&'a [u8]>, +} + +impl<'a> Member<'a> { + fn from_stored(entry: &'a StoredEntry<'a>) -> Self { + Self { + name: entry.name, + bytes: MemberBytes::Borrowed(entry.data), + file_time: entry.file_time, + file_attr: entry.file_attr, + password: entry.password, + file_comment: entry.file_comment, + } + } + + fn from_file(entry: &'a FileEntry<'a>) -> Self { + Self { + name: entry.name, + bytes: MemberBytes::Borrowed(entry.data), + file_time: entry.file_time, + file_attr: entry.file_attr, + password: entry.password, + file_comment: entry.file_comment, + } + } + + fn from_streaming(entry: &'a StreamingEntry) -> Self { + Self { + name: &entry.name, + bytes: MemberBytes::Source(&entry.source), + file_time: entry.file_time, + file_attr: entry.file_attr, + password: entry.password.as_deref(), + file_comment: entry.file_comment.as_deref(), + } + } + + fn unpacked_size(&self) -> Result { + usize::try_from(self.bytes.len()?) + .map_err(|_| Error::InvalidArgument("RAR 1.3 file is larger than 32-bit size fields")) + } +} + +/// Runs the streaming writer into a buffer, for the callers that want the +/// archive as bytes. +fn collect_archive( + members: &[Member<'_>], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + progress: Option<&dyn WriteProgress>, +) -> Result> { + let mut out = Vec::new(); + write_archive_to( + members, + options, + coding, + archive_comment, + &WriterResources::default(), + progress, + &mut out, + )?; + Ok(out) +} + +fn write_archive_to( + members: &[Member<'_>], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + output: &mut dyn Write, +) -> Result<()> { + if members.iter().any(|member| member.password.is_some()) { + crate::crypto::require_encryption()?; + } + + let control = + crate::write_progress::ResourceProgress::new(resources, progress.map(ProgressReporter)); + let progress = Some(&control as &dyn WriteProgress); + crate::write_progress::check_cancelled(progress.map(ProgressReporter))?; + let mut output = crate::write_progress::CancellableIo { + inner: output, + progress: progress.map(ProgressReporter), + }; + let output = &mut output as &mut dyn Write; + if !options.target.is_rar13_family() { + return Err(Error::UnsupportedVersion(options.target)); + } + validate_plan(options, coding.shape())?; + if coding.is_filtered() { + return Err(Error::UnsupportedWriterOption { + target: options.target, + option: crate::write_plan::WriterOption::Filter, + because: None, + }); + } + + let mut head = Vec::new(); + write_main_header(&mut head, options.features, archive_comment)?; + output.write_all(&head)?; + + let encode_options = rar15_encode_options_for_level(options.compression_level); + let mut solid_encoder = (options.features.solid && coding.compresses()) + .then(|| Unpack15Encoder::with_options(encode_options)); + + let mut total_bytes = 0u64; + for member in members { + total_bytes = total_bytes.saturating_add(member.unpacked_size().map_err(|error| { + crate::write_stream::member_error(error, member.name, "reading source") + })? as u64); + } + // Without a solid chain the writer tries each fallback setting in turn, so + // its progress total counts every byte once per attempt. + let attempts = + if !coding.compresses() || options.features.solid || options.compression_level == Some(0) { + 1 + } else { + rar15_encode_fallback_options(encode_options).len() as u64 + }; + let total_work = total_bytes.saturating_mul(attempts); + let reporting = &control as &dyn WriteProgress; + report_compression_operation(reporting, true, total_work, members.len()); + let work = WorkTracker::new( + Some(ProgressReporter(reporting)), + WriteOperation::Compression, + total_work, + ); + + for (index, member) in members.iter().enumerate() { + work.check()?; + let unpacked_size = member.unpacked_size().map_err(|error| { + crate::write_stream::member_error(error, member.name, "reading source") + })?; + report_compression_entry(reporting, true, index, members.len(), member, unpacked_size); + work.check()?; + let encoded = encode_member( + member, + options, + &coding, + encode_options, + solid_encoder.as_mut(), + resources, + &work, + ) + .map_err(|error| crate::write_stream::member_error(error, member.name, "preparing"))?; + write_member(output, member, encoded, options, work.reporter()) + .map_err(|error| crate::write_stream::member_error(error, member.name, "writing"))?; + member.bytes.release(); + report_compression_entry( + reporting, + false, + index, + members.len(), + member, + unpacked_size, + ); + } + + if !work.finish() { + return Err(Error::Cancelled); + } + report_compression_operation(reporting, false, total_work, members.len()); + work.check()?; + Ok(()) +} + +struct EncodedMember<'a> { + payload: MemberPayload<'a>, + method: u8, + unpacked_size: usize, + file_crc: u16, +} + +/// Working memory one member needs while it is coded. This is the admission +/// weight that keeps concurrent writers from piling up, not a prediction. +fn member_workspace(unpacked: u64, compressing: bool) -> u64 { + if !compressing { + return 1024 * 1024; + } + unpacked.saturating_mul(4).saturating_add(2 * 1024 * 1024) +} + +/// Codes one member, holding its bytes only while it is being coded. +#[allow(clippy::too_many_arguments)] +fn encode_member<'a>( + member: &Member<'a>, + options: WriterOptions, + coding: &MemberCoding, + encode_options: Rar15EncodeOptions, + solid_encoder: Option<&mut Unpack15Encoder>, + resources: &WriterResources, + work: &WorkTracker<'_>, +) -> Result> { + work.check()?; + let unpacked_size = member.unpacked_size()?; + validate_member(member.name, unpacked_size)?; + if member.file_attr & 0x10 != 0 { + if unpacked_size != 0 { + return Err(Error::InvalidArgument( + "RAR1.3/1.4 directories must have no payload", + )); + } + return Ok(EncodedMember { + payload: MemberPayload::Packed(Vec::new()), + method: METHOD_STORE, + unpacked_size: 0, + file_crc: file_checksum(&[]), + }); + } + let _permit = resources.acquire_serialising_cancellable( + member_workspace(unpacked_size as u64, coding.compresses()), + &|| work.is_cancelled(), + )?; + + // A stored member never needs to be resident: checksum it from its source + // and let the writer copy it straight through. + if let (MemberCoding::Stored, None, Some(source)) = + (coding, member.password, member.bytes.source()) + { + return Ok(EncodedMember { + payload: MemberPayload::Copied(source), + method: METHOD_STORE, + unpacked_size, + file_crc: { + let mut checksum = Rar13Checksum::new(); + member.bytes.walk_exact_with_progress( + unpacked_size as u64, + work.reporter(), + |chunk| { + checksum.update(chunk); + work.advance(chunk.len() as u64); + }, + )?; + checksum.finish() + }, + }); + } + + let data = member + .bytes + .load_exact_with_progress(unpacked_size as u64, work.reporter())?; + let mut checksum = Rar13Checksum::new(); + for chunk in data.chunks(64 * 1024) { + work.check()?; + checksum.update(chunk); + if !coding.compresses() { + work.advance(chunk.len() as u64); + } + } + let file_crc = checksum.finish(); + work.check()?; + if !coding.compresses() { + return Ok(EncodedMember { + payload: MemberPayload::Packed(data.into_owned()), + method: METHOD_STORE, + unpacked_size, + file_crc, + }); + } + + let solid = solid_encoder.is_some(); + let mut last = 0usize; + let mut advance = |position: usize| advance_rar15_attempt(&mut last, work, position); + // Every arm that ends up stored hands the payload back below, so none of + // them builds a copy of the member to be thrown away. At level zero and + // where the encoder gave nothing back, that copy was the whole member. + let packed = if let Some(encoder) = solid_encoder { + Some(encoder.encode_member_with_progress(&data, &mut advance)?) + } else if options.compression_level == Some(0) { + None + } else { + encode_verified_rar15_payload_with_progress(&data, encode_options, &mut advance)? + }; + let (packed, method) = select_verified_payload(data, packed, solid); + Ok(EncodedMember { + payload: MemberPayload::Packed(packed), + method, + unpacked_size, + file_crc, + }) +} + +/// Writes one member's header and payload. +fn write_member( + output: &mut dyn Write, + member: &Member<'_>, + encoded: EncodedMember<'_>, + options: WriterOptions, + progress: Option>, +) -> Result<()> { + crate::write_progress::check_cancelled(progress)?; + let payload = match encoded.payload { + MemberPayload::Packed(mut packed) => { + if let Some(password) = member.password { + let mut cipher = Rar13Cipher::for_password(password)?; + for chunk in packed.chunks_mut(64 * 1024) { + crate::write_progress::check_cancelled(progress)?; + for byte in chunk { + *byte = cipher.encrypt_byte(*byte); + } + } + } + MemberPayload::Packed(packed) + } + // Unencrypted by construction, so the stored bytes are their own + // payload. + copied => copied, + }; + let packed_size = payload.size(encoded.unpacked_size as u64); + let packed_size = u32::try_from(packed_size) + .map_err(|_| Error::InvalidArgument("RAR 1.3 file is larger than 32-bit size fields"))?; + + let mut flags = 0u8; + if options.features.solid { + flags |= LHD_SOLID; + } + if member.password.is_some() { + flags |= LHD_PASSWORD; + } + if member.file_comment.is_some() { + flags |= LHD_COMMENT; + } + let file_extra = encode_file_comment(member.file_comment)?; + let mut header = Vec::new(); + write_file_header( + &mut header, + FileEntryRecord { + name: member.name, + unpacked_size: encoded.unpacked_size as u32, + file_crc: encoded.file_crc, + packed_size, + file_time: member.file_time, + file_attr: member.file_attr, + flags, + unp_ver: DEFAULT_UNP_VER, + method: encoded.method, + extra: &file_extra, + }, + )?; + output.write_all(&header)?; + if matches!(payload, MemberPayload::Copied(_)) { + let mut checksum = Rar13Checksum::new(); + payload.write_to( + &mut Rar13ChecksumWriter { + inner: output, + checksum: &mut checksum, + }, + u64::from(packed_size), + )?; + if checksum.finish() != encoded.file_crc { + return Err(Error::SourceChanged( + "entry source contents changed while writing", + )); + } + Ok(()) + } else { + payload.write_to(output, u64::from(packed_size)) + } +} + +pub fn write_stored_volumes( + entry: StoredEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, +) -> Result>> { + write_stored_volumes_with_progress(entry, options, max_packed_per_volume, None) +} + +pub(crate) fn write_stored_volumes_with_progress( + entry: StoredEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, + progress: Option<&dyn WriteProgress>, +) -> Result>> { + let resources = WriterResources::default(); + let control = + crate::write_progress::ResourceProgress::new(&resources, progress.map(ProgressReporter)); + let progress = Some(&control as &dyn WriteProgress); + crate::write_progress::check_cancelled(progress.map(ProgressReporter))?; + if !options.target.is_rar13_family() { + return Err(Error::UnsupportedVersion(options.target)); + } + validate_plan(options, PlanShape::new())?; + validate_volume_writer_inputs( + entry.name, + entry.data, + entry.password, + entry.file_comment, + options, + )?; + + let body = entry.data.to_vec(); + write_split_volumes(SplitVolumeRecord { + name: entry.name, + unpacked: entry.data, + packed: &body, + progress: ProgressReporter(&control), + file_time: entry.file_time, + file_attr: entry.file_attr, + method: METHOD_STORE, + base_flags: 0, + features: options.features, + max_packed_per_volume, + }) +} + +pub fn write_compressed_volumes( + entry: FileEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, +) -> Result>> { + write_compressed_volumes_with_progress(entry, options, max_packed_per_volume, None) +} + +pub fn write_compressed_volumes_with_progress( + entry: FileEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, + progress: Option<&dyn WriteProgress>, +) -> Result>> { + let resources = WriterResources::default(); + let control = + crate::write_progress::ResourceProgress::new(&resources, progress.map(ProgressReporter)); + let progress = Some(&control as &dyn WriteProgress); + crate::write_progress::check_cancelled(progress.map(ProgressReporter))?; + if !options.target.is_rar13_family() { + return Err(Error::UnsupportedVersion(options.target)); + } + validate_plan(options, PlanShape::new().compressed(true))?; + validate_volume_writer_inputs( + entry.name, + entry.data, + entry.password, + entry.file_comment, + options, + )?; + + let encode_options = rar15_encode_options_for_level(options.compression_level); + let total_work = (entry.data.len() as u64) + .saturating_mul(rar15_encode_fallback_options(encode_options).len() as u64); + let reporting = &control as &dyn WriteProgress; + report_compression_operation(reporting, true, total_work, 1); + let work = WorkTracker::new( + progress.map(ProgressReporter), + WriteOperation::Compression, + total_work, + ); + report_compression_entry( + reporting, + true, + 0, + 1, + &Member::from_file(&entry), + entry.data.len(), + ); + let mut last = 0usize; + let mut advance = |position: usize| advance_rar15_attempt(&mut last, &work, position); + let packed = + encode_verified_rar15_payload_with_progress(entry.data, encode_options, &mut advance) + .map_err(|error| { + crate::write_stream::member_error(error, entry.name, "compressing volume member") + })?; + let (packed, method) = + select_verified_payload(std::borrow::Cow::Borrowed(entry.data), packed, false); + let result = write_split_volumes(SplitVolumeRecord { + name: entry.name, + unpacked: entry.data, + packed: &packed, + progress: ProgressReporter(&control), + file_time: entry.file_time, + file_attr: entry.file_attr, + method, + base_flags: 0, + features: options.features, + max_packed_per_volume, + }); + report_compression_entry( + reporting, + false, + 0, + 1, + &Member::from_file(&entry), + entry.data.len(), + ); + let result = result?; + if !work.finish() { + return Err(Error::Cancelled); + } + report_compression_operation(reporting, false, total_work, 1); + work.check()?; + Ok(result) +} + +fn report_compression_operation( + progress: &dyn WriteProgress, + started: bool, + total_bytes: u64, + total_entries: usize, +) { + if started { + progress.report(WriteProgressEvent::OperationStarted { + operation: WriteOperation::Compression, + total_bytes: Some(total_bytes), + total_entries: Some(total_entries), + pass: 1, + }); + } else { + progress.report(WriteProgressEvent::OperationFinished { + operation: WriteOperation::Compression, + total_bytes: Some(total_bytes), + total_entries: Some(total_entries), + pass: 1, + }); + } +} + +fn report_compression_entry( + progress: &dyn WriteProgress, + started: bool, + index: usize, + total_entries: usize, + member: &Member<'_>, + input_bytes: usize, +) { + let name = member.name; + let input_bytes = input_bytes as u64; + if started { + progress.report(WriteProgressEvent::EntryStarted { + operation: WriteOperation::Compression, + index, + total_entries, + name, + input_bytes, + }); + } else { + progress.report(WriteProgressEvent::EntryFinished { + operation: WriteOperation::Compression, + index, + total_entries, + name, + input_bytes, + }); + } +} + +/// Everything this writer refuses, in one place, before anything is written. +fn validate_plan(options: WriterOptions, shape: PlanShape) -> Result<()> { + crate::write_plan::validate_features(options.target, options.features, shape)?; + crate::write_plan::validate_compression_level(options.target, options.compression_level) +} + +fn validate_volume_writer_inputs( + name: &[u8], + data: &[u8], + password: Option<&[u8]>, + file_comment: Option<&[u8]>, + options: WriterOptions, +) -> Result<()> { + validate_file_entry(name, data).map_err(|error| { + crate::write_stream::member_error(error, name, "preparing volume member") + })?; + if password.is_some() { + return Err(Error::UnsupportedWriterOption { + target: options.target, + option: WriterOption::Password, + because: Some("in a volume set"), + }); + } + if file_comment.is_some() { + return Err(Error::UnsupportedWriterOption { + target: options.target, + option: WriterOption::FileComment, + because: Some("in a volume set"), + }); + } + Ok(()) +} + +fn rar15_encode_options_for_level(level: Option) -> Rar15EncodeOptions { + // Both callers have validated 0..=5 before resolving the encoder policy. + let level = usize::from(level.unwrap_or(5)); + let options = Rar15EncodeOptions::new() + .with_lazy_matching(level == 5) + .with_stmode_literal_runs(level >= 3); + let max_distance = [ + Some(0), + Some(4 * 1024), + Some(8 * 1024), + Some(16 * 1024), + Some(24 * 1024), + None, + ][level]; + match max_distance { + Some(distance) => options.with_max_long_match_distance(distance), + None => options, + } +} + +/// The encoder reports positions within each attempt, restarting at zero for +/// a verification fallback. Account only the new bytes in that attempt. +fn advance_rar15_attempt(last: &mut usize, work: &WorkTracker<'_>, position: usize) -> bool { + if position < *last { + *last = 0; + } + let delta = position.saturating_sub(*last); + *last = position; + work.advance(delta as u64) +} + +fn select_verified_payload( + data: std::borrow::Cow<'_, [u8]>, + packed: Option>, + solid: bool, +) -> (Vec, u8) { + match packed { + Some(packed) + if !crate::write_plan::StoreFallback::new().applies( + solid, + data.len(), + packed.len(), + ) => + { + (packed, METHOD_BEST) + } + _ => (data.into_owned(), METHOD_STORE), + } +} + +fn encode_verified_rar15_payload_with_progress( + data: &[u8], + options: Rar15EncodeOptions, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result>> { + encode_verified_rar15_payload_using( + data, + options, + progress, + unpack15_encode_with_options_and_progress, + ) +} + +fn encode_verified_rar15_payload_using( + data: &[u8], + options: Rar15EncodeOptions, + progress: &mut dyn FnMut(usize) -> bool, + mut encode: impl FnMut( + &[u8], + Rar15EncodeOptions, + &mut dyn FnMut(usize) -> bool, + ) -> crate::codec::Result>, +) -> Result>> { + for candidate_options in rar15_encode_fallback_options(options) { + let packed = match encode(data, candidate_options, progress) { + Err(crate::codec::Error::Cancelled) => return Err(Error::Cancelled), + result => result?, + }; + if unpack15_payload_matches(&packed, data)? { + return Ok(Some(packed)); + } + } + Ok(None) +} + +fn rar15_encode_fallback_options(options: Rar15EncodeOptions) -> Vec { + let mut candidates = vec![options]; + let distance_limited = options.with_max_long_match_distance(24 * 1024); + if distance_limited != options { + candidates.push(distance_limited); + } + let conservative = options + .with_lazy_matching(false) + .with_stmode_literal_runs(false) + .with_max_long_match_distance(8 * 1024); + if !candidates.contains(&conservative) { + candidates.push(conservative); + } + candidates +} + +fn unpack15_payload_matches(packed: &[u8], data: &[u8]) -> Result { + match unpack15_decode(packed, data.len()) { + Ok(decoded) => Ok(decoded == data), + Err(_) => Ok(false), + } +} + +fn write_main_header( + out: &mut Vec, + features: FeatureSet, + archive_comment: Option<&[u8]>, +) -> Result<()> { + write_main_header_with_flags(out, features, archive_comment, 0) +} + +fn write_main_header_with_flags( + out: &mut Vec, + features: FeatureSet, + archive_comment: Option<&[u8]>, + extra_flags: u8, +) -> Result<()> { + let comment_extra = encode_archive_comment(archive_comment)?; + let mut flags = MHD_ALWAYS_SET | extra_flags; + if archive_comment.is_some() { + flags |= MHD_COMMENT; + flags |= MHD_PACK_COMMENT; + } + if features.solid { + flags |= MHD_SOLID; + } + out.extend_from_slice(RAR13_SIGNATURE); + let head_size = MAIN_HEAD_SIZE as usize + comment_extra.len(); + if head_size > u16::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.3 main header comment extension is too large", + )); + } + out.extend_from_slice(&(head_size as u16).to_le_bytes()); + out.push(flags); + out.extend_from_slice(&comment_extra); + Ok(()) +} + +struct FileEntryRecord<'a> { + name: &'a [u8], + unpacked_size: u32, + file_crc: u16, + packed_size: u32, + file_time: u32, + file_attr: u8, + flags: u8, + unp_ver: u8, + method: u8, + extra: &'a [u8], +} + +fn write_file_header(out: &mut Vec, entry: FileEntryRecord<'_>) -> Result<()> { + let head_size = FILE_HEAD_BASE_SIZE + entry.name.len() + entry.extra.len(); + let head_size = u16::try_from(head_size) + .map_err(|_| Error::InvalidArgument("RAR 1.3 file header is longer than 65535 bytes"))?; + out.extend_from_slice(&entry.packed_size.to_le_bytes()); + out.extend_from_slice(&entry.unpacked_size.to_le_bytes()); + out.extend_from_slice(&entry.file_crc.to_le_bytes()); + out.extend_from_slice(&head_size.to_le_bytes()); + out.extend_from_slice(&entry.file_time.to_le_bytes()); + out.push(entry.file_attr); + out.push(entry.flags); + out.push(entry.unp_ver); + out.push(entry.name.len() as u8); + out.push(entry.method); + out.extend_from_slice(entry.name); + out.extend_from_slice(entry.extra); + Ok(()) +} + +struct SplitVolumeRecord<'a> { + progress: ProgressReporter<'a>, + name: &'a [u8], + unpacked: &'a [u8], + packed: &'a [u8], + file_time: u32, + file_attr: u8, + method: u8, + base_flags: u8, + features: FeatureSet, + max_packed_per_volume: usize, +} + +fn write_split_volumes(entry: SplitVolumeRecord<'_>) -> Result>> { + crate::write_progress::check_cancelled(Some(entry.progress))?; + if entry.max_packed_per_volume == 0 { + return Err(Error::InvalidArgument( + "RAR 1.3 volume payload size must be non-zero", + )); + } + if entry.packed.is_empty() { + return Err(Error::InvalidArgument( + "RAR 1.3 volume writer needs a non-empty packed payload", + ) + .at_entry(entry.name.to_vec(), "preparing volume member")); + } + + // One volume is a legitimate answer when the payload fits; see the note in + // rar15_40::write::write_split_volumes. + let chunks: Vec<&[u8]> = entry.packed.chunks(entry.max_packed_per_volume).collect(); + + let mut volumes = Vec::with_capacity(chunks.len()); + for (index, chunk) in chunks.iter().enumerate() { + crate::write_progress::check_cancelled(Some(entry.progress))?; + let split_before = index > 0; + let split_after = index + 1 < chunks.len(); + let mut flags = entry.base_flags; + if split_before { + flags |= LHD_SPLIT_BEFORE; + } + if split_after { + flags |= LHD_SPLIT_AFTER; + } + if entry.features.solid { + flags |= LHD_SOLID; + } + + let mut out = Vec::new(); + write_main_header_with_flags(&mut out, entry.features, None, MHD_VOLUME)?; + let checksum_data = if split_after { *chunk } else { entry.unpacked }; + // Both public callers validate the 255-byte name limit and forbid + // comments, so this header is at most 21 + 255 bytes. + write_file_header( + &mut out, + FileEntryRecord { + name: entry.name, + unpacked_size: entry.unpacked.len() as u32, + file_crc: file_checksum(checksum_data), + packed_size: chunk.len() as u32, + file_time: entry.file_time, + file_attr: entry.file_attr, + flags, + unp_ver: DEFAULT_UNP_VER, + method: entry.method, + extra: &[], + }, + ) + .expect("validated legacy volume header fits the u16 size field"); + out.extend_from_slice(chunk); + entry.progress.report(WriteProgressEvent::VolumeFinished { + volume_number: index + 1, + total_volumes: Some(chunks.len()), + bytes: out.len() as u64, + }); + crate::write_progress::check_cancelled(Some(entry.progress))?; + volumes.push(out); + } + + Ok(volumes) +} + +fn encode_archive_comment(comment: Option<&[u8]>) -> Result> { + let Some(comment) = comment else { + return Ok(Vec::new()); + }; + if comment.len() > u16::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.3 archive comment is longer than 65535 bytes", + )); + } + let mut packed = unpack15_encode(comment)?; + Rar13Cipher::new_comment().encrypt_in_place(&mut packed); + let packed_field_len = packed.len().checked_add(2).ok_or(Error::InvalidArgument( + "RAR 1.3 archive comment size overflows", + ))?; + if packed_field_len > u16::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.3 packed archive comment is longer than 65535 bytes", + )); + } + + let mut out = Vec::with_capacity(4 + packed.len()); + out.extend_from_slice(&(packed_field_len as u16).to_le_bytes()); + out.extend_from_slice(&(comment.len() as u16).to_le_bytes()); + out.extend_from_slice(&packed); + Ok(out) +} + +fn encode_file_comment(comment: Option<&[u8]>) -> Result> { + let Some(comment) = comment else { + return Ok(Vec::new()); + }; + if comment.len() > u16::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.3 file comment is longer than 65535 bytes", + )); + } + let mut out = Vec::with_capacity(2 + comment.len()); + out.extend_from_slice(&(comment.len() as u16).to_le_bytes()); + out.extend_from_slice(comment); + Ok(out) +} + +fn validate_file_entry(name: &[u8], data: &[u8]) -> Result<()> { + validate_member(name, data.len()) +} + +fn validate_member(name: &[u8], unpacked_size: usize) -> Result<()> { + if name.is_empty() { + return Err(Error::InvalidArgument("RAR 1.3 file name is empty")); + } + if name.len() > u8::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.3 file name is longer than 255 bytes", + )); + } + if unpacked_size > u32::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.3 file is larger than 32-bit size fields", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn writer_cancellation_during_workspace_admission_is_preserved() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CancelOnSecondPoll(AtomicUsize); + impl crate::WriteProgress for CancelOnSecondPoll { + fn report(&self, _: crate::WriteProgressEvent<'_>) {} + + fn is_cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::Relaxed) != 0 + } + } + + let entry = super::FileEntry { + name: b"file", + data: b"payload", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let callback = CancelOnSecondPoll(AtomicUsize::new(0)); + let work = super::WorkTracker::new( + Some(super::ProgressReporter(&callback)), + crate::WriteOperation::Compression, + 7, + ); + let resources = crate::WriterResources::new(1); + let options = super::WriterOptions::default(); + assert!(matches!( + super::encode_member( + &super::Member::from_file(&entry), + options, + &crate::MemberCoding::Stored, + super::rar15_encode_options_for_level(options.compression_level), + None, + &resources, + &work, + ), + Err(crate::Error::Cancelled) + )); + assert_eq!(callback.0.load(Ordering::Relaxed), 2); + assert_eq!(resources.workspace_in_use(), 0); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_extraction_preserves_output_io_failures() { + struct FailingWriter; + impl std::io::Write for FailingWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "closed output", + )) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let data = b"encrypted output failure".repeat(64); + let input = [FileEntry { + name: b"encrypted.bin", + data: &data, + file_time: 0, + file_attr: 0x20, + password: Some(b"pw"), + file_comment: None, + }]; + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.entries[0].is_encrypted()); + assert!(!archive.entries[0].is_stored()); + let error = archive + .extract_to(Some(b"pw"), |_| Ok(Box::new(FailingWriter))) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert!(error.to_string().contains("closed output")); + } + + #[test] + fn volume_headers_accept_the_largest_legacy_name() { + let name = vec![b'N'; 255]; + let payload = b"abcdef"; + let volumes = write_stored_volumes( + StoredEntry { + name: &name, + data: payload, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + WriterOptions::default(), + 3, + ) + .unwrap(); + assert_eq!(volumes.len(), 2); + let archives: Vec<_> = volumes.iter().map(|v| Archive::parse(v).unwrap()).collect(); + for archive in &archives { + assert_eq!(archive.entries[0].name, name); + } + assert_eq!( + collect_extract_volumes(&archives, None).unwrap()[0].data, + payload + ); + } + + #[test] + fn reader_workspace_rar13_stored_archives_need_no_dictionary_allocation() { + for encrypted in crate::test_features::encryption_cases() { + let password = encrypted.then_some(b"pw".as_slice()); + let input = [StoredEntry { + name: b"stored.bin", + data: b"stored payload", + file_time: 0, + file_attr: 0x20, + password, + file_comment: None, + }]; + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let quota = Allowance::limited(0); + archive + .extract_with_allowance( + crate::ArchiveReadOptions::with_optional_password(password), + &mut |_| Ok(Box::new(std::io::sink())), + None, + None, + "a, + ) + .unwrap(); + assert_eq!(quota.used(), 0); + } + } + + #[test] + fn reader_workspace_rar13_refusals_release_dictionary_input_and_stream_buffers() { + use crate::codec::workspace::RefusingBudget; + let data = b"abcabcabc".repeat(64); + for encrypted in crate::test_features::encryption_cases() { + let password = encrypted.then_some(b"pw".as_slice()); + let input = [FileEntry { + name: b"compressed.bin", + data: &data, + file_time: 0, + file_attr: 0x20, + password, + file_comment: None, + }]; + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let entry = &archive.entries[0]; + assert!(!entry.is_stored()); + let run = |quota: &RefusingBudget| -> Result<()> { + let mut decoder = Decoder15::with_allowance(quota); + let mut out = Buffer::new(quota); + entry.write_compressed_to(&archive, password, &mut decoder, false, &mut out)?; + assert_eq!(&out[..], &data); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let quota = RefusingBudget::new(index); + let error = run("a).unwrap_err(); + assert_eq!( + entry.entry_error("extracting", error).kind(), + crate::ErrorKind::Cancelled, + "allocation {index}" + ); + assert_eq!(quota.used(), 0); + } + let quota = Allowance::limited(1); + let error = archive + .extract_with_allowance( + crate::ArchiveReadOptions::with_optional_password(password), + &mut |_| Ok(Box::new(std::io::sink())), + None, + None, + "a, + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(quota.used(), 0); + } + } + + #[test] + fn writer_limits_are_invalid_arguments() { + for error in [ + super::validate_member(b"", 0).unwrap_err(), + super::validate_member(&[b'a'; 256], 0).unwrap_err(), + super::encode_file_comment(Some(&vec![0; 65536])).unwrap_err(), + super::encode_archive_comment(Some(&vec![0; 65536])).unwrap_err(), + ] { + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument, "{error}"); + } + #[cfg(target_pointer_width = "64")] + assert_eq!( + super::validate_member(b"large", u32::MAX as usize + 1) + .unwrap_err() + .kind(), + crate::ErrorKind::InvalidArgument + ); + } + + #[test] + fn header_budget_refuses_full_main_read_after_prefix() { + let mut prefix = super::RAR13_SIGNATURE.to_vec(); + prefix.extend_from_slice(&64u16.to_le_bytes()); + prefix.push(0); + let mut reader = crate::parse_budget::PrefixReader::new(prefix); + let e = super::Archive::parse_seekable( + &mut reader, + 128, + 0, + super::ArchiveSource::Memory(std::sync::Arc::new([])), + crate::ArchiveReadOptions::new().with_max_header_bytes(63), + ) + .unwrap_err(); + assert!(matches!( + e.root_cause(), + crate::Error::HeaderBytesLimitExceeded { required: 64, .. } + )); + assert_eq!(reader.reads, [7]); + } + + #[test] + fn limited_header_admission_rejects_malformed_fixed_prefixes() { + let options = crate::ArchiveReadOptions::new().with_max_header_count(2); + let check = |prefix: &[u8], remaining, main| { + let mut budget = crate::parse_budget::ParseBudget::new(options); + super::admit_header(prefix, remaining, main, &mut budget, 0).unwrap_err() + }; + + let mut main = [0u8; super::MAIN_HEAD_SIZE as usize]; + main[..4].copy_from_slice(super::RAR13_SIGNATURE); + main[4..6].copy_from_slice(&7u16.to_le_bytes()); + assert_eq!(check(&main[..6], 7, true), Error::TooShort); + let mut wrong_signature = main; + wrong_signature[0] = 0; + assert_eq!( + check(&wrong_signature, 7, true), + Error::UnsupportedSignature + ); + main[4..6].copy_from_slice(&6u16.to_le_bytes()); + assert_eq!( + check(&main, 7, true), + Error::InvalidHeader("RAR 1.3 main header is shorter than 7 bytes") + ); + main[4..6].copy_from_slice(&8u16.to_le_bytes()); + assert_eq!(check(&main, 7, true), Error::TooShort); + + let mut file = [0u8; super::FILE_HEAD_BASE_SIZE]; + file[19] = 3; + file[10..12].copy_from_slice(&24u16.to_le_bytes()); + assert_eq!(check(&file[..20], 24, false), Error::TooShort); + file[10..12].copy_from_slice(&23u16.to_le_bytes()); + assert_eq!( + check(&file, 24, false), + Error::InvalidHeader("RAR 1.3 file header is shorter than its name") + ); + file[10..12].copy_from_slice(&25u16.to_le_bytes()); + assert_eq!(check(&file, 24, false), Error::TooShort); + } + use super::*; + use crate::codec::rar13::{find_long_lz, LongLz}; + use std::cell::RefCell; + use std::rc::Rc; + use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + + struct CollectWriter(Rc>>); + + #[test] + fn compressed_writer_reports_balanced_progress_events() { + let entries = [ + FileEntry { + name: b"one.txt", + data: b"one one one one", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + FileEntry { + name: b"two.txt", + data: b"two two two two", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + ]; + let operation_starts = AtomicUsize::new(0); + let operation_finishes = AtomicUsize::new(0); + let entry_starts = AtomicUsize::new(0); + let entry_finishes = AtomicUsize::new(0); + let advances = AtomicUsize::new(0); + let last_completed = AtomicU64::new(0); + let expected_total = AtomicU64::new(0); + let reporter = |event: WriteProgressEvent<'_>| match event { + WriteProgressEvent::OperationStarted { total_bytes, .. } => { + operation_starts.fetch_add(1, Ordering::Relaxed); + expected_total.store(total_bytes.unwrap_or(0), Ordering::Relaxed); + } + WriteProgressEvent::OperationFinished { .. } => { + operation_finishes.fetch_add(1, Ordering::Relaxed); + } + WriteProgressEvent::EntryStarted { .. } => { + entry_starts.fetch_add(1, Ordering::Relaxed); + } + WriteProgressEvent::EntryFinished { .. } => { + entry_finishes.fetch_add(1, Ordering::Relaxed); + } + WriteProgressEvent::Advanced { + completed_bytes, + total_bytes, + .. + } => { + assert!(completed_bytes >= last_completed.swap(completed_bytes, Ordering::Relaxed)); + assert!(completed_bytes <= total_bytes); + advances.fetch_add(1, Ordering::Relaxed); + } + _ => {} + }; + + write_compressed_archive_with_comment_and_progress( + &entries, + WriterOptions::default(), + None, + Some(&reporter), + ) + .unwrap(); + + assert_eq!(operation_starts.load(Ordering::Relaxed), 1); + assert_eq!(operation_finishes.load(Ordering::Relaxed), 1); + assert_eq!(entry_starts.load(Ordering::Relaxed), entries.len()); + assert_eq!(entry_finishes.load(Ordering::Relaxed), entries.len()); + assert!(advances.load(Ordering::Relaxed) >= entries.len()); + assert_eq!( + last_completed.load(Ordering::Relaxed), + expected_total.load(Ordering::Relaxed) + ); + } + + #[derive(Debug, Clone, PartialEq, Eq)] + struct CollectedEntry { + name: Vec, + data: Vec, + file_time: u32, + file_attr: u8, + is_directory: bool, + } + + impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + fn collect_extract(archive: &Archive, password: Option<&[u8]>) -> Result> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(password, |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter(data))) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + file_attr: meta.file_attr, + is_directory: meta.is_directory, + }) + .collect()) + } + + fn collect_extract_volumes( + volumes: &[Archive], + password: Option<&[u8]>, + ) -> Result> { + let entries = RefCell::new(Vec::new()); + extract_volumes_to(volumes, password, |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter(data))) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + file_attr: meta.file_attr, + is_directory: meta.is_directory, + }) + .collect()) + } + + fn synthetic_log_payload(lines: usize) -> Vec { + let mut data = Vec::new(); + for index in 0..lines { + data.extend_from_slice( + format!( + "2026-05-12T12:{:02}:{:02}.000Z INFO worker-{:02} request_id={:04x}-{:05} path=/api/v1/items/{} status={} elapsed_ms={} bytes={} message=processed archive chunk retry={} user=service-{}\n", + index % 60, + (index * 7) % 60, + index % 16, + index % 10000, + (index * 17) % 100000, + index % 2048, + 200 + (index % 5), + (index * 37) % 5000, + (index * 911) % 65536, + index % 3, + index % 32 + ) + .as_bytes(), + ); + } + data + } + + #[test] + fn writes_and_reads_stored_archive() { + let input = [ + StoredEntry { + name: b"README.md", + data: b"hello rar 1.3", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + StoredEntry { + name: b"docs", + data: b"", + file_time: 0, + file_attr: 0x10, + password: None, + file_comment: None, + }, + ]; + + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.main.flags, 0x80); + assert_eq!(archive.entries.len(), 2); + assert_eq!(archive.entries[0].name_bytes(), b"README.md"); + assert_eq!(archive.entries[0].name_lossy(), "README.md"); + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"hello rar 1.3"); + assert!(archive.entries[1].is_directory()); + assert!(extracted[1].is_directory); + } + + #[test] + fn rejects_malformed_main_header_boundaries() { + assert_eq!(MainHeader::parse(b"RE~"), Err(Error::TooShort)); + + let mut too_small = Vec::from(&b"RE~^"[..]); + too_small.extend_from_slice(&6u16.to_le_bytes()); + too_small.push(0x80); + assert_eq!( + MainHeader::parse(&too_small), + Err(Error::InvalidHeader( + "RAR 1.3 main header is shorter than 7 bytes" + )) + ); + + let mut truncated_extra = Vec::from(&b"RE~^"[..]); + truncated_extra.extend_from_slice(&8u16.to_le_bytes()); + truncated_extra.push(0x80); + assert_eq!(MainHeader::parse(&truncated_extra), Err(Error::TooShort)); + + assert!(matches!( + Archive::parse(b"Rar!\x1a\x07\x00"), + Err(Error::UnsupportedSignature) + )); + } + + #[test] + fn rejects_file_header_shorter_than_its_name() { + let mut bytes = Vec::from(&b"RE~^"[..]); + bytes.extend_from_slice(&7u16.to_le_bytes()); + bytes.push(0x80); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.extend_from_slice(&(FILE_HEAD_BASE_SIZE as u16).to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.push(0x20); + bytes.push(0); + bytes.push(DEFAULT_UNP_VER); + bytes.push(10); + bytes.push(METHOD_STORE); + + assert!(matches!( + Archive::parse(&bytes), + Err(Error::InvalidHeader( + "RAR 1.3 file header is shorter than its name" + )) + )); + } + + #[test] + fn rejects_truncated_file_payload_during_parse() { + let input = [StoredEntry { + name: b"hello.txt", + data: b"hello", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + let mut bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + bytes.pop(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::TooShort))); + } + + #[test] + fn returns_none_for_absent_archive_comment() { + let bytes = write_stored_archive(&[], WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.archive_comment().unwrap(), None); + } + + #[test] + fn rejects_normal_extract_on_split_entries() { + let entry = StoredEntry { + name: b"split.bin", + data: b"abcdefghijklmnopqrstuvwxyz", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let volumes = write_stored_volumes(entry, WriterOptions::default(), 8).unwrap(); + let first = Archive::parse(&volumes[0]).unwrap(); + + assert_eq!( + collect_extract(&first, None), + Err(Error::InvalidHeader( + "RAR 1.3 split entry requires multivolume extraction" + )) + ); + assert_eq!( + collect_extract(&first, None), + Err(Error::InvalidHeader( + "RAR 1.3 split entry requires multivolume extraction" + )) + ); + } + + #[test] + fn controlled_extraction_rejects_split_member_without_volume_set() { + let entry = StoredEntry { + name: b"split.bin", + data: b"abcdefghijklmnopqrstuvwxyz", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let volumes = write_stored_volumes(entry, WriterOptions::default(), 8).unwrap(); + for volume in [&volumes[0], volumes.last().unwrap()] { + let archive = crate::Archive::Rar13(Archive::parse(volume).unwrap()); + let error = archive + .extract_with_control(crate::ArchiveReadOptions::new(), |_| { + Ok(crate::ExtractionDecision::Extract(Box::new(Vec::new()))) + }) + .unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::InvalidHeader("RAR 1.3 split entry requires multivolume extraction") + ); + } + } + + #[test] + fn rejects_malformed_comment_extensions() { + let packed_too_short = Archive { + sfx_offset: 0, + main: MainHeader { + flags: MHD_COMMENT | MHD_PACK_COMMENT, + head_size: MAIN_HEAD_SIZE, + extra: 1u16.to_le_bytes().to_vec(), + }, + entries: Vec::new(), + source: ArchiveSource::Memory(Arc::new([])), + }; + assert_eq!( + packed_too_short.archive_comment(), + Err(Error::InvalidHeader( + "RAR 1.3 packed archive comment is shorter than size field" + )) + ); + + let unpacked_too_short = Archive { + sfx_offset: 0, + main: MainHeader { + flags: MHD_COMMENT, + head_size: MAIN_HEAD_SIZE, + extra: 4u16.to_le_bytes().to_vec(), + }, + entries: Vec::new(), + source: ArchiveSource::Memory(Arc::new([])), + }; + assert_eq!(unpacked_too_short.archive_comment(), Err(Error::TooShort)); + } + + #[test] + fn rejects_malformed_av_extensions() { + let too_short = Archive { + sfx_offset: 0, + main: MainHeader { + flags: MHD_AV, + head_size: MAIN_HEAD_SIZE, + extra: 5u16.to_le_bytes().to_vec(), + }, + entries: Vec::new(), + source: ArchiveSource::Memory(Arc::new([])), + }; + assert_eq!( + too_short.authenticity_verification(), + Err(Error::InvalidHeader("RAR 1.3 AV payload is too short")) + ); + + let bad_prefix = Archive { + sfx_offset: 0, + main: MainHeader { + flags: MHD_AV, + head_size: MAIN_HEAD_SIZE, + extra: { + let mut extra = 6u16.to_le_bytes().to_vec(); + extra.extend_from_slice(b"badbad"); + extra + }, + }, + entries: Vec::new(), + source: ArchiveSource::Memory(Arc::new([])), + }; + assert_eq!( + bad_prefix.authenticity_verification(), + Err(Error::InvalidHeader("RAR 1.3 AV prefix mismatch")) + ); + + let mut truncated_payload = bad_prefix; + truncated_payload.main.extra[0..2].copy_from_slice(&7u16.to_le_bytes()); + assert_eq!( + truncated_payload.authenticity_verification(), + Err(Error::TooShort) + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn writes_and_reads_encrypted_stored_archive() { + let input = [StoredEntry { + name: b"secret.txt", + data: b"secret bytes", + file_time: 0, + file_attr: 0x20, + password: Some(b"pass"), + file_comment: None, + }]; + + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.entries[0].is_encrypted()); + match collect_extract(&archive, None) { + Err(Error::NeedPassword) => {} + Err(Error::AtEntry { source, .. }) if matches!(*source, Error::NeedPassword) => {} + other => panic!("expected missing password error, got {other:?}"), + } + + let extracted = collect_extract(&archive, Some(b"pass")).unwrap(); + assert_eq!(extracted[0].data, b"secret bytes"); + } + + #[test] + fn writes_and_reads_archive_comment() { + let input = [StoredEntry { + name: b"README.md", + data: b"hello rar 1.3", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_stored_archive_with_comment( + &input, + WriterOptions::default(), + Some(b"This is an archive comment."), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_archive_comment()); + assert!(archive.main.has_packed_comment()); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"This is an archive comment."[..]) + ); + assert_eq!( + collect_extract(&archive, None).unwrap()[0].data, + b"hello rar 1.3" + ); + } + + #[test] + fn writes_and_reads_file_comment() { + let input = [StoredEntry { + name: b"README.md", + data: b"hello rar 1.3", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: Some(b"file comment\r\n"), + }]; + + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.entries[0].has_file_comment()); + assert_eq!( + archive.entries[0].file_comment().unwrap().as_deref(), + Some(&b"file comment\r\n"[..]) + ); + assert_eq!( + collect_extract(&archive, None).unwrap()[0].data, + b"hello rar 1.3" + ); + + let mut truncated_comment = archive.entries[0].clone(); + truncated_comment.extra = vec![5, 0, b'x']; + assert_eq!(truncated_comment.file_comment(), Err(Error::TooShort)); + assert!(matches!( + archive.entries[0].verify_checksum(b"wrong data"), + Err(Error::CrcMismatch { .. }) + )); + } + + #[test] + fn writes_and_reads_literal_only_compressed_archive() { + let input = [FileEntry { + name: b"tiny.txt", + data: b"literal bytes over sixteen", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.main.flags, 0x80); + assert_eq!(archive.entries.len(), 1); + assert_eq!(archive.entries[0].name, b"tiny.txt"); + assert!(archive.entries[0].is_stored()); + assert_eq!(archive.entries[0].header.method, METHOD_STORE); + assert_eq!( + archive.entries[0].header.pack_size, + input[0].data.len() as u32 + ); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b"literal bytes over sixteen"); + } + + #[test] + fn writes_and_reads_literal_only_compressed_archive_with_repeated_stmode() { + let data = + b"this literal-only payload is long enough to enter and exit stmode more than once"; + let input = [FileEntry { + name: b"long.txt", + data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.entries[0].header.method, METHOD_BEST); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn compressed_writer_levels_control_rar15_encoder_policy() { + let mut data: Vec<_> = (0..5000).map(|index| (index * 73 + 19) as u8).collect(); + data.extend_from_within(..256); + let input = [FileEntry { + name: b"level-policy.bin", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let level_one = + write_compressed_archive(&input, WriterOptions::default().with_compression_level(1)) + .unwrap(); + let level_five = + write_compressed_archive(&input, WriterOptions::default().with_compression_level(5)) + .unwrap(); + let level_one = Archive::parse(&level_one).unwrap(); + let level_five = Archive::parse(&level_five).unwrap(); + let level_one_file = &level_one.entries[0]; + let level_five_file = &level_five.entries[0]; + + assert_eq!(level_one_file.header.method, METHOD_BEST); + assert_eq!(level_five_file.header.method, METHOD_BEST); + assert!(level_five_file.header.pack_size < level_one_file.header.pack_size); + assert_eq!(collect_extract(&level_one, None).unwrap()[0].data, data); + assert_eq!(collect_extract(&level_five, None).unwrap()[0].data, data); + } + + #[test] + fn rar14_writer_uses_old_distance_tokens() { + for level in 0..=5 { + let options = rar15_encode_options_for_level(Some(level)); + assert!( + options.old_distance_tokens_enabled(), + "RAR 1.4 level {level} should consider old-distance tokens" + ); + } + } + + #[test] + fn rar14_level_five_uses_lazy_matching() { + for level in 0..5 { + assert!(!rar15_encode_options_for_level(Some(level)).lazy_matching_enabled()); + } + assert!(rar15_encode_options_for_level(Some(5)).lazy_matching_enabled()); + } + + #[test] + fn compressed_writer_keeps_adaptive_lz_planning_in_sync_after_literals() { + let data = synthetic_log_payload(8000); + let input = [FileEntry { + name: b"synthetic.log", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = + write_compressed_archive(&input, WriterOptions::default().with_compression_level(2)) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + + assert_eq!(extracted[0].data, data); + } + + #[test] + fn compressed_writer_emits_short_lz_matches() { + let data = b"abcabcabcabcabcabcabcabcabcabcabcabc"; + let input = [FileEntry { + name: b"repeat.txt", + data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.entries[0].header.method, METHOD_BEST); + assert!( + archive.entries[0].header.pack_size < data.len() as u32, + "ShortLZ should make the repeated payload smaller than stored data" + ); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn compressed_writer_emits_long_lz_matches() { + let mut data = short_lz_resistant_prefix(300); + data.extend_from_within(..32); + assert_eq!( + find_long_lz(&data, 300, 0x8000), + Some(LongLz { + distance: 44, + length: 32 + }) + ); + let input = [FileEntry { + name: b"far.txt", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let literal_only = Unpack15Encoder::new() + .encode_literals_only(&data) + .unwrap() + .len(); + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.entries[0].header.method, METHOD_BEST); + assert!( + (archive.entries[0].header.pack_size as usize) < literal_only, + "LongLZ should make a >256-byte-distance repeat smaller than literal-only output" + ); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn compressed_writer_stores_incompressible_member_when_smaller() { + let mut state = 0x8765_4321u32; + let data: Vec<_> = (0..8192) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let input = [FileEntry { + name: b"randomish.bin", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.entries[0].header.method, METHOD_STORE); + assert_eq!(archive.entries[0].header.pack_size, data.len() as u32); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, data); + } + + #[test] + fn compressed_writer_stores_tiny_incompressible_member_when_smaller() { + let data = b"\x00\xff\x12\xed\x34\xcb\x56\xa9\x78\x87\x9a\x65\xbc\x43\xde\x21"; + let input = [FileEntry { + name: b"tiny.bin", + data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.entries[0].header.method, METHOD_STORE); + assert_eq!(archive.entries[0].header.pack_size, data.len() as u32); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, data); + } + + #[test] + fn writes_and_reads_solid_compressed_archive() { + let input = [ + FileEntry { + name: b"first.txt", + data: b"first member primes the adaptive unpack15 state", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + FileEntry { + name: b"second.txt", + data: b"second member is encoded without resetting that state", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = WriterOptions { + target: ArchiveVersion::Rar14, + features, + ..WriterOptions::default() + }; + + let bytes = write_compressed_archive(&input, options).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.is_solid()); + assert_eq!(archive.entries.len(), 2); + assert!(archive + .entries + .iter() + .all(|entry| entry.header.flags & LHD_SOLID != 0)); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, input[0].data); + assert_eq!(extracted[1].data, input[1].data); + } + + #[test] + #[cfg(feature = "encryption")] + fn writes_and_reads_encrypted_compressed_archive() { + let input = [FileEntry { + name: b"secret.txt", + data: b"secret compressed bytes over sixteen", + file_time: 0, + file_attr: 0x20, + password: Some(b"pass"), + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.entries[0].is_encrypted()); + assert_eq!(archive.entries[0].header.method, METHOD_STORE); + assert!(matches!( + collect_extract(&archive, None), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract(&archive, Some(b"pass")).unwrap(); + assert_eq!(extracted[0].data, input[0].data); + } + + #[test] + fn writes_and_reads_compressed_file_comment() { + let input = [FileEntry { + name: b"commented.txt", + data: b"compressed member with file comment", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: Some(b"compressed file comment"), + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!( + archive.entries[0].file_comment().unwrap().as_deref(), + Some(&b"compressed file comment"[..]) + ); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, input[0].data); + } + + #[test] + fn writes_and_reads_stored_multivolume_archive() { + let entry = StoredEntry { + name: b"random.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + + let bytes = write_stored_volumes(entry, WriterOptions::default(), 10).unwrap(); + assert_eq!(bytes.len(), 4); + let volumes: Vec<_> = bytes + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect(); + assert!(volumes.iter().all(|archive| archive.main.is_volume())); + assert!(!volumes[0].entries[0].is_split_before()); + assert!(volumes[0].entries[0].is_split_after()); + assert!(volumes[1].entries[0].is_split_before()); + assert!(volumes[1].entries[0].is_split_after()); + assert!(volumes[3].entries[0].is_split_before()); + assert!(!volumes[3].entries[0].is_split_after()); + assert!(volumes.iter().all(|archive| archive.entries[0].is_stored())); + + let extracted = collect_extract_volumes(&volumes, None).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"random.bin"); + assert_eq!(extracted[0].data, entry.data); + } + + #[test] + fn writes_and_reads_compressed_multivolume_archive() { + let data = b"abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc"; + let entry = FileEntry { + name: b"repeat.txt", + data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + + let bytes = write_compressed_volumes(entry, WriterOptions::default(), 8).unwrap(); + assert!(bytes.len() >= 2); + let volumes: Vec<_> = bytes + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect(); + assert!(volumes.iter().all(|archive| archive.main.is_volume())); + assert!(!volumes[0].entries[0].is_stored()); + assert!(volumes[0].entries[0].is_split_after()); + assert!(volumes.last().unwrap().entries[0].is_split_before()); + assert!(!volumes.last().unwrap().entries[0].is_split_after()); + + let extracted = collect_extract_volumes(&volumes, None).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"repeat.txt"); + assert_eq!(extracted[0].data, data); + } + + fn short_lz_resistant_prefix(len: usize) -> Vec { + let mut data = Vec::with_capacity(len); + while data.len() < len { + let next = (0u8..=u8::MAX) + .find(|&candidate| { + if data.len() < 2 { + return true; + } + let start = data.len().saturating_sub(256); + !data[start..].windows(3).any(|window| { + window == [data[data.len() - 2], data[data.len() - 1], candidate] + }) + }) + .expect("byte alphabet can avoid local 3-byte repeats"); + data.push(next); + } + data + } + + #[test] + fn writes_empty_compressed_archive_member() { + let input = [FileEntry { + name: b"empty.bin", + data: b"", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.entries[0].header.method, METHOD_STORE); + assert_eq!(archive.entries[0].header.pack_size, 0); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, b""); + } + + #[test] + fn rejects_rar5_only_features_for_rar13() { + let mut features = FeatureSet::store_only(); + features.quick_open = true; + + let options = WriterOptions { + target: ArchiveVersion::Rar13, + features, + ..WriterOptions::default() + }; + let err = write_stored_archive(&[], options).unwrap_err(); + assert_eq!( + err, + Error::UnsupportedWriterOption { + target: ArchiveVersion::Rar13, + option: WriterOption::Feature(crate::Feature::QuickOpen), + because: None, + } + ); + assert_eq!( + err.to_string(), + "a quick-open index is not supported by rar13" + ); + } + + #[test] + fn rejects_header_encryption_for_rar13() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + + let options = WriterOptions { + target: ArchiveVersion::Rar14, + features, + ..WriterOptions::default() + }; + let err = write_stored_archive(&[], options).unwrap_err(); + if cfg!(feature = "encryption") { + assert_eq!( + err.to_string(), + "header encryption is not supported by rar14" + ); + } else { + assert_eq!( + err, + Error::FeatureDisabled { + feature: "encryption" + } + ); + } + } + + #[test] + fn file_checksum_matches_rar13_algorithm() { + assert_eq!(file_checksum(b""), 0x0000); + assert_eq!(file_checksum(b"123456789"), 0xc78a); + } + + #[test] + fn rar13_checksum_writer_flush_propagates_to_inner_writer() { + struct FlushSpy { + data: Vec, + flushed: usize, + } + impl Write for FlushSpy { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.data.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.flushed += 1; + Ok(()) + } + } + let mut inner = FlushSpy { + data: Vec::new(), + flushed: 0, + }; + let mut checksum = Rar13Checksum::new(); + let mut writer = Rar13ChecksumWriter { + inner: &mut inner, + checksum: &mut checksum, + }; + writer.write_all(b"hello").unwrap(); + writer.flush().unwrap(); + assert_eq!(inner.data, b"hello"); + assert_eq!(inner.flushed, 1); + } + + #[test] + fn entry_packed_data_returns_borrowed_slice_for_memory_archives() { + let payload = b"packed_data direct accessor coverage"; + let input = [StoredEntry { + name: b"slice.bin", + data: payload, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let entry = &archive.entries[0]; + + let packed = entry.packed_data(&archive).unwrap(); + assert_eq!(packed, payload); + assert!(!packed.is_empty()); + } + + #[test] + fn extract_volumes_to_annotates_failed_non_split_entry_with_at_entry() { + let payload = b"corrupt-me-please"; + let input = [StoredEntry { + name: b"plain.bin", + data: payload, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let mut bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let range = archive.entries[0].packed_range.clone(); + // Flip a byte in the stored payload so the checksum no longer matches. + bytes[range.start] ^= 0xff; + + let corrupted = Archive::parse(&bytes).unwrap(); + let err = collect_extract_volumes(std::slice::from_ref(&corrupted), None).unwrap_err(); + match err { + Error::AtEntry { + name, + operation, + source, + } => { + assert_eq!(name, b"plain.bin"); + assert_eq!(operation, "extracting"); + assert!(matches!(*source, Error::CrcMismatch { .. })); + } + other => panic!("expected AtEntry annotation, got {other:?}"), + } + } + + #[test] + fn extract_volumes_to_annotates_failed_split_completion_with_at_entry() { + let entry = StoredEntry { + name: b"split.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + + let mut volume_bytes = write_stored_volumes(entry, WriterOptions::default(), 10).unwrap(); + assert!( + volume_bytes.len() >= 2, + "need at least two volumes to exercise the split-completion path" + ); + + // Corrupt the last fragment so PendingSplitRefs::write_to fails on assembly. + let last_index = volume_bytes.len() - 1; + let last_archive = Archive::parse(&volume_bytes[last_index]).unwrap(); + let last_range = last_archive.entries[0].packed_range.clone(); + volume_bytes[last_index][last_range.start] ^= 0x7f; + + let volumes: Vec<_> = volume_bytes + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect(); + + let err = collect_extract_volumes(&volumes, None).unwrap_err(); + match err { + Error::AtEntry { + name, + operation, + source, + } => { + assert_eq!(name, b"split.bin"); + assert_eq!(operation, "extracting"); + assert!( + matches!(*source, Error::CrcMismatch { .. }), + "expected CrcMismatch source, got {source:?}" + ); + } + other => panic!("expected AtEntry annotation, got {other:?}"), + } + } + + #[test] + fn entry_packed_data_refuses_to_buffer_file_backed_archives() { + let payload = b"packed_data refuses file-backed"; + let input = [StoredEntry { + name: b"file.bin", + data: payload, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + + let dir = crate::scratch::case("rars-rar13-packed-data"); + let path = dir.join("entry.rar"); + std::fs::write(&path, &bytes).unwrap(); + + let archive = Archive::parse_path(&path).unwrap(); + let result = archive.entries[0].packed_data(&archive); + assert_eq!( + result, + Err(Error::InvalidHeader( + "RAR 1.3 file-backed packed data requires owned read" + )) + ); + } + + fn parse_volumes(bytes: &[Vec]) -> Vec { + bytes.iter().map(|b| Archive::parse(b).unwrap()).collect() + } + + fn split_volumes_for(name: &[u8], data: &[u8]) -> Vec> { + write_stored_volumes( + StoredEntry { + name, + data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + WriterOptions::default(), + 10, + ) + .unwrap() + } + + #[test] + fn extract_volumes_to_rejects_pending_split_interrupted_by_regular_entry() { + let bytes = split_volumes_for(b"split.bin", b"abcdefghijklmnopqrstuvwxyz"); + let mut volumes = parse_volumes(&bytes); + + // After volume 0's split_after entry, append a regular entry to the + // same volume so the loop sees pending=Some when it hits a non-split. + let mut intruder = volumes[0].entries[0].clone(); + intruder.header.flags &= !(LHD_SPLIT_BEFORE | LHD_SPLIT_AFTER); + intruder.name = b"intruder.bin".to_vec(); + volumes[0].entries.push(intruder); + + let err = collect_extract_volumes(&volumes, None).unwrap_err(); + assert_eq!( + err, + Error::InvalidHeader("RAR 1.3 split entry is interrupted by a regular entry"), + ); + } + + #[test] + fn extract_volumes_to_rejects_split_with_inconsistent_flags() { + let bytes = split_volumes_for(b"split.bin", b"abcdefghijklmnopqrstuvwxyz"); + let volumes = parse_volumes(&bytes); + + // Take just the *middle* volume in isolation: it has split_before=true + // but no preceding pending state, which the match arm treats as + // structurally inconsistent. + let middle = volumes.into_iter().nth(1).unwrap(); + let err = collect_extract_volumes(std::slice::from_ref(&middle), None).unwrap_err(); + assert_eq!( + err, + Error::InvalidHeader("RAR 1.3 split entry flags are inconsistent"), + ); + } + + #[test] + fn extract_volumes_to_rejects_pending_split_left_incomplete_at_end() { + let bytes = split_volumes_for(b"split.bin", b"abcdefghijklmnopqrstuvwxyz"); + let volumes = parse_volumes(&bytes); + + // Use only the first volume, which leaves pending=Some after the loop. + let err = collect_extract_volumes(std::slice::from_ref(&volumes[0]), None).unwrap_err(); + assert_eq!( + err, + Error::InvalidHeader("RAR 1.3 split entry is incomplete") + ); + } + + #[test] + fn extract_volumes_to_rejects_split_fragments_with_drifted_attributes() { + let bytes = split_volumes_for(b"split.bin", b"abcdefghijklmnopqrstuvwxyz"); + let mut volumes = parse_volumes(&bytes); + + // Mutate the second volume's entry name so PendingSplitRefs::append + // refuses it on a name-mismatch. + volumes[1].entries[0].name = b"different.bin".to_vec(); + + let err = collect_extract_volumes(&volumes, None).unwrap_err(); + assert_eq!( + err, + Error::InvalidHeader("RAR 1.3 split entry name changed") + ); + } + + #[test] + fn extract_volumes_to_rejects_split_fragments_with_drifted_method() { + let bytes = split_volumes_for(b"split.bin", b"abcdefghijklmnopqrstuvwxyz"); + let mut volumes = parse_volumes(&bytes); + + // Drift the compression method on the second fragment. + volumes[1].entries[0].header.method = METHOD_BEST; + let err = collect_extract_volumes(&volumes, None).unwrap_err(); + assert_eq!( + err, + Error::InvalidHeader("RAR 1.3 split entry compression method changed"), + ); + } + + #[test] + fn extract_volumes_to_rejects_split_version_and_encryption_drift() { + let bytes = split_volumes_for(b"split.bin", b"abcdefghijklmnopqrstuvwxyz"); + let mut volumes = parse_volumes(&bytes); + volumes[1].entries[0].header.unp_ver += 1; + assert_eq!( + collect_extract_volumes(&volumes, None).unwrap_err(), + Error::InvalidHeader("RAR 1.3 split entry unpack version changed") + ); + + let mut volumes = parse_volumes(&bytes); + volumes[1].entries[0].header.flags ^= LHD_PASSWORD; + assert_eq!( + collect_extract_volumes(&volumes, None).unwrap_err(), + Error::InvalidHeader("RAR 1.3 split entry encryption flag changed") + ); + } + + #[test] + fn extract_volumes_to_carries_directory_entries_across_volume_array() { + // A directory entry has zero-length data and gets the open() callback + // invoked but no payload write. Putting it in a volumes array keeps + // the directory branch in extract_volumes_to (rather than extract_to) + // exercised. + let input = [StoredEntry { + name: b"docs", + data: b"", + file_time: 0, + file_attr: 0x10, + password: None, + file_comment: None, + }]; + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract_volumes(std::slice::from_ref(&archive), None).unwrap(); + assert_eq!(extracted.len(), 1); + assert!(extracted[0].is_directory); + assert_eq!(extracted[0].name, b"docs"); + } + + #[test] + fn extract_volumes_to_routes_pending_split_reader_through_fragment_chain() { + // Larger payload over more volumes guarantees that the chained + // fragment reader reads from each volume's range_reader at least once, + // exercising the success arms of fragment_reader, write_to, and + // ChainedReader::read across multiple volumes. + let payload: Vec = (0..96).map(|i| ((i * 53) ^ 0xa5) as u8).collect(); + let bytes = split_volumes_for(b"chain.bin", &payload); + assert!( + bytes.len() >= 3, + "need at least three volumes for the chain" + ); + let volumes = parse_volumes(&bytes); + + let extracted = collect_extract_volumes(&volumes, None).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, payload); + } + + #[test] + fn write_compressed_archive_with_comment_round_trips_through_archive_comment() { + let data = b"compressed archive comment payload payload payload"; + let comment = b"This is a compressed archive comment."; + let input = [FileEntry { + name: b"payload.txt", + data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + + let bytes = + write_compressed_archive_with_comment(&input, WriterOptions::default(), Some(comment)) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_archive_comment()); + assert!(archive.main.has_packed_comment()); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&comment[..]) + ); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn write_compressed_archive_with_comment_emits_solid_compressed_archive() { + let data1 = b"solid compressed payload one with overlap overlap overlap"; + let data2 = b"solid compressed payload two with overlap overlap overlap"; + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = WriterOptions { + target: ArchiveVersion::Rar14, + features, + ..WriterOptions::default() + }; + let input = [ + FileEntry { + name: b"a.txt", + data: data1, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + FileEntry { + name: b"b.txt", + data: data2, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + ]; + + let bytes = write_compressed_archive_with_comment(&input, options, None).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.is_solid()); + assert_eq!(archive.entries.len(), 2); + + let extracted = collect_extract(&archive, None).unwrap(); + assert_eq!(extracted[0].data, data1); + assert_eq!(extracted[1].data, data2); + } + + #[test] + fn write_compressed_archive_with_comment_rejects_non_rar13_target() { + let options = WriterOptions { + target: ArchiveVersion::Rar15, + ..WriterOptions::default() + }; + let err = write_compressed_archive_with_comment(&[], options, None).unwrap_err(); + assert_eq!(err, Error::UnsupportedVersion(ArchiveVersion::Rar15)); + } + + #[test] + fn parse_path_round_trips_multi_entry_archive_via_file_backed_seekable_path() { + // Two entries plus an archive comment forces parse_seekable to walk + // through more than one file-header read iteration. + let input = [ + StoredEntry { + name: b"first.txt", + data: b"first payload", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + StoredEntry { + name: b"second.txt", + data: b"second payload", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + ]; + let bytes = write_stored_archive_with_comment( + &input, + WriterOptions::default(), + Some(b"file-backed comment"), + ) + .unwrap(); + + let dir = crate::scratch::case("rars-rar13-parse-seekable"); + let path = dir.join("multi.rar"); + std::fs::write(&path, &bytes).unwrap(); + + let archive = Archive::parse_path(&path).unwrap(); + assert_eq!(archive.entries.len(), 2); + assert_eq!(archive.entries[0].name, b"first.txt"); + assert_eq!(archive.entries[1].name, b"second.txt"); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"file-backed comment"[..]) + ); + } + + #[test] + fn parse_path_rejects_files_without_rar13_signature() { + let dir = crate::scratch::case("rars-rar13-parse-path-bad"); + let path = dir.join("not_a_rar.bin"); + std::fs::write(&path, [0u8; 64]).unwrap(); + + let err = Archive::parse_path(&path).unwrap_err(); + assert_eq!(err, Error::UnsupportedSignature); + } + + #[test] + fn controlled_extraction_uses_selected_directory_and_file_writers() { + let entries = [ + StoredEntry { + name: b"docs", + data: b"", + file_time: 0, + file_attr: 0x10, + password: None, + file_comment: None, + }, + StoredEntry { + name: b"docs/readme.txt", + data: b"old archive data", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + ]; + let bytes = write_stored_archive(&entries, WriterOptions::default()).unwrap(); + let archive = crate::Archive::Rar13(Archive::parse(&bytes).unwrap()); + let output = Rc::new(RefCell::new(Vec::new())); + let mut seen = Vec::new(); + let outcome = archive + .extract_with_control(crate::ArchiveReadOptions::new(), |member| { + seen.push(member.meta.name.clone()); + Ok(crate::ExtractionDecision::Extract(Box::new(CollectWriter( + Rc::clone(&output), + )))) + }) + .unwrap(); + + assert_eq!(outcome, crate::ExtractionOutcome::Complete); + assert_eq!(seen, [b"docs".to_vec(), b"docs/readme.txt".to_vec()]); + assert_eq!(&*output.borrow(), b"old archive data"); + } + + #[test] + fn directory_extraction_skips_payload_even_with_compressed_method() { + let entries = [StoredEntry { + name: b"DIR", + data: b"", + file_time: 0, + file_attr: 0x10, + password: None, + file_comment: None, + }]; + let bytes = write_stored_archive(&entries, WriterOptions::default()).unwrap(); + let mut archive = Archive::parse(&bytes).unwrap(); + archive.entries[0].header.method = METHOD_BEST; + assert!(collect_extract(&archive, None).unwrap()[0].is_directory); + assert!( + collect_extract_volumes(std::slice::from_ref(&archive), None).unwrap()[0].is_directory + ); + } + + #[test] + fn empty_rar13_archive_cannot_supply_preservation_target_version() { + let bytes = write_stored_archive(&[], WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.entries.is_empty()); + assert!(archive + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("empty legacy archive"))); + } + + #[test] + fn streaming_writer_refuses_unsupported_aggregate_quotas_before_output() { + for (resources, feature) in [ + ( + WriterResources::default().with_max_preparation_bytes(0), + "preparation memory quota", + ), + ( + WriterResources::default().with_max_memory_bytes(0), + "aggregate managed-memory limit", + ), + ] { + let mut output = Vec::new(); + let error = write_streaming_archive_to( + &[], + WriterOptions::default(), + MemberCoding::Stored, + None, + &resources, + None, + &mut output, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::UnsupportedFamilyFeature { feature: actual, .. } if actual == feature + )); + assert!(output.is_empty()); + } + } + + #[test] + fn volume_writer_rejects_empty_payload_for_both_codings() { + let stored = StoredEntry { + name: b"empty", + data: b"", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let compressed = FileEntry { + name: stored.name, + data: stored.data, + file_time: stored.file_time, + file_attr: stored.file_attr, + password: None, + file_comment: None, + }; + for error in [ + write_stored_volumes(stored, WriterOptions::default(), 1024).unwrap_err(), + write_compressed_volumes(compressed, WriterOptions::default(), 1024).unwrap_err(), + ] { + assert!(matches!( + error.root_cause(), + Error::InvalidArgument("RAR 1.3 volume writer needs a non-empty packed payload") + )); + } + } + + #[test] + fn archive_comment_rejects_packed_output_beyond_header_field() { + let mut state = 0x1234_5678u32; + let comment: Vec<_> = (0..u16::MAX) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let valid = write_stored_archive_with_comment( + &[], + WriterOptions::default(), + Some(&comment[..64755]), + ) + .unwrap(); + assert_eq!( + Archive::parse(&valid) + .unwrap() + .archive_comment() + .unwrap() + .unwrap(), + comment[..64755] + ); + assert_eq!( + encode_archive_comment(Some(&comment[..64760])) + .unwrap() + .len(), + 65530 + ); + assert_eq!( + write_stored_archive_with_comment( + &[], + WriterOptions::default(), + Some(&comment[..64760]), + ) + .unwrap_err(), + Error::InvalidArgument("RAR 1.3 main header comment extension is too large") + ); + let error = + write_stored_archive_with_comment(&[], WriterOptions::default(), Some(&comment)) + .unwrap_err(); + assert_eq!( + error, + Error::InvalidArgument("RAR 1.3 packed archive comment is longer than 65535 bytes") + ); + } + + #[test] + fn supplied_rar13_signature_and_file_length_are_checked() { + let entries = [StoredEntry { + name: b"member", + data: b"payload", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + let bytes = write_stored_archive(&entries, WriterOptions::default()).unwrap(); + let signature = crate::detect_archive_family(&bytes).unwrap(); + let dir = crate::scratch::case("rars-rar13-supplied-signature"); + let path = dir.join("archive.rar"); + std::fs::write(&path, &bytes).unwrap(); + + assert_eq!( + Archive::parse_owned(bytes.clone()).unwrap().entries.len(), + 1 + ); + assert_eq!( + Archive::parse_path_with_signature(&path, signature) + .unwrap() + .entries + .len(), + 1 + ); + assert_eq!( + Archive::parse_path_with_signature_and_options( + &path, + crate::ArchiveSignature { + family: ArchiveFamily::Rar15To40, + ..signature + }, + crate::ArchiveReadOptions::new(), + ) + .unwrap_err(), + Error::UnsupportedSignature + ); + + let mut wrong_magic = bytes.clone(); + wrong_magic[0] = 0; + std::fs::write(&path, &wrong_magic).unwrap(); + assert_eq!( + Archive::parse_path_with_signature(&path, signature).unwrap_err(), + Error::UnsupportedSignature + ); + + let mut oversized_payload = bytes; + oversized_payload[7..11].copy_from_slice(&u32::MAX.to_le_bytes()); + std::fs::write(&path, &oversized_payload).unwrap(); + assert_eq!( + Archive::parse_owned(oversized_payload).unwrap_err(), + Error::TooShort + ); + assert_eq!( + Archive::parse_path_with_signature(&path, signature).unwrap_err(), + Error::TooShort + ); + + std::fs::write(&path, crate::detect::RAR15_SIGNATURE).unwrap(); + assert_eq!( + Archive::parse_path(&path).unwrap_err(), + Error::UnsupportedSignature + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn extract_to_encrypted_archive_reads_through_file_backed_decrypted_range() { + // The Memory-backed path is already exercised; this test takes the + // same encrypted archive out to disk so copy_decrypted_range_to runs + // its ArchiveSource::File branch. + let input = [StoredEntry { + name: b"secret.bin", + data: b"file-backed secret payload", + file_time: 0, + file_attr: 0x20, + password: Some(b"pw"), + file_comment: None, + }]; + let bytes = write_stored_archive(&input, WriterOptions::default()).unwrap(); + + let dir = crate::scratch::case("rars-rar13-decrypt-file"); + let path = dir.join("encrypted.rar"); + std::fs::write(&path, &bytes).unwrap(); + + let archive = Archive::parse_path(&path).unwrap(); + let extracted = collect_extract(&archive, Some(b"pw")).unwrap(); + assert_eq!(extracted[0].data, b"file-backed secret payload"); + } + + #[test] + fn write_stored_volumes_rejects_password_protected_entries() { + let entry = StoredEntry { + name: b"locked.bin", + data: b"data", + file_time: 0, + file_attr: 0x20, + password: Some(b"pw"), + file_comment: None, + }; + let err = write_stored_volumes(entry, WriterOptions::default(), 16).unwrap_err(); + assert_eq!( + err, + Error::UnsupportedWriterOption { + target: ArchiveVersion::Rar14, + option: WriterOption::Password, + because: Some("in a volume set"), + } + ); + assert_eq!( + err.to_string(), + "encryption is not supported by rar14 (in a volume set)" + ); + } + + #[test] + fn write_compressed_volumes_rejects_file_comments() { + let entry = FileEntry { + name: b"with-comment.bin", + data: b"data", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: Some(b"note"), + }; + let err = write_compressed_volumes(entry, WriterOptions::default(), 16).unwrap_err(); + assert_eq!( + err.to_string(), + "a per-file comment is not supported by rar14 (in a volume set)" + ); + } + + #[test] + fn write_compressed_volumes_rejects_non_rar13_target() { + let options = WriterOptions { + target: ArchiveVersion::Rar20, + ..WriterOptions::default() + }; + let entry = FileEntry { + name: b"x.bin", + data: b"data", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let err = write_compressed_volumes(entry, options, 16).unwrap_err(); + assert_eq!(err, Error::UnsupportedVersion(ArchiveVersion::Rar20)); + } + + #[test] + fn file_header_parse_rejects_input_below_base_size() { + let err = FileHeader::parse(&[0u8; FILE_HEAD_BASE_SIZE - 1]).unwrap_err(); + assert_eq!(err, Error::TooShort); + } + + #[test] + fn file_header_parse_rejects_truncated_input_against_declared_head_size() { + // Build a syntactically OK FILE_HEAD_BASE_SIZE buffer that declares a + // head_size larger than the slice we pass in — exercises the + // post-name-size length check at the end of FileHeader::parse. + let mut header = [0u8; FILE_HEAD_BASE_SIZE]; + // pack_size, unp_size, file_crc, file_time stay zero. + let declared_head_size: u16 = (FILE_HEAD_BASE_SIZE + 32) as u16; + header[10..12].copy_from_slice(&declared_head_size.to_le_bytes()); + // name_size = 0 keeps minimum_size == FILE_HEAD_BASE_SIZE so the + // earlier "shorter than its name" branch is bypassed. + header[19] = 0; + let err = FileHeader::parse(&header).unwrap_err(); + assert_eq!(err, Error::TooShort); + } + + #[test] + fn archive_comment_rejects_size_field_shorter_than_two_bytes() { + // Build a valid stored archive then patch its main header so the + // declared comment field is shorter than two bytes — exercises the + // "packed archive comment is shorter than size field" arm. + let input = [StoredEntry { + name: b"file.bin", + data: b"data", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + let bytes = + write_stored_archive_with_comment(&input, WriterOptions::default(), Some(b"hi")) + .unwrap(); + let mut archive = Archive::parse(&bytes).unwrap(); + // The first two bytes of `main.extra` are the comment_field length — + // overwrite them with 1 to declare a sub-2-byte payload while keeping + // the packed-comment flag set. + archive.main.extra[0] = 1; + archive.main.extra[1] = 0; + assert_eq!( + archive.archive_comment(), + Err(Error::InvalidHeader( + "RAR 1.3 packed archive comment is shorter than size field" + )) + ); + } + + #[test] + fn archive_comment_rejects_packed_payload_extending_past_extra_buffer() { + let input = [StoredEntry { + name: b"file.bin", + data: b"data", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }]; + let bytes = + write_stored_archive_with_comment(&input, WriterOptions::default(), Some(b"hi")) + .unwrap(); + let mut archive = Archive::parse(&bytes).unwrap(); + // Pump the declared comment field length up so the packed range walks + // past the end of `main.extra`. + let inflated = (archive.main.extra.len() as u16 + 16).to_le_bytes(); + archive.main.extra[0] = inflated[0]; + archive.main.extra[1] = inflated[1]; + assert_eq!(archive.archive_comment(), Err(Error::TooShort)); + } + + #[test] + fn streaming_writer_attributes_source_size_failures() { + let missing = crate::scratch::case("rars-rar13-missing-source").join("absent"); + for (source, expected, operation) in [ + ( + EntrySource::from_path(missing), + crate::ErrorKind::Io, + "reading source", + ), + ( + EntrySource::from_opener(u64::from(u32::MAX) + 1, || { + unreachable!("oversized source must fail before opening") + }), + crate::ErrorKind::InvalidArgument, + "preparing", + ), + ] { + let entry = StreamingEntry::new(b"SOURCE.BIN".to_vec(), source); + let mut output = Vec::new(); + let error = write_streaming_archive_to( + &[entry], + WriterOptions::default(), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), expected); + assert_eq!(error.entry_context(), Some((&b"SOURCE.BIN"[..], operation))); + } + } + + #[test] + fn file_comment_respects_the_total_file_header_limit() { + let max_comment_len = u16::MAX as usize - FILE_HEAD_BASE_SIZE - 1 - 2; + let comment = vec![b'x'; max_comment_len + 1]; + let entry = StoredEntry { + name: b"f", + data: b"payload", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: Some(&comment[..max_comment_len]), + }; + let valid = write_stored_archive(&[entry], WriterOptions::default()).unwrap(); + let archive = Archive::parse(&valid).unwrap(); + assert_eq!( + archive.entries[0].file_comment().unwrap(), + Some(comment[..max_comment_len].to_vec()) + ); + + let oversized = StoredEntry { + file_comment: Some(&comment), + ..entry + }; + assert!(matches!( + write_stored_archive(&[oversized], WriterOptions::default()) + .unwrap_err() + .root_cause(), + Error::InvalidArgument("RAR 1.3 file header is longer than 65535 bytes") + )); + } + + #[test] + fn verified_encoder_retries_corrupt_payload_then_accepts_valid_fallback() { + let data = b"legacy archive payload legacy archive payload"; + let valid = unpack15_encode(data).unwrap(); + let options = Rar15EncodeOptions::new(); + let mut attempts = 0; + let packed = + encode_verified_rar15_payload_using(data, options, &mut |_| true, |_, _, _| { + attempts += 1; + Ok(if attempts == 1 { + Vec::new() + } else { + valid.clone() + }) + }) + .unwrap(); + assert_eq!(packed, Some(valid)); + assert_eq!(attempts, 2); + } + + #[test] + fn verified_encoder_progress_counts_each_retry_once() { + let data = b"legacy fallback progress payload"; + let valid = unpack15_encode(data).unwrap(); + let completed = std::sync::Mutex::new(Vec::new()); + let reporter = |event: WriteProgressEvent<'_>| { + if let WriteProgressEvent::Advanced { + completed_bytes, .. + } = event + { + completed.lock().unwrap().push(completed_bytes); + } + }; + let work = WorkTracker::new( + Some(ProgressReporter(&reporter)), + WriteOperation::Compression, + (data.len() * 3) as u64, + ); + let mut last = 0; + let mut progress = |position| advance_rar15_attempt(&mut last, &work, position); + let mut attempts = 0; + let packed = encode_verified_rar15_payload_using( + data, + Rar15EncodeOptions::new(), + &mut progress, + |input, _, report| { + attempts += 1; + assert!(report(0)); + assert!(report(input.len() / 2)); + assert!(report(input.len())); + Ok(if attempts == 1 { + Vec::new() + } else { + valid.clone() + }) + }, + ) + .unwrap(); + assert_eq!(packed, Some(valid)); + assert_eq!(attempts, 2); + let half = (data.len() / 2) as u64; + let full = data.len() as u64; + assert_eq!( + *completed.lock().unwrap(), + [0, half, full, full, full + half, full * 2] + ); + } + + #[test] + fn verified_payload_selection_preserves_storage_and_solid_history() { + let data = b"original payload"; + let (payload, method) = + select_verified_payload(std::borrow::Cow::Borrowed(data), Some(vec![1, 2]), false); + assert_eq!(payload, [1, 2]); + assert_eq!(method, METHOD_BEST); + + let owned = data.to_vec(); + let allocation = owned.as_ptr(); + let (payload, method) = select_verified_payload( + std::borrow::Cow::Owned(owned), + Some(vec![0; data.len() + 1]), + false, + ); + assert_eq!(payload, data); + assert_eq!(payload.as_ptr(), allocation); + assert_eq!(method, METHOD_STORE); + + let packed = vec![0; data.len() + 1]; + let (payload, method) = + select_verified_payload(std::borrow::Cow::Borrowed(data), Some(packed.clone()), true); + assert_eq!(payload, packed); + assert_eq!(method, METHOD_BEST); + } + + #[test] + fn verified_encoder_stores_when_every_candidate_fails_verification() { + let data = b"legacy archive payload"; + let mut attempts = 0; + let packed = encode_verified_rar15_payload_using( + data, + Rar15EncodeOptions::new(), + &mut |_| true, + |_, _, _| { + attempts += 1; + Ok(Vec::new()) + }, + ) + .unwrap(); + assert_eq!(packed, None); + let (payload, method) = + select_verified_payload(std::borrow::Cow::Borrowed(data), packed, false); + assert_eq!(payload, data); + assert_eq!(method, METHOD_STORE); + assert_eq!( + attempts, + rar15_encode_fallback_options(Rar15EncodeOptions::new()).len() + ); + } + + #[test] + fn verified_encoder_propagates_cancellation_during_fallback() { + let data = b"legacy archive payload"; + let mut attempts = 0; + let error = encode_verified_rar15_payload_using( + data, + Rar15EncodeOptions::new(), + &mut |_| true, + |_, _, _| { + attempts += 1; + if attempts == 1 { + Ok(Vec::new()) + } else { + Err(crate::codec::Error::Cancelled) + } + }, + ) + .unwrap_err(); + assert_eq!(error, Error::Cancelled); + assert_eq!(attempts, 2); + } + + #[test] + fn streaming_writer_detects_source_size_change_between_passes() { + struct ChangingLength(AtomicUsize); + impl crate::streaming::SourceFactory for ChangingLength { + fn len(&self) -> Result { + if self.0.fetch_add(1, Ordering::Relaxed) == 0 { + Ok(3) + } else { + Err(Error::SourceChanged("source size changed")) + } + } + + fn open(&self) -> Result> { + unreachable!("changed source must fail before opening") + } + } + let entry = StreamingEntry::new( + b"CHANGED.BIN".to_vec(), + EntrySource::from_factory(ChangingLength(AtomicUsize::new(0))), + ); + let error = write_streaming_archive_to( + &[entry], + WriterOptions::default(), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::SourceChanged); + assert_eq!( + error.entry_context(), + Some((&b"CHANGED.BIN"[..], "reading source")) + ); + } + + #[test] + fn compression_level_zero_writes_a_stored_member() { + let entry = FileEntry { + name: b"ZERO.TXT", + data: b"zero-level compression leaves these bytes stored", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let bytes = + write_compressed_archive(&[entry], WriterOptions::default().with_compression_level(0)) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.entries[0].is_stored()); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, entry.data); + } + + #[test] + fn level_four_uses_a_distinct_verified_fallback_plan() { + let options = rar15_encode_options_for_level(Some(4)); + let candidates = rar15_encode_fallback_options(options); + assert_eq!(candidates.len(), 2); + + let data = b"level four compressed legacy payload ".repeat(32); + let entry = FileEntry { + name: b"LEVEL4.TXT", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let bytes = + write_compressed_archive(&[entry], WriterOptions::default().with_compression_level(4)) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(collect_extract(&archive, None).unwrap()[0].data, data); + } + + #[test] + fn stored_volume_writer_refuses_non_legacy_target() { + let entry = StoredEntry { + name: b"WRONG.TXT", + data: b"contents", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let options = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()); + assert_eq!( + write_stored_volumes(entry, options, 8).unwrap_err(), + Error::UnsupportedVersion(ArchiveVersion::Rar50) + ); + } + + #[test] + fn solid_volume_writer_marks_every_fragment() { + let data = b"a stored member across several solid volumes".repeat(16); + let entry = FileEntry { + name: b"SOLID.TXT", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = WriterOptions::new(ArchiveVersion::Rar14, features); + let volumes = write_compressed_volumes(entry, options, 8).unwrap(); + assert!(volumes.len() > 1); + let archives = parse_volumes(&volumes); + assert!(archives + .iter() + .all(|archive| archive.entries[0].header.flags & LHD_SOLID != 0)); + assert_eq!( + collect_extract_volumes(&archives, None).unwrap()[0].data, + entry.data + ); + } + + #[test] + fn final_archive_progress_report_can_cancel() { + struct CancelOnAdvance(AtomicBool); + impl WriteProgress for CancelOnAdvance { + fn report(&self, event: WriteProgressEvent<'_>) { + if matches!(event, WriteProgressEvent::Advanced { .. }) { + self.0.store(true, Ordering::Relaxed); + } + } + + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + let reporter = CancelOnAdvance(AtomicBool::new(false)); + let error = write_streaming_archive_to( + &[], + WriterOptions::default(), + MemberCoding::Stored, + None, + &WriterResources::default(), + Some(&reporter), + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!(error, Error::Cancelled); + } + + #[test] + fn final_volume_progress_report_can_cancel() { + struct CancelOnFinalAdvance { + entry_finished: AtomicBool, + cancelled: AtomicBool, + } + impl WriteProgress for CancelOnFinalAdvance { + fn report(&self, event: WriteProgressEvent<'_>) { + match event { + WriteProgressEvent::EntryFinished { .. } => { + self.entry_finished.store(true, Ordering::Relaxed); + } + WriteProgressEvent::Advanced { .. } + if self.entry_finished.load(Ordering::Relaxed) => + { + self.cancelled.store(true, Ordering::Relaxed); + } + _ => {} + } + } + + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } + } + let reporter = CancelOnFinalAdvance { + entry_finished: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + }; + let entry = FileEntry { + name: b"FINAL.TXT", + data: b"final progress cancellation final progress cancellation", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let error = write_compressed_volumes_with_progress( + entry, + WriterOptions::default(), + 16, + Some(&reporter), + ) + .unwrap_err(); + assert_eq!(error, Error::Cancelled); + assert!(reporter.entry_finished.load(Ordering::Relaxed)); + } + + #[test] + fn preservation_reports_unsupported_legacy_header_variants() { + let entry = StoredEntry { + name: b"DIR", + data: b"", + file_time: 0, + file_attr: 0x10, + password: None, + file_comment: None, + }; + let bytes = write_stored_archive(&[entry], WriterOptions::default()).unwrap(); + let base = Archive::parse(&bytes).unwrap(); + assert!(base.rewrite_preservation_issues().is_empty()); + + for mutate in [ + (|archive: &mut Archive| archive.main.flags |= MHD_PACK_COMMENT) as fn(&mut Archive), + |archive| archive.main.extra.push(0), + |archive| archive.entries[0].header.method = METHOD_BEST + 1, + |archive| archive.entries[0].header.flags |= LHD_SOLID, + |archive| archive.entries[0].header.pack_size = 1, + |archive| archive.entries[0].header.unp_size = 1, + ] { + let mut archive = base.clone(); + mutate(&mut archive); + assert!(!archive.rewrite_preservation_issues().is_empty()); + } + } + + #[test] + fn volume_extraction_keeps_solid_history_across_regular_members() { + let entries = [ + FileEntry { + name: b"FIRST.TXT", + data: b"first member establishes solid history first member", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + FileEntry { + name: b"SECOND.TXT", + data: b"first member establishes solid history plus a tail", + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar14, features), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract_volumes(std::slice::from_ref(&archive), None).unwrap(); + assert_eq!(extracted.len(), entries.len()); + for (actual, entry) in extracted.iter().zip(entries) { + assert_eq!(actual.data, entry.data); + } + } + + #[test] + fn streaming_writer_rejects_directory_payload() { + let entry = StreamingEntry::new( + b"directory".to_vec(), + EntrySource::from_bytes(b"unexpected payload".to_vec()), + ) + .with_file_attr(0x10); + let error = write_streaming_archive_to( + &[entry], + WriterOptions::default(), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut Vec::new(), + ) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::InvalidArgument("RAR1.3/1.4 directories must have no payload") + )); + assert_eq!( + error.entry_context(), + Some((&b"directory"[..], "preparing")) + ); + } + + #[test] + fn compressed_volumes_report_each_volume_and_honor_cancellation() { + let data = b"a repeated sequence of archive bytes ".repeat(64); + let entry = FileEntry { + name: b"VOL.TXT", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let volume_count = AtomicUsize::new(0); + let reporter = |event: WriteProgressEvent<'_>| { + if let WriteProgressEvent::VolumeFinished { volume_number, .. } = event { + assert_eq!( + volume_number, + volume_count.fetch_add(1, Ordering::Relaxed) + 1 + ); + } + }; + let volumes = write_compressed_volumes_with_progress( + entry, + WriterOptions::default(), + 32, + Some(&reporter), + ) + .unwrap(); + assert_eq!(volume_count.load(Ordering::Relaxed), volumes.len()); + assert!(volumes.len() > 1); + + struct CancelAfterFirstVolume(AtomicUsize); + impl WriteProgress for CancelAfterFirstVolume { + fn report(&self, event: WriteProgressEvent<'_>) { + if matches!(event, WriteProgressEvent::VolumeFinished { .. }) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) != 0 + } + } + let cancelling = CancelAfterFirstVolume(AtomicUsize::new(0)); + let error = write_compressed_volumes_with_progress( + entry, + WriterOptions::default(), + 32, + Some(&cancelling), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Cancelled); + assert_eq!(cancelling.0.load(Ordering::Relaxed), 1); + } +} diff --git a/arca-rar/vendor/rars/src/rar15_40.rs b/arca-rar/vendor/rars/src/rar15_40.rs new file mode 100644 index 00000000..5710bdd6 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar15_40.rs @@ -0,0 +1,5281 @@ +use crate::codec::rar13::Reader15State; +use crate::codec::rar20::Reader20State; +use crate::codec::workspace::{Allowance, Budget, Buffer}; +use crate::crc32::{crc32, Crc32}; +#[cfg(feature = "encryption")] +use crate::crypto::rar15::Rar15Cipher; +#[cfg(feature = "encryption")] +use crate::crypto::rar20::Rar20Cipher; +#[cfg(feature = "encryption")] +use crate::crypto::rar30::{Error as Rar30Error, Rar30Cipher}; +use crate::detect::{find_archive_start, ArchiveSignature, RAR15_SIGNATURE, SFX_SCAN_LIMIT}; +use crate::error::{Error, Result}; +use crate::io_util::{read_exact_at, read_u32}; +pub(crate) use crate::source::ArchiveSource; +use crate::version::ArchiveFamily; +#[cfg(any(feature = "write", feature = "recovery"))] +use crate::ArchiveVersion; +use std::fs::File; +use std::io::{Read, Write}; +use std::ops::Range; +use std::path::Path; +use std::sync::Arc; + +pub use crate::filter::{FilterKind, FilterPolicy, FilterSpec}; +#[cfg(feature = "write")] +pub use write::{FileEntry, Rar29Method, StoredEntry, StreamingEntry, WriterOptions}; + +mod extract; +#[cfg(feature = "write")] +mod write; +#[cfg(feature = "write")] +pub use crate::streaming::{EntrySource, WriterResources}; +#[cfg(feature = "write")] +pub use crate::write_plan::MemberCoding; +pub use extract::extract_volumes_to; +use extract::{DecoderSession, DecryptingReader, PackedReader}; +#[cfg(feature = "write")] +pub(crate) use write::write_stored_volumes_with_progress; +#[cfg(feature = "write")] +pub(crate) use write::{ + write_archive_with_retained_metadata, RetainedFileEntry, RetainedMemberMetadata, +}; +#[cfg(feature = "write")] +pub use write::{ + write_compressed_archive, write_compressed_archive_with_comment, + write_compressed_archive_with_comment_and_progress, write_compressed_volumes, + write_compressed_volumes_with_progress, write_rar29_compressed_archive_with_filter_policy, + write_rar29_compressed_archive_with_filter_policy_and_progress, write_stored_archive, + write_stored_archive_with_comment, write_stored_volumes, write_streaming_archive_to, +}; + +const MARK_HEAD: u8 = 0x72; +const MAIN_HEAD: u8 = 0x73; +const FILE_HEAD: u8 = 0x74; +const COMM_HEAD: u8 = 0x75; +const PROTECT_HEAD: u8 = 0x78; +const NEWSUB_HEAD: u8 = 0x7a; +const ENDARC_HEAD: u8 = 0x7b; + +const LONG_BLOCK: u16 = 0x8000; +const MHD_VOLUME: u16 = 0x0001; +const MHD_COMMENT: u16 = 0x0002; +const MHD_SOLID: u16 = 0x0008; +const MHD_NEWNUMBERING: u16 = 0x0010; +const MHD_PROTECT: u16 = 0x0040; +const MHD_PASSWORD: u16 = 0x0080; +const MHD_FIRSTVOLUME: u16 = 0x0100; +const MHD_ENCRYPTVER: u16 = 0x0200; + +const FHD_SPLIT_BEFORE: u16 = 0x0001; +const FHD_SPLIT_AFTER: u16 = 0x0002; +const FHD_PASSWORD: u16 = 0x0004; +const FHD_COMMENT: u16 = 0x0008; +const FHD_SOLID: u16 = 0x0010; +const FHD_LARGE: u16 = 0x0100; +const FHD_UNICODE: u16 = 0x0200; +const FHD_SALT: u16 = 0x0400; +const FHD_EXTTIME: u16 = 0x1000; +const FHD_DIRECTORY_MASK: u16 = 0x00e0; + +/// Bytes in a comment block before its data starts. +const COMMENT_HEADER_SIZE: usize = 13; +/// Bytes in a main header before a nested comment block starts. +const MAIN_HEADER_SIZE: usize = 13; + +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Archive { + pub sfx_offset: usize, + pub main: MainHeader, + pub blocks: Vec, + source: ArchiveSource, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct MainHeader { + pub head_crc: u16, + pub flags: u16, + pub head_size: u16, + pub reserved1: u16, + pub reserved2: u32, + pub encrypt_version: Option, +} + +impl MainHeader { + pub fn has_archive_comment(&self) -> bool { + self.flags & MHD_COMMENT != 0 + } + + pub fn is_volume(&self) -> bool { + self.flags & MHD_VOLUME != 0 + } + + pub fn is_solid(&self) -> bool { + self.flags & MHD_SOLID != 0 + } + + pub fn uses_new_numbering(&self) -> bool { + self.flags & MHD_NEWNUMBERING != 0 + } + + pub fn has_recovery_record(&self) -> bool { + self.flags & MHD_PROTECT != 0 + } + + pub fn has_encrypted_headers(&self) -> bool { + self.flags & MHD_PASSWORD != 0 + } + + pub fn is_first_volume(&self) -> bool { + self.flags & MHD_FIRSTVOLUME != 0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Block { + File(FileHeader), + Comment(CommentHeader), + Protect(ProtectHeader), + NewSub(NewSubHeader), + End(BlockHeader), + Unknown(BlockHeader), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct BlockHeader { + pub head_crc: u16, + pub head_type: u8, + pub flags: u16, + pub head_size: u16, + pub add_size: Option, + pub offset: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct FileHeader { + pub block: BlockHeader, + pub pack_size: u64, + pub unp_size: u64, + pub host_os: u8, + pub file_crc: u32, + pub file_time: u32, + pub unp_ver: u8, + pub method: u8, + pub name: Vec, + /// Original Unicode wire name, including its legacy fallback when present. + pub unicode_name: Option>, + pub attr: u32, + pub salt: Option<[u8; 8]>, + pub file_comment: Vec, + pub ext_time: Vec, + pub packed_range: Range, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct NewSubHeader { + pub file: FileHeader, + pub kind: NewSubKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct CommentHeader { + pub block: BlockHeader, + pub unp_size: u16, + pub unp_ver: u8, + pub method: u8, + pub comment_crc: u16, + pub packed_range: Range, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct ProtectHeader { + pub block: BlockHeader, + pub version: u8, + pub rec_sectors: u16, + pub total_blocks: u32, + pub mark: [u8; 8], + pub data_range: Range, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum NewSubKind { + ArchiveComment, + RecoveryRecord, + Unknown(Vec), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct ExtractedEntryMeta { + pub name_is_unicode: bool, + pub name: Vec, + pub file_time: u32, + /// Sub-second detail the DOS `file_time` cannot express, from the extended + /// time field. See [`FileHeader::mtime_refinement`]. + pub mtime_refinement: Option, + pub attr: u32, + pub host_os: u8, + pub is_directory: bool, +} + +impl FileHeader { + pub fn name_bytes(&self) -> &[u8] { + &self.name + } + + /// Returns the file name with invalid UTF-8 replaced for display only. + /// + /// Use [`Self::name_bytes`] when exact archive bytes matter. + pub fn name_lossy(&self) -> String { + String::from_utf8_lossy(&self.name).into_owned() + } + + pub fn is_split_before(&self) -> bool { + self.block.flags & FHD_SPLIT_BEFORE != 0 + } + + pub fn is_split_after(&self) -> bool { + self.block.flags & FHD_SPLIT_AFTER != 0 + } + + pub fn is_encrypted(&self) -> bool { + self.block.flags & FHD_PASSWORD != 0 + } + + pub fn is_solid(&self) -> bool { + self.block.flags & FHD_SOLID != 0 + } + + pub fn is_directory(&self) -> bool { + self.block.flags & FHD_DIRECTORY_MASK == FHD_DIRECTORY_MASK + || (self.unp_ver < 20 && self.attr & 0x10 != 0) + } + + pub fn has_ext_time(&self) -> bool { + self.block.flags & FHD_EXTTIME != 0 + } + + pub fn has_file_comment(&self) -> bool { + self.block.flags & FHD_COMMENT != 0 && !self.file_comment.is_empty() + } + + /// Decodes the comment block RAR 1.5 to 2.9 stored inside the file header. + /// + /// The block is a comment header followed by its data, both covered by the + /// file header's size but not by its CRC. + pub fn file_comment(&self) -> Result>> { + if !self.has_file_comment() { + return Ok(None); + } + let block = parse_block_header(&self.file_comment, 0)?; + if block.head_type != COMM_HEAD { + return Err(Error::InvalidHeader( + "RAR 1.5 file comment is not a comment block", + )); + } + let head_size = block.head_size as usize; + let header = parse_comment_header(&self.file_comment, block)?; + // Block admission bounded head_size; comment admission requires >=13. + let packed = &self.file_comment[COMMENT_HEADER_SIZE..head_size]; + header.decode(packed).map(Some) + } + + pub fn is_stored(&self) -> bool { + self.method == 0x30 + } + + pub fn packed_data(&self, archive: &Archive) -> Result> { + archive.read_range(self.packed_range.clone()) + } + + pub fn write_packed_data(&self, archive: &Archive, out: &mut impl Write) -> Result<()> { + archive.copy_range_to(self.packed_range.clone(), out) + } + + fn packed_reader_with_allowance<'a, B: crate::codec::workspace::Budget>( + &self, + archive: &'a Archive, + password: Option<&[u8]>, + allowance: &B, + ) -> Result, B>> { + let reader = archive.range_reader(self.packed_range.clone())?; + if !self.is_encrypted() { + return Ok(PackedReader::Plain(reader)); + } + crate::crypto::require_encryption()?; + let Some(password) = password else { + return Err(Error::NeedPassword); + }; + if (self.unp_ver == 20 || self.unp_ver == 26 || self.unp_ver >= 29) + && !self.packed_range.len().is_multiple_of(16) + { + return Err(Error::InvalidHeader( + "RAR encrypted payload is not block aligned", + )); + } + Ok(PackedReader::Encrypted( + crate::codec::workspace::Boxed::try_new( + || { + DecryptingReader::with_allowance( + reader, + self.unp_ver, + password, + self.salt, + allowance, + ) + }, + allowance, + )?, + )) + } + + pub fn verify_crc32(&self, data: &[u8]) -> Result<()> { + let actual = crc32(data); + if actual == self.file_crc { + Ok(()) + } else { + Err(Error::Crc32Mismatch { + expected: self.file_crc, + actual, + }) + } + } + + /// Decodes the modification time's extended-time detail, if the header + /// carries any. + /// + /// A DOS timestamp counts in two-second steps and holds nothing smaller, + /// so the extended field adds an odd-second flag and up to three bytes of + /// 100-nanosecond ticks. Ignoring it costs up to a second of accuracy on + /// every RAR 1.5-4.x archive that has one. + /// + /// The flags word splits into four nibbles, mtime first at bits 15-12, + /// then ctime, atime and arctime. Within a nibble, `0x8` marks the + /// timestamp present, `0x4` adds one second, and the low two bits count + /// the sub-second bytes. Only mtime is decoded here, and it is the first + /// entry, so nothing after it has to be walked. mtime alone reuses the + /// header's DOS time rather than carrying its own copy. + /// + /// Sub-second bytes arrive high end first: each shifts into the top of a + /// 24-bit accumulator while what is there shifts down. Fewer bytes mean a + /// coarser tick, not a smaller number, so one byte resolves to 6.55 ms and + /// three to the full 100 ns. + pub fn mtime_refinement(&self) -> Option { + const PRESENT: u8 = 0x8; + const ADD_SECOND: u8 = 0x4; + const TICK_NANOSECONDS: u32 = 100; + + let flag_bytes = self.ext_time.get(..2)?; + let flags = u16::from_le_bytes([flag_bytes[0], flag_bytes[1]]); + let rmode = ((flags >> 12) & 0xf) as u8; + if rmode & PRESENT == 0 { + return None; + } + let mut ticks = 0u32; + for &byte in self.ext_time.get(2..2 + usize::from(rmode & 0x3))? { + ticks = (u32::from(byte) << 16) | (ticks >> 8); + } + Some(crate::TimeRefinement { + add_second: rmode & ADD_SECOND != 0, + nanoseconds: ticks * TICK_NANOSECONDS, + }) + } + + pub fn metadata(&self) -> ExtractedEntryMeta { + ExtractedEntryMeta { + name_is_unicode: self.unicode_name.is_some(), + name: self.name.clone(), + file_time: self.file_time, + mtime_refinement: self.mtime_refinement(), + attr: self.attr, + host_os: self.host_os, + is_directory: self.is_directory(), + } + } + + pub fn write_to( + &self, + archive: &Archive, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + if self.is_directory() { + return Ok(()); + } + if self.is_stored() { + return self.write_stored_to(archive, password, out); + } + let mut session = DecoderSession::new_with_password(false, password); + session.write_file_to(archive, self, out) + } + + fn write_stored_with_allowance( + &self, + archive: &Archive, + password: Option<&[u8]>, + out: &mut impl Write, + allowance: &B, + ) -> Result<()> { + if !self.is_encrypted() && self.pack_size != self.unp_size { + return Err(Error::InvalidHeader( + "RAR 1.5 stored file has mismatched packed and unpacked sizes", + )); + } + let mut reader = self + .packed_reader_with_allowance(archive, password, allowance) + .map_err(|error| self.map_encrypted_payload_error(password, error))?; + let expected_len = usize::try_from(self.unp_size) + .map_err(|_| Error::InvalidHeader("RAR 1.5 unpacked size overflows usize"))?; + let mut crc = Crc32::new(); + let mut crc_writer = CrcWriter { + inner: out, + crc: &mut crc, + }; + let copied = std::io::copy( + &mut reader.by_ref().take(expected_len as u64), + &mut crc_writer, + )?; + if copied != expected_len as u64 { + return Err(self.map_encrypted_payload_error( + password, + Error::InvalidHeader("RAR 1.5 stored file ended before unpacked size"), + )); + } + let actual = crc.finish(); + if actual == self.file_crc { + Ok(()) + } else { + Err(self.map_encrypted_payload_error( + password, + Error::Crc32Mismatch { + expected: self.file_crc, + actual, + }, + )) + } + } + + fn write_stored_to( + &self, + archive: &Archive, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + self.write_stored_with_allowance( + archive, + password, + out, + &crate::codec::workspace::Allowance::default(), + ) + } + + fn map_encrypted_payload_error(&self, password: Option<&[u8]>, error: Error) -> Error { + if !self.is_encrypted() || password.is_none() { + return error; + } + if matches!( + error.kind(), + crate::ErrorKind::ResourceLimit | crate::ErrorKind::Cancelled | crate::ErrorKind::Io + ) { + return error; + } + match error { + Error::FeatureDisabled { .. } + | Error::NeedPassword + | Error::UnsupportedSignature + | Error::UnsupportedVersion(_) + | Error::UnsupportedFeature { .. } + | Error::UnsupportedWriterOption { .. } + | Error::CannotSkipSolidMember + | Error::MemberOutputLimitExceeded { .. } + | Error::HeaderCountLimitExceeded { .. } + | Error::HeaderBytesLimitExceeded { .. } + | Error::TotalOutputLimitExceeded { .. } + | Error::Rar50DictionaryLimitExceeded { .. } + | Error::Rar50BufferedDecodeLimitExceeded { .. } + | Error::MemoryLimitExceeded { .. } + | Error::WriterSpoolLimitExceeded { .. } + | Error::WriterSpoolMemoryLimitExceeded { .. } + | Error::WriterPreparedHeaderLimitExceeded { .. } + | Error::WriterPreparationLimitExceeded { .. } + | Error::Rar50ScratchLimitExceeded { .. } + | Error::RewriteStagingLimitExceeded { .. } + | Error::Rar50FilterMemoryLimitExceeded { .. } + | Error::UnsupportedFamilyFeature { .. } + | Error::UnsupportedCompression { .. } + | Error::UnsupportedEncryption { .. } + | Error::TooShort + | Error::Io(_) + | Error::AtArchiveOffset { .. } + | Error::AtEntry { .. } + | Error::InVolume { .. } + | Error::Cancelled + | Error::InvalidArgument(_) + | Error::WriterFailure(_) + | Error::EntryNotFound + | Error::DuplicateEntry + | Error::InputSymlink + | Error::UnsafePath(_) + | Error::SourceChanged(_) => error, + Error::InvalidHeader(_) + | Error::Codec(_) + | Error::Rar3Recovery(_) + | Error::Rar5Recovery(_) + | Error::Rar20Crypto(_) + | Error::Rar30Crypto(_) + | Error::Rar50Crypto(_) + | Error::CrcMismatch { .. } + | Error::Crc32Mismatch { .. } + | Error::HashMismatch { .. } + | Error::WrongPasswordOrCorruptData => Error::WrongPasswordOrCorruptData, + } + } + + fn entry_error(&self, operation: &'static str, error: Error) -> Error { + if matches!( + error, + Error::NeedPassword | Error::WrongPasswordOrCorruptData + ) { + return error; + } + error.at_entry(self.name.clone(), operation) + } + + fn crc_result(&self, actual: u32, password: Option<&[u8]>) -> Result<()> { + if actual == self.file_crc { + Ok(()) + } else { + Err(self.map_encrypted_payload_error( + password, + Error::Crc32Mismatch { + expected: self.file_crc, + actual, + }, + )) + } + } +} + +impl NewSubHeader { + pub fn name_bytes(&self) -> &[u8] { + self.file.name_bytes() + } + + /// Returns the service-block name with invalid UTF-8 replaced for display only. + /// + /// Use [`Self::name_bytes`] when exact archive bytes matter. + pub fn name_lossy(&self) -> String { + self.file.name_lossy() + } +} + +impl CommentHeader { + #[cfg(feature = "write")] + fn supports_rewrite(&self) -> bool { + self.block.head_type == COMM_HEAD + && self.block.flags == 0 + && self.block.add_size.is_none() + && self.block.head_size >= COMMENT_HEADER_SIZE as u16 + && usize::from(self.unp_size) <= usize::from(u16::MAX) - COMMENT_HEADER_SIZE + && (self.method == 0x30 + || ((0x31..=0x35).contains(&self.method) && matches!(self.unp_ver, 15 | 20 | 26))) + } + + /// Unpacks comment data and checks it against the header's 16-bit CRC. + fn decode(&self, packed: &[u8]) -> Result> { + self.decode_with_budget( + packed, + &mut crate::output_limit::OutputBudget::new(crate::ArchiveReadOptions::new()), + ) + } + + fn decode_with_budget( + &self, + packed: &[u8], + budget: &mut crate::output_limit::OutputBudget, + ) -> Result> { + self.decode_with_allowance(packed, budget, &Allowance::default()) + } + + fn decode_with_allowance( + &self, + packed: &[u8], + budget: &mut crate::output_limit::OutputBudget, + allowance: &B, + ) -> Result> { + let target = usize::from(self.unp_size); + budget.check(target as u64, b"CMT")?; + let control = budget.control.clone(); + let mut data = Vec::new(); + budget.run(b"CMT", &mut data, |writer| { + if self.method == 0x30 { + if packed.len() != target { + return Err(Error::InvalidHeader( + "RAR 1.5 stored comment has mismatched packed and unpacked sizes", + )); + } + writer.write_all(packed)?; + } else if self.unp_ver == 15 { + let mut decoder = crate::codec::workspace::Boxed::try_new( + || Reader15State::with_allowance(allowance), + allowance, + )?; + decoder.read_control = control; + decoder.decode_member_to(packed, target, false, writer)?; + } else if self.unp_ver == 20 || self.unp_ver == 26 { + let mut decoder = crate::codec::workspace::Boxed::try_new( + || Ok::<_, crate::codec::Error>(Reader20State::with_allowance(allowance)), + allowance, + )?; + decoder.read_control = control; + decoder.decode_member_from_reader(&mut &packed[..], target, writer)?; + } else { + return Err(Error::UnsupportedCompression { + family: "RAR 1.5 comment", + unpack_version: self.unp_ver, + method: self.method, + }); + } + Ok(()) + })?; + let actual = (crc32(&data) & 0xffff) as u16; + if actual == self.comment_crc { + budget.control.check()?; + Ok(data) + } else { + Err(Error::CrcMismatch { + expected: self.comment_crc, + actual, + }) + } + } +} + +impl Archive { + pub fn parse(input: &[u8]) -> Result { + Self::parse_with_options(input, crate::ArchiveReadOptions::default()) + } + + pub fn parse_owned(input: Vec) -> Result { + Self::parse_owned_with_options(input, crate::ArchiveReadOptions::default()) + } + + pub fn parse_with_options( + input: &[u8], + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let data: Arc<[u8]> = Arc::from(input.to_vec().into_boxed_slice()); + Self::parse_shared(data, options) + } + + pub fn parse_owned_with_options( + input: Vec, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + Self::parse_shared(Arc::from(input.into_boxed_slice()), options) + } + + pub fn parse_with_password(input: &[u8], password: Option<&[u8]>) -> Result { + Self::parse_with_options( + input, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_owned_with_password(input: Vec, password: Option<&[u8]>) -> Result { + Self::parse_owned_with_options( + input, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_path(path: impl AsRef) -> Result { + Self::parse_path_with_options(path, crate::ArchiveReadOptions::default()) + } + + pub fn parse_path_with_password( + path: impl AsRef, + password: Option<&[u8]>, + ) -> Result { + Self::parse_path_with_options( + path, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_path_with_options( + path: impl AsRef, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let path = Arc::new(path.as_ref().to_path_buf()); + let mut file = File::open(path.as_ref())?; + let len = file.metadata()?.len(); + let scan_len = len.min(SFX_SCAN_LIMIT as u64) as usize; + let mut scan = vec![0; scan_len]; + file.read_exact(&mut scan)?; + options.check_cancelled()?; + let sig = find_archive_start(&scan, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + if sig.family != ArchiveFamily::Rar15To40 { + return Err(Error::UnsupportedSignature); + } + Self::parse_seekable(file, len, sig.offset, ArchiveSource::File(path), options) + } + + pub fn parse_path_with_signature_and_password( + path: impl AsRef, + signature: ArchiveSignature, + password: Option<&[u8]>, + ) -> Result { + Self::parse_path_with_signature( + path, + signature, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_path_with_signature( + path: impl AsRef, + signature: ArchiveSignature, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + if signature.family != ArchiveFamily::Rar15To40 { + return Err(Error::UnsupportedSignature); + } + let path = Arc::new(path.as_ref().to_path_buf()); + let file = File::open(path.as_ref())?; + let len = file.metadata()?.len(); + Self::parse_seekable( + file, + len, + signature.offset, + ArchiveSource::File(path), + options, + ) + } + + fn parse_shared(input: Arc<[u8]>, options: crate::ArchiveReadOptions<'_>) -> Result { + options.check_cancelled()?; + let sig = find_archive_start(&input, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + if sig.family != ArchiveFamily::Rar15To40 { + return Err(Error::UnsupportedSignature); + } + + let password = options.password; + let mut budget = crate::parse_budget::ParseBudget::new(options); + let archive = &input[sig.offset..]; + // find_archive_start matched these exact marker bytes, including the + // fixed MARK_HEAD type and seven-byte header length. + let marker_size = RAR15_SIGNATURE.len(); + admit_plain_header(archive, marker_size, &mut budget)?; + let main_block = parse_block_header(archive, marker_size)?; + if main_block.head_type != MAIN_HEAD { + return Err(Error::InvalidHeader("RAR 1.5 main header is missing")); + } + let main = parse_main_header(archive, &main_block)?; + let mut pos = main_block.offset + main_block.head_size as usize; + let mut blocks = Vec::new(); + if let Some(comment) = nested_main_comment( + &archive[main_block.offset..pos], + &main_block, + sig.offset + main_block.offset, + )? { + blocks.push(Block::Comment(comment)); + } + let mut encrypted_header_ciphers = EncryptedHeaderCipherCache::default(); + + while pos < archive.len() { + if archive.len() - pos < 7 { + break; + } + let (block, header, total) = if main.has_encrypted_headers() { + crate::crypto::require_encryption()?; + let password = password.ok_or(Error::NeedPassword)?; + let encrypted = decrypt_encrypted_header_at( + archive, + pos, + password, + &mut encrypted_header_ciphers, + &mut budget, + )?; + (encrypted.block, encrypted.header, encrypted.total_size) + } else { + admit_plain_header(archive, pos, &mut budget)?; + let block = parse_block_header(archive, pos)?; + let total = block_total_size(&block)?; + let header = archive[pos..pos + block.head_size as usize].to_vec(); + (block, header, total) + }; + match block.head_type { + FILE_HEAD => { + let mut file = parse_file_like_header(&header, relative_block(&block), 0)?; + let total = file_block_total_size(&block, total, file.pack_size)?; + let next = checked_block_next(&block, total, archive.len())?; + file.block.offset = block.offset; + file.packed_range = packed_range(sig.offset, next, file.pack_size); + blocks.push(Block::File(file)); + pos = next; + } + NEWSUB_HEAD => { + let mut file = parse_file_like_header(&header, relative_block(&block), 0)?; + let total = file_block_total_size(&block, total, file.pack_size)?; + let next = checked_block_next(&block, total, archive.len())?; + file.block.offset = block.offset; + file.packed_range = packed_range(sig.offset, next, file.pack_size); + let kind = classify_new_sub(&file.name); + blocks.push(Block::NewSub(NewSubHeader { file, kind })); + pos = next; + } + COMM_HEAD => { + let next = checked_block_next(&block, total, archive.len())?; + let mut comment = parse_comment_header(&header, relative_block(&block))?; + comment.block.offset = block.offset; + comment.packed_range = + sig.offset + block.offset + 13..sig.offset + block.offset + total; + blocks.push(Block::Comment(comment)); + pos = next; + } + PROTECT_HEAD => { + let next = checked_block_next(&block, total, archive.len())?; + let protect = parse_protect_header(&header, &block, sig.offset, total)?; + blocks.push(Block::Protect(protect)); + pos = next; + } + ENDARC_HEAD => { + let _next = checked_block_next(&block, total, archive.len())?; + blocks.push(Block::End(block)); + break; + } + _ => { + let next = checked_block_next(&block, total, archive.len())?; + blocks.push(Block::Unknown(block)); + pos = next; + } + } + } + + options.check_cancelled()?; + Ok(Self { + sfx_offset: sig.offset, + main, + blocks, + source: ArchiveSource::Memory(input), + }) + } + + pub(crate) fn parse_seekable( + file: impl Read + std::io::Seek, + file_len: u64, + sfx_offset: usize, + source: ArchiveSource, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let password = options.password; + let mut budget = crate::parse_budget::ParseBudget::new(options); + let mut file = budget.control.reader(file); + let marker = read_block_header_at(&mut file, file_len, sfx_offset, 0, &mut budget)?; + if marker.head_type != MARK_HEAD || marker.head_size != RAR15_SIGNATURE.len() as u16 { + return Err(Error::InvalidHeader("RAR 1.5 marker block is invalid")); + } + + let main_block = read_block_header_at( + &mut file, + file_len, + sfx_offset, + marker.head_size as usize, + &mut budget, + )?; + if main_block.head_type != MAIN_HEAD { + return Err(Error::InvalidHeader("RAR 1.5 main header is missing")); + } + // File lengths are u64, so even a complete header can cross usize on 32-bit hosts. + let mut pos = checked_file_block_next( + sfx_offset, + &main_block, + usize::from(main_block.head_size), + file_len, + )?; + let main_header = read_exact_at( + &mut file, + sfx_offset + main_block.offset, + main_block.head_size as usize, + )?; + let main = parse_main_header(&main_header, &relative_block(&main_block))?; + let mut blocks = Vec::new(); + if let Some(comment) = + nested_main_comment(&main_header, &main_block, sfx_offset + main_block.offset)? + { + blocks.push(Block::Comment(comment)); + } + let mut encrypted_header_ciphers = EncryptedHeaderCipherCache::default(); + + while file_len.saturating_sub((sfx_offset + pos) as u64) >= 7 { + let (block, header, total) = if main.has_encrypted_headers() { + crate::crypto::require_encryption()?; + let password = password.ok_or(Error::NeedPassword)?; + let encrypted = read_encrypted_header_at( + &mut file, + file_len, + sfx_offset, + pos, + password, + &mut encrypted_header_ciphers, + &mut budget, + )?; + (encrypted.block, encrypted.header, encrypted.total_size) + } else { + let block = + read_block_header_at(&mut file, file_len, sfx_offset, pos, &mut budget)?; + let total = block_total_size(&block)?; + let header = read_exact_at(&mut file, sfx_offset + pos, block.head_size as usize)?; + (block, header, total) + }; + match block.head_type { + FILE_HEAD => { + let mut file_header = + parse_file_like_header(&header, relative_block(&block), 0)?; + let total = file_block_total_size(&block, total, file_header.pack_size)?; + let next = checked_file_block_next(sfx_offset, &block, total, file_len)?; + file_header.block.offset = block.offset; + file_header.packed_range = + packed_range(sfx_offset, next, file_header.pack_size); + blocks.push(Block::File(file_header)); + pos = next; + } + NEWSUB_HEAD => { + let mut file_header = + parse_file_like_header(&header, relative_block(&block), 0)?; + let total = file_block_total_size(&block, total, file_header.pack_size)?; + let next = checked_file_block_next(sfx_offset, &block, total, file_len)?; + file_header.block.offset = block.offset; + file_header.packed_range = + packed_range(sfx_offset, next, file_header.pack_size); + let kind = classify_new_sub(&file_header.name); + blocks.push(Block::NewSub(NewSubHeader { + file: file_header, + kind, + })); + pos = next; + } + COMM_HEAD => { + let next = checked_file_block_next(sfx_offset, &block, total, file_len)?; + let mut comment = parse_comment_header(&header, relative_block(&block))?; + comment.block.offset = block.offset; + comment.packed_range = + sfx_offset + block.offset + 13..sfx_offset + block.offset + total; + blocks.push(Block::Comment(comment)); + pos = next; + } + PROTECT_HEAD => { + let next = checked_file_block_next(sfx_offset, &block, total, file_len)?; + let protect = parse_protect_header(&header, &block, sfx_offset, total)?; + blocks.push(Block::Protect(protect)); + pos = next; + } + ENDARC_HEAD => { + let _next = checked_file_block_next(sfx_offset, &block, total, file_len)?; + blocks.push(Block::End(block)); + break; + } + _ => { + let next = checked_file_block_next(sfx_offset, &block, total, file_len)?; + blocks.push(Block::Unknown(block)); + pos = next; + } + } + } + + options.check_cancelled()?; + Ok(Self { + sfx_offset, + main, + blocks, + source, + }) + } + + /// The highest format requirement present in this container, not its creator release. + #[cfg(feature = "write")] + pub(crate) fn preservation_version(&self) -> ArchiveVersion { + if self.main.has_encrypted_headers() + || self + .new_subs() + .any(|sub| sub.kind == NewSubKind::ArchiveComment) + { + ArchiveVersion::Rar30 + } else { + match self.files().map(|file| file.unp_ver).max().unwrap_or(15) { + 29.. => ArchiveVersion::Rar29, + 20..=28 => ArchiveVersion::Rar20, + _ => ArchiveVersion::Rar15, + } + } + } + + /// Native metadata checks for the supported RAR 1.5–4.x rewrite subsets. + #[cfg(feature = "write")] + pub(crate) fn rewrite_preservation_issues(&self) -> Vec { + let mut issues = Vec::new(); + if self.main.is_volume() { + issues.push("legacy volume layout".into()); + } + if self.main.has_recovery_record() { + issues.push("legacy recovery records".into()); + } + let nested_comment_size = match self.blocks.first() { + Some(Block::Comment(comment)) + if self.main.has_archive_comment() + && self.main.head_size > MAIN_HEADER_SIZE as u16 + && comment.block.offset == RAR15_SIGNATURE.len() + MAIN_HEADER_SIZE => + { + usize::from(comment.block.head_size) + } + _ => 0, + }; + if self.main.flags & !(MHD_SOLID | MHD_PASSWORD | MHD_COMMENT) != 0 + || usize::from(self.main.head_size) != MAIN_HEADER_SIZE + nested_comment_size + || self.main.reserved1 != 0 + || self.main.reserved2 != 0 + { + issues.push("legacy main header settings, comments or extra metadata".into()); + } + if self.files().next().is_none() { + issues.push("empty legacy archive (source format cannot be inferred)".into()); + } + let version = self.preservation_version(); + let supports_extended_times = matches!( + version, + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 + ); + for (index, file) in self.files().enumerate() { + let label = format!("legacy member {index} ({:?})", file.name_lossy()); + if !matches!(file.unp_ver, 15 | 20 | 26 | 29) { + issues.push(format!("{label}: legacy source format requires unpacker {} (only 15, 20, 26 and 29 are supported for preservation)", file.unp_ver)); + } + if (file.unp_ver < 29 && file.salt.is_some()) + || (file.unp_ver >= 29 && file.is_encrypted() != file.salt.is_some()) + { + issues.push(format!( + "{label}: unsupported legacy encryption salt settings" + )); + } + if !supports_extended_times && file.has_ext_time() { + issues.push(format!( + "{label}: Pre-RAR2.9 extended timestamps are unsupported" + )); + } + if file.has_ext_time() + && crate::file_times::validate_legacy_extended_times(&file.ext_time).is_err() + { + issues.push(format!( + "{label}: legacy extended timestamps are incomplete or invalid" + )); + } + if version == ArchiveVersion::Rar15 && file.host_os == 3 { + issues.push(format!( + "{label}: RAR1.5 Unix metadata cannot be represented by the compatible writer" + )); + } + if file.block.flags & FHD_COMMENT != 0 { + if self.main.has_encrypted_headers() { + issues.push(format!("{label}: embedded legacy file comments with encrypted headers are incompatible with reference readers")); + } + let supported = parse_block_header(&file.file_comment, 0) + .and_then(|block| parse_comment_header(&file.file_comment, block)) + .is_ok_and(|comment| { + comment.supports_rewrite() + && usize::from(comment.block.head_size) == file.file_comment.len() + }); + if !supported { + issues.push(format!( + "{label}: legacy file comments have unsupported or incomplete metadata" + )); + } + } + if file.is_directory() && (file.pack_size != 0 || file.unp_size != 0 || file.is_solid()) + { + issues.push(format!( + "{label}: unsupported legacy directory payload or compression" + )); + } + if let Some(raw) = &file.unicode_name { + if version == ArchiveVersion::Rar15 + || validate_unicode_name(raw, &file.name).is_err() + { + issues.push(format!( + "{label}: malformed or unsupported legacy Unicode name" + )); + } + } + if !matches!(file.host_os, 0..=3) { + issues.push(format!("{label}: legacy host metadata")); + } + if file.is_solid() && !self.main.is_solid() { + issues.push(format!( + "{label}: solid dependency without archive solid flag" + )); + } + // Only native times, salt and embedded comments are retained. Check + // the declared length too: the reader tolerates unflagged extras. + if file.block.flags + & !(LONG_BLOCK + | FHD_SOLID + | FHD_DIRECTORY_MASK + | FHD_EXTTIME + | FHD_PASSWORD + | FHD_SALT + | FHD_COMMENT + | FHD_UNICODE) + != 0 + || usize::from(file.block.head_size) + != 32 + + file.unicode_name.as_ref().map_or(file.name.len(), Vec::len) + + file.ext_time.len() + + file.file_comment.len() + + if file.salt.is_some() { 8 } else { 0 } + { + issues.push(format!("{label}: legacy file flags or extra metadata")); + } + if !(0x30..=0x35).contains(&file.method) || file.unp_size > u64::from(u32::MAX) { + issues.push(format!("{label}: unsupported legacy method or size")); + } + } + let mut archive_comments = 0; + let mut seen_file = false; + for block in &self.blocks { + match block { + Block::File(_) => seen_file = true, + Block::End(_) => {} + Block::Comment(comment) => { + archive_comments += 1; + if seen_file || !comment.supports_rewrite() { + issues.push( + "legacy archive comment location or metadata is unsupported".into(), + ); + } + } + Block::NewSub(sub) if sub.kind == NewSubKind::ArchiveComment => { + archive_comments += 1; + let file = &sub.file; + if seen_file + || file.is_directory() + || file.is_encrypted() != file.salt.is_some() + || file.block.flags + & !(LONG_BLOCK | FHD_DIRECTORY_MASK | 0x4000 | FHD_PASSWORD | FHD_SALT) + != 0 + || file.block.head_size != 35 + if file.salt.is_some() { 8 } else { 0 } + || file.unp_ver != 29 + || !(0x30..=0x35).contains(&file.method) + || file.attr != 0 + || !matches!(file.host_os, 0..=3) + || file.unp_size > u64::from(u32::MAX) + { + issues.push( + "legacy CMT service metadata or encryption is unsupported".into(), + ); + } + } + _ => issues.push("legacy recovery or other service records".into()), + } + } + if archive_comments > 1 { + issues.push("duplicate legacy archive comments".into()); + } + if self.main.has_archive_comment() && archive_comments == 0 { + issues.push("missing or malformed legacy archive comment".into()); + } + let complete_end = match self.blocks.last() { + Some(Block::End(end)) => { + end.flags & !0x4000 == 0 + && end.head_size == 7 + && end.add_size.is_none() + && end + .offset + .checked_add(if self.main.has_encrypted_headers() { + // Eight-byte salt plus one AES block for this header. + 24 + } else { + 7 + }) + .and_then(|end| end.checked_add(self.sfx_offset)) + .is_some_and(|end| Some(end) == self.source.len().ok()) + } + // Legacy archives may terminate immediately after the last file. + Some(Block::File(file)) => Some(file.packed_range.end) == self.source.len().ok(), + _ => false, + }; + if !complete_end { + issues.push("legacy end header metadata or trailing bytes".into()); + } + issues + } + + fn read_range(&self, range: Range) -> Result> { + self.source.read_range(range) + } + + fn copy_range_to(&self, range: Range, out: &mut impl Write) -> Result<()> { + self.source.copy_range_to(range, out) + } + + fn range_reader(&self, range: Range) -> Result> { + self.source.range_reader(range) + } + + pub fn files(&self) -> impl Iterator { + self.blocks.iter().filter_map(|block| match block { + Block::File(file) => Some(file), + _ => None, + }) + } + + pub fn new_subs(&self) -> impl Iterator { + self.blocks.iter().filter_map(|block| match block { + Block::NewSub(sub) => Some(sub), + _ => None, + }) + } + + pub fn protect_records(&self) -> impl Iterator { + self.blocks.iter().filter_map(|block| match block { + Block::Protect(protect) => Some(protect), + _ => None, + }) + } + + #[cfg(feature = "recovery")] + fn source_bytes(&self) -> Result> { + self.source.bytes() + } + + pub fn repair_protect_head(&self) -> Result> { + Ok(self.repair_protect_head_with_report()?.data) + } + + /// Repaired archive bytes plus whether any protected sector was rebuilt. + pub fn repair_protect_head_with_report(&self) -> Result { + self.repair_protect_head_with_options(crate::ArchiveReadOptions::new()) + } + + /// Repairs protected sectors with cooperative cancellation; other read limits do not apply. + pub fn repair_protect_head_with_options( + &self, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + #[cfg(not(feature = "recovery"))] + { + let _ = (self, options); + Err(Error::FeatureDisabled { + feature: "recovery", + }) + } + #[cfg(feature = "recovery")] + { + options.check_cancelled()?; + let control = crate::read_control::ReadControl::new(options.cancellation); + let (data, data_repaired) = if let Some(recovery) = self + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + { + repair_newsub_recovery_bytes( + &self.source_bytes()?, + self.sfx_offset, + self, + recovery, + &control, + )? + } else { + let protect = self.protect_records().next().ok_or(Error::InvalidHeader( + "RAR 2.x archive does not contain a PROTECT_HEAD recovery record", + ))?; + repair_protect_head_bytes( + &self.source_bytes()?, + self.sfx_offset, + protect, + &control, + )? + }; + control.check()?; + Ok(crate::RecoveryRepairResult { + data, + report: crate::RecoveryRepairReport { + changed: data_repaired, + data_repaired, + ..Default::default() + }, + }) + } + } + + /// Streams extracted entries to caller-provided writers. + pub fn extract_to(&self, options: crate::ArchiveReadOptions<'_>, mut open: F) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + self.extract_impl(options, &mut open, None, None) + .map(|_| ()) + } + + pub(crate) fn extract_controlled( + &self, + options: crate::ArchiveReadOptions<'_>, + selector: &mut crate::extraction_control::Selector<'_>, + on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + ) -> Result { + self.extract_impl( + options, + &mut |_| unreachable!("controlled selection supplies writer"), + Some(selector), + on_error, + ) + } + + fn extract_impl( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + selector: Option<&mut crate::extraction_control::Selector<'_>>, + on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + ) -> Result + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + if let Some(limit) = options.max_reader_workspace_bytes { + return self.extract_with_allowance( + options, + open, + selector, + on_error, + &crate::codec::workspace::Allowance::limited(limit), + ); + } + self.extract_with_allowance( + options, + open, + selector, + on_error, + &crate::codec::workspace::Allowance::default(), + ) + } + + fn extract_with_allowance( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + mut selector: Option<&mut crate::extraction_control::Selector<'_>>, + mut on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + allowance: &B, + ) -> Result + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + let password = options.password; + let mut budget = crate::output_limit::OutputBudget::new(options); + let mut session = DecoderSession::with_allowance(self.main.is_solid(), password, allowance); + session.read_control = budget.control.clone(); + let solid = selector.is_some() + && (self.main.is_solid() || self.files().any(|file| file.is_solid())); + for file in self.files() { + let selected = crate::extraction_control::select( + &mut selector, + options, + || crate::rar15_40_member(file), + solid, + )?; + let mut selected_writer = match selected { + Some(crate::ExtractionDecision::Skip) => continue, + Some(crate::ExtractionDecision::Stop) => { + return Ok(crate::ExtractionOutcome::Stopped) + } + Some(crate::ExtractionDecision::Extract(writer)) => Some(writer), + None => None, + }; + let result = (|| { + options.check_cancelled()?; + if file.is_split_before() || file.is_split_after() { + return Err(Error::InvalidHeader( + "RAR 1.5 split entry requires multivolume extraction", + )); + } + let meta = file.metadata(); + if meta.is_directory { + options.check_cancelled()?; + let _ = match selected_writer.take() { + Some(writer) => writer, + None => open(&meta)?, + }; + options.check_cancelled()?; + return Ok(()); + } + budget.check(file.unp_size, &file.name)?; + options.check_cancelled()?; + let mut writer = match selected_writer.take() { + Some(writer) => writer, + None => open(&meta)?, + }; + options.check_cancelled()?; + budget.run(&file.name, &mut writer, |mut writer| { + if file.is_stored() { + file.write_stored_with_allowance(self, password, &mut writer, allowance) + .map_err(|error| file.entry_error("extracting", error))?; + } else { + session + .write_file_to(self, file, &mut writer) + .map_err(|error| file.entry_error("extracting", error))?; + } + Ok(()) + })?; + Ok(()) + })(); + if crate::extraction_control::finish_member( + &mut on_error, + options, + || crate::rar15_40_member(file), + solid, + result, + )? { + session = DecoderSession::with_allowance(false, password, allowance); + session.read_control = budget.control.clone(); + } + } + options.check_cancelled()?; + Ok(crate::ExtractionOutcome::Complete) + } + + /// Decodes independent members in batches of at most the worker count, + /// emitting each batch in archive order before decoding the next. + /// Completed batches can have been emitted when a later batch fails. + /// This bounds retained result count, not payload bytes or decoder memory. + /// A configured total output ceiling selects sequential extraction. + pub fn extract_to_parallel_buffered( + &self, + options: crate::ArchiveReadOptions<'_>, + mut open: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + // Total accounting belongs to the operation, before any worker dispatch. + if options.max_total_output_bytes.is_some() + || self.main.is_solid() + || self + .files() + .any(|file| file.is_solid() || file.is_split_before() || file.is_split_after()) + { + return self.extract_to(options, open); + } + + if let Some(limit) = options.max_reader_workspace_bytes { + let ledger = Allowance::limited(limit); + let publication = crate::read_control::ReadControl::new(options.cancellation); + return crate::codec::workspace::coordinator::extract_windowed( + self.files(), + &ledger, + |file| { + if file.is_directory() { + 0 + } else if file.is_stored() { + file.unp_size + } else { + file.unp_size + .saturating_add(file.pack_size) + .saturating_add(if file.unp_ver == 15 { 65536 } else { 0 }) + .saturating_add(8192) + } + }, + |file, local| { + options.check_cancelled()?; + decode_parallel_entry_with_allowance(self, file, options, local) + }, + |entry| write_parallel_entry(entry, &mut open, &publication), + ); + } + + let mut files = self.files().peekable(); + let window = crate::parallel::default_window().max(1); + let publication = crate::read_control::ReadControl::new(options.cancellation); + while files.peek().is_some() { + options.check_cancelled()?; + let batch = files.by_ref().take(window).collect(); + // Write the verified members ahead of a failure before reporting it. + let (entries, failure) = crate::parallel::map_prefix(batch, |file| { + decode_parallel_entry(self, file, options) + }); + for entry in entries { + write_parallel_entry(entry, &mut open, &publication)?; + } + if let Some(error) = failure { + return Err(error); + } + } + options.check_cancelled()?; + Ok(()) + } + + pub fn archive_comment(&self) -> Result>> { + self.archive_comment_with_password(None) + } + + pub fn archive_comment_with_password( + &self, + password: Option<&[u8]>, + ) -> Result>> { + self.archive_comment_with_options(crate::ArchiveReadOptions::with_optional_password( + password, + )) + } + + /// Decodes the archive comment under the same policy as [`crate::Archive::comment_with_options`]. + pub fn archive_comment_with_options( + &self, + options: crate::ArchiveReadOptions<'_>, + ) -> Result>> { + if let Some(limit) = options.max_reader_workspace_bytes { + return self.archive_comment_with_allowance(options, &Allowance::limited(limit)); + } + self.archive_comment_with_allowance(options, &Allowance::default()) + } + + fn archive_comment_with_allowance( + &self, + options: crate::ArchiveReadOptions<'_>, + allowance: &B, + ) -> Result>> { + options.check_cancelled()?; + let mut budget = crate::output_limit::OutputBudget::new(options); + if let Some(comment) = self.blocks.iter().find_map(|block| match block { + Block::Comment(comment) => Some(comment), + _ => None, + }) { + budget.check(u64::from(comment.unp_size), b"CMT")?; + let mut packed = Buffer::new(allowance); + let mut reader = budget + .control + .reader(self.range_reader(comment.packed_range.clone())?); + packed.read_to_end(&mut reader)?; + return comment + .decode_with_allowance(&packed, &mut budget, allowance) + .map(Some); + } + + let Some(comment) = self + .new_subs() + .find(|sub| sub.kind == NewSubKind::ArchiveComment) + else { + return Ok(None); + }; + let file = &comment.file; + budget.check(file.unp_size, b"CMT")?; + let mut data = Vec::new(); + let mut session = DecoderSession::with_allowance(false, options.password, allowance); + session.read_control = budget.control.clone(); + budget.run(b"CMT", &mut data, |mut writer| { + if file.is_stored() { + file.write_stored_with_allowance(self, options.password, &mut writer, allowance) + } else { + session.write_file_to(self, file, &mut writer) + } + })?; + Ok(Some(data)) + } +} + +fn decode_parallel_entry( + archive: &Archive, + file: &FileHeader, + options: crate::ArchiveReadOptions<'_>, +) -> Result { + decode_parallel_entry_with_allowance(archive, file, options, &Allowance::default()) +} + +enum ParallelExtractedEntry { + Directory(ExtractedEntryMeta), + File { + meta: ExtractedEntryMeta, + data: Buffer, + }, +} + +fn decode_parallel_entry_with_allowance( + archive: &Archive, + file: &FileHeader, + options: crate::ArchiveReadOptions<'_>, + allowance: &B, +) -> Result> { + options.check_cancelled()?; + let password = options.password; + let mut budget = crate::output_limit::OutputBudget::new(options); + // The only caller dispatches workers after excluding split members. + let meta = file.metadata(); + if meta.is_directory { + return Ok(ParallelExtractedEntry::Directory(meta)); + } + budget.check(file.unp_size, &file.name)?; + let mut data = Buffer::new(allowance); + let control = budget.control.clone(); + budget.run(&file.name, &mut data, |mut data| { + if file.is_stored() { + file.write_stored_with_allowance(archive, password, &mut data, allowance) + .map_err(|error| file.entry_error("extracting", error))?; + } else { + let mut session = DecoderSession::with_allowance(false, password, allowance); + session.read_control = control.clone(); + session + .write_file_to(archive, file, &mut data) + .map_err(|error| file.entry_error("extracting", error))?; + } + options.check_cancelled()?; + Ok(()) + })?; + Ok(ParallelExtractedEntry::File { meta, data }) +} + +fn write_parallel_entry( + entry: ParallelExtractedEntry, + open: &mut F, + control: &crate::read_control::ReadControl, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + control.check()?; + match entry { + ParallelExtractedEntry::Directory(meta) => { + let _ = open(&meta)?; + } + ParallelExtractedEntry::File { meta, data } => { + let mut writer = open(&meta)?; + control.check()?; + control + .write_all(&mut writer, &data) + .map_err(|error| Error::from(error).at_entry(meta.name.clone(), "extracting"))?; + } + } + control.check()?; + Ok(()) +} + +fn classify_new_sub(name: &[u8]) -> NewSubKind { + match name { + b"CMT" => NewSubKind::ArchiveComment, + b"RR" => NewSubKind::RecoveryRecord, + _ => NewSubKind::Unknown(name.to_vec()), + } +} + +fn parse_main_header(input: &[u8], block: &BlockHeader) -> Result { + if block.head_size < 13 { + return Err(Error::InvalidHeader("RAR 1.5 main header is too short")); + } + let start = block.offset; + let head_end = start + block.head_size as usize; + // Both callers have already read and validated this entire block. + let fixed = &input[start + 7..start + 13]; + + let encrypt_version = if block.flags & MHD_ENCRYPTVER != 0 { + Some( + *input + .get(start + 13..head_end) + .and_then(|bytes| bytes.first()) + .ok_or(Error::TooShort)?, + ) + } else { + None + }; + + Ok(MainHeader { + head_crc: block.head_crc, + flags: block.flags, + head_size: block.head_size, + reserved1: u16::from_le_bytes([fixed[0], fixed[1]]), + reserved2: u32::from_le_bytes([fixed[2], fixed[3], fixed[4], fixed[5]]), + encrypt_version, + }) +} + +fn parse_comment_header(input: &[u8], block: BlockHeader) -> Result { + if block.head_size < 13 { + return Err(Error::InvalidHeader("RAR 1.5 comment header is too short")); + } + let start = block.offset; + // Callers admitted the complete block before its fixed fields are decoded. + let fixed = &input[start + 7..start + 13]; + Ok(CommentHeader { + block, + unp_size: u16::from_le_bytes([fixed[0], fixed[1]]), + unp_ver: fixed[2], + method: fixed[3], + comment_crc: u16::from_le_bytes([fixed[4], fixed[5]]), + packed_range: 0..0, + }) +} + +/// Finds where a comment block starting at `start` ends, or `None` when the +/// bytes there are not one that fits inside `head_end`. +/// +/// Old readers skipped what they could not make sense of here rather than +/// rejecting the file, and a comment is never worth losing a member over. +fn comment_block_end(input: &[u8], start: usize, head_end: usize) -> Option { + if start + COMMENT_HEADER_SIZE > head_end { + return None; + } + // Callers supply a complete u16-sized header and an in-header position. + let head_size = u16::from_le_bytes([input[start + 5], input[start + 6]]) as usize; + let end = start + head_size; + (input[start + 2] == COMM_HEAD && head_size >= COMMENT_HEADER_SIZE && end <= head_end) + .then_some(end) +} + +/// Reads the archive comment RAR 1.5 to 2.9 could nest inside the main header. +/// +/// The main header's size covers the comment block, so the block loop steps +/// straight over it and this is the only place it can be seen. `main` locates +/// the main header inside the archive, `main_header` holds its bytes on their +/// own, and `source_offset` is where those bytes sit in the file. +fn nested_main_comment( + main_header: &[u8], + main: &BlockHeader, + source_offset: usize, +) -> Result> { + if main.flags & MHD_COMMENT == 0 { + return Ok(None); + } + let head_size = main.head_size as usize; + let Some(end) = comment_block_end(main_header, MAIN_HEADER_SIZE, head_size) else { + return Ok(None); + }; + let block = parse_block_header(main_header, MAIN_HEADER_SIZE)?; + let mut comment = parse_comment_header(main_header, block)?; + comment.block.offset = main.offset + MAIN_HEADER_SIZE; + comment.packed_range = + source_offset + MAIN_HEADER_SIZE + COMMENT_HEADER_SIZE..source_offset + end; + Ok(Some(comment)) +} + +fn parse_protect_header( + input: &[u8], + block: &BlockHeader, + archive_offset: usize, + total_size: usize, +) -> Result { + if block.head_size != 26 { + return Err(Error::InvalidHeader( + "RAR 2.x recovery header size is invalid", + )); + } + let add_size = block.add_size.ok_or(Error::InvalidHeader( + "RAR 2.x recovery header is missing data size", + ))?; + // The parser admitted the complete 26-byte block before dispatching here. + let fixed = &input[..26]; + let rec_sectors = u16::from_le_bytes([fixed[12], fixed[13]]); + let total_blocks = u32::from_le_bytes([fixed[14], fixed[15], fixed[16], fixed[17]]); + // These wire fields are u32 and u16; their maximum sum fits in u64. + let expected_add_size = u64::from(total_blocks) * 2 + u64::from(rec_sectors) * 512; + if add_size != expected_add_size { + return Err(Error::InvalidHeader( + "RAR 2.x recovery data size does not match header", + )); + } + let mark: [u8; 8] = fixed[18..26].try_into().expect("RAR protect mark size"); + // Both parser callers admitted this entire block's absolute next offset. + let data_start = archive_offset + block.offset + usize::from(block.head_size); + let data_end = archive_offset + block.offset + total_size; + Ok(ProtectHeader { + block: block.clone(), + version: fixed[11], + rec_sectors, + total_blocks, + mark, + data_range: data_start..data_end, + }) +} + +/// Repaired bytes plus whether any sector was actually rebuilt. +#[cfg(feature = "recovery")] +fn repair_protect_head_bytes( + source: &[u8], + sfx_offset: usize, + protect: &ProtectHeader, + control: &crate::read_control::ReadControl, +) -> Result<(Vec, bool)> { + control.check()?; + if protect.rec_sectors == 0 { + return Err(Error::InvalidHeader( + "RAR 2.x recovery record has no parity sectors", + )); + } + if protect.mark != *b"Protect!" { + return Err(Error::InvalidHeader("RAR 2.x recovery mark is invalid")); + } + let protected_start = sfx_offset; + // A u32 count fits the supported 32/64-bit host address sizes. + let declared_blocks = protect.total_blocks as usize; + let protected_len = declared_blocks + .checked_mul(512) + .ok_or(Error::InvalidHeader( + "RAR 2.x protected sector size overflows", + ))?; + let protected_end = protected_start + .checked_add(protected_len) + .ok_or(Error::InvalidHeader( + "RAR 2.x protected sector range overflows", + ))?; + if protected_end > source.len() { + return Err(Error::InvalidHeader( + "RAR 2.x protected sector range is invalid", + )); + } + let recovery_data = source + .get(protect.data_range.clone()) + .ok_or(Error::TooShort)?; + // The successful 512-byte sector calculation also bounds two-byte tags. + let tag_len = declared_blocks * 2; + // A u16 sector count needs at most 33553920 parity bytes on either host. + let parity_len = usize::from(protect.rec_sectors) * 512; + if recovery_data.len() != tag_len + parity_len { + return Err(Error::InvalidHeader( + "RAR 2.x recovery data size is invalid", + )); + } + let tags = &recovery_data[..tag_len]; + let parity = &recovery_data[tag_len..]; + // RAR 2.50 records may declare a final sector that starts before + // PROTECT_HEAD but overlaps the recovery block. Only complete sectors + // before the recovery block are safely repairable. + let repairable_blocks = declared_blocks.min(protect.block.offset / 512); + + let mut damaged = Vec::new(); + for index in 0..repairable_blocks { + control.check()?; + let sector_start = protected_start + index * 512; + let sector = &source[sector_start..sector_start + 512]; + let actual = (!crc32(sector) & 0xffff) as u16; + let expected = u16::from_le_bytes([tags[index * 2], tags[index * 2 + 1]]); + if actual != expected { + damaged.push(index); + } + } + if damaged.is_empty() { + return Ok((source.to_vec(), false)); + } + if damaged.len() > usize::from(protect.rec_sectors) { + return Err(Error::InvalidHeader( + "RAR 2.x recovery damage exceeds parity sector count", + )); + } + + let mut used_slots = vec![false; usize::from(protect.rec_sectors)]; + for &index in &damaged { + control.check()?; + let slot = index % usize::from(protect.rec_sectors); + if used_slots[slot] { + return Err(Error::InvalidHeader( + "RAR 2.x recovery cannot repair multiple sectors in the same parity group", + )); + } + used_slots[slot] = true; + } + + let mut repaired = source.to_vec(); + for &missing_index in &damaged { + control.check()?; + let slot = missing_index % usize::from(protect.rec_sectors); + let mut sector = parity[slot * 512..slot * 512 + 512].to_vec(); + for index in (slot..repairable_blocks).step_by(usize::from(protect.rec_sectors)) { + control.check()?; + if index == missing_index { + continue; + } + let sector_start = protected_start + index * 512; + for (out, byte) in sector + .iter_mut() + .zip(&repaired[sector_start..sector_start + 512]) + { + *out ^= *byte; + } + } + let sector_start = protected_start + missing_index * 512; + repaired[sector_start..sector_start + 512].copy_from_slice(§or); + let actual = (!crc32(§or) & 0xffff) as u16; + let expected = u16::from_le_bytes( + tags[missing_index * 2..missing_index * 2 + 2] + .try_into() + .unwrap(), + ); + if actual != expected { + return Err(Error::CrcMismatch { expected, actual }); + } + } + Ok((repaired, true)) +} + +/// Repaired bytes plus whether any sector was actually rebuilt. +#[cfg(feature = "recovery")] +fn repair_newsub_recovery_bytes( + source: &[u8], + sfx_offset: usize, + archive: &Archive, + recovery: &NewSubHeader, + control: &crate::read_control::ReadControl, +) -> Result<(Vec, bool)> { + control.check()?; + let recovery_data = newsub_recovery_data(archive, recovery, control)?; + let expected_unpacked = usize::try_from(recovery.file.unp_size) + .map_err(|_| Error::InvalidHeader("RAR 3.x recovery unpacked size overflows usize"))?; + if recovery_data.len() != expected_unpacked { + return Err(Error::InvalidHeader( + "RAR 3.x recovery data size does not match unpacked size", + )); + } + let protected_start = sfx_offset; + let protected_end = + sfx_offset + .checked_add(recovery.file.block.offset) + .ok_or(Error::InvalidHeader( + "RAR 3.x recovery protected range overflows", + ))?; + if protected_end > source.len() { + return Err(Error::InvalidHeader( + "RAR 3.x recovery protected range is invalid", + )); + } + let protected_len = protected_end - protected_start; + let protected_sectors = protected_len.div_ceil(512); + if protected_sectors == 0 { + return Err(Error::InvalidHeader( + "RAR 3.x recovery record has no protected sectors", + )); + } + // Sector count is ceil(an already-admitted physical extent / 512). + let tag_len = protected_sectors * 2; + if recovery_data.len() <= tag_len || !(recovery_data.len() - tag_len).is_multiple_of(512) { + return Err(Error::InvalidHeader( + "RAR 3.x recovery data size is invalid", + )); + } + let parity_sectors = (recovery_data.len() - tag_len) / 512; + let tags = &recovery_data[..tag_len]; + let parity = &recovery_data[tag_len..]; + + let mut damaged = Vec::new(); + for index in 0..protected_sectors { + control.check()?; + let sector = protected_sector(source, protected_start, protected_len, index); + let actual = (!crc32(§or) & 0xffff) as u16; + let expected = u16::from_le_bytes([tags[index * 2], tags[index * 2 + 1]]); + if actual != expected { + damaged.push(index); + } + } + if damaged.is_empty() { + return Ok((source.to_vec(), false)); + } + if damaged.len() > parity_sectors { + return Err(Error::InvalidHeader( + "RAR 3.x recovery damage exceeds parity sector count", + )); + } + + let mut used_slots = vec![false; parity_sectors]; + for &index in &damaged { + control.check()?; + let slot = index % parity_sectors; + if used_slots[slot] { + return Err(Error::InvalidHeader( + "RAR 3.x recovery cannot repair multiple sectors in the same parity group", + )); + } + used_slots[slot] = true; + } + + let mut repaired = source.to_vec(); + for &missing_index in &damaged { + control.check()?; + let slot = missing_index % parity_sectors; + let mut sector = parity[slot * 512..slot * 512 + 512].to_vec(); + for index in (slot..protected_sectors).step_by(parity_sectors) { + control.check()?; + if index == missing_index { + continue; + } + let other = protected_sector(&repaired, protected_start, protected_len, index); + for (out, byte) in sector.iter_mut().zip(other) { + control.check()?; + *out ^= byte; + } + } + let actual = (!crc32(§or) & 0xffff) as u16; + let expected = u16::from_le_bytes( + tags[missing_index * 2..missing_index * 2 + 2] + .try_into() + .unwrap(), + ); + if actual != expected { + return Err(Error::CrcMismatch { expected, actual }); + } + let sector_start = protected_start + missing_index * 512; + let write_len = 512.min(protected_end - sector_start); + repaired[sector_start..sector_start + write_len].copy_from_slice(§or[..write_len]); + } + + Ok((repaired, true)) +} + +#[cfg(feature = "recovery")] +fn newsub_recovery_data( + archive: &Archive, + recovery: &NewSubHeader, + control: &crate::read_control::ReadControl, +) -> Result> { + if recovery.file.is_encrypted() { + return Err(Error::UnsupportedFeature { + version: ArchiveVersion::Rar30, + feature: "encrypted RAR 3.x NEWSUB recovery record", + }); + } + if recovery.file.method == 0x30 { + if recovery.file.pack_size != recovery.file.unp_size { + return Err(Error::InvalidHeader( + "RAR 3.x recovery record packed size does not match unpacked size", + )); + } + return recovery.file.packed_data(archive); + } + let mut session = DecoderSession::new(false); + session.read_control = control.clone(); + control.finish(session.decode_file_data(archive, &recovery.file)) +} + +#[cfg(feature = "recovery")] +fn protected_sector( + source: &[u8], + protected_start: usize, + protected_len: usize, + index: usize, +) -> [u8; 512] { + // Both callers bound index by ceil(protected_len / 512), after validating + // the complete protected range against source. + let sector_offset = index * 512; + debug_assert!(sector_offset < protected_len); + let sector_start = protected_start + sector_offset; + let available = 512.min(protected_len - sector_offset); + let mut sector = [0u8; 512]; + sector[..available].copy_from_slice(&source[sector_start..sector_start + available]); + sector +} + +pub fn repair_rev3_volumes_to( + data_volumes: &[Option<&[u8]>], + recovery_count: usize, + recovery_volumes: &[(usize, &[u8])], + mut write: F, +) -> Result<()> +where + F: FnMut(usize, &[u8]) -> Result<()>, +{ + #[cfg(not(feature = "recovery"))] + { + let _ = (data_volumes, recovery_volumes, recovery_count, &mut write); + Err(Error::FeatureDisabled { + feature: "recovery", + }) + } + #[cfg(feature = "recovery")] + { + for (index, bytes) in crate::recovery::rar3::reconstruct_data_volumes( + data_volumes, + recovery_count, + recovery_volumes, + ) + .map_err(Error::from)? + .into_iter() + .enumerate() + { + let bytes = truncate_repaired_rev3_volume(bytes); + write(index, &bytes)?; + } + Ok(()) + } +} + +#[cfg(feature = "recovery")] +fn truncate_repaired_rev3_volume(mut bytes: Vec) -> Vec { + let Ok(archive) = Archive::parse(&bytes) else { + return bytes; + }; + let Some(end) = archive.blocks.iter().find_map(|block| match block { + Block::End(end) => Some(end), + _ => None, + }) else { + return bytes; + }; + // Successful parsing admitted this end block and its full physical extent. + let end_size = (u64::from(end.head_size) + end.add_size.unwrap_or(0)) as usize; + let end_pos = archive.sfx_offset + end.offset + end_size; + if end_pos < bytes.len() && bytes[end_pos..].iter().all(|&byte| byte == 0) { + bytes.truncate(end_pos); + } + bytes +} + +struct EncryptedHeader { + block: BlockHeader, + header: Vec, + total_size: usize, +} + +#[derive(Default)] +struct EncryptedHeaderCipherCache { + #[cfg(feature = "encryption")] + salt: Option<[u8; 8]>, + #[cfg(feature = "encryption")] + cipher: Option, +} + +#[cfg(feature = "encryption")] +impl EncryptedHeaderCipherCache { + fn cipher(&mut self, password: &[u8], salt: [u8; 8]) -> Result { + if self.salt != Some(salt) { + self.salt = Some(salt); + self.cipher = + Some(Rar30Cipher::new(password, Some(salt)).map_err(map_rar30_crypto_error)?); + } + Ok(self + .cipher + .as_ref() + .expect("RAR 3 encrypted header cipher cache initialized") + .clone()) + } +} + +#[cfg(feature = "encryption")] +fn map_rar30_crypto_error(error: Rar30Error) -> Error { + Error::from(error) +} + +fn decrypt_encrypted_header_at( + archive: &[u8], + offset: usize, + password: &[u8], + cipher_cache: &mut EncryptedHeaderCipherCache, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + #[cfg(not(feature = "encryption"))] + { + let _ = (archive, offset, password, cipher_cache, budget); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + let salt = read_header_salt(archive, offset)?; + let first_ciphertext = archive + .get(offset + 8..offset + 24) + .ok_or(Error::TooShort)?; + budget.check_count(offset)?; + let mut cipher = cipher_cache.cipher(password, salt)?; + let mut first_block = [0u8; 16]; + first_block.copy_from_slice(first_ciphertext); + cipher.decrypt_block(&mut first_block); + let head_size = u16::from_le_bytes([first_block[5], first_block[6]]) as usize; + if head_size < 7 { + return Err(Error::InvalidHeader("RAR 1.5 block header is too short")); + } + // A u16 wire length rounds to at most 65536, including on 32-bit hosts. + let encrypted_header_size = head_size.next_multiple_of(16); + // offset is within a physical slice (at most isize::MAX); this u16 wire + // length adds at most 65544, which fits usize on both supported widths. + let encrypted_end = offset + 8 + encrypted_header_size; + if encrypted_end > archive.len() { + return Err(Error::TooShort); + } + budget.admit(head_size, offset)?; + // The first 24 bytes and the rounded header end were admitted above. + let encrypted_rest = &archive[offset + 24..encrypted_end]; + let mut header = Vec::with_capacity(encrypted_header_size); + header.extend_from_slice(&first_block); + header.extend_from_slice(encrypted_rest); + // The rounded header minus its first block is a whole number of AES blocks. + for block in header[16..].as_chunks_mut::<16>().0.iter_mut() { + cipher.decrypt_block(block); + } + header.truncate(head_size); + + let mut block = parse_block_header(&header, 0)?; + block.offset = offset; + // Parsed add_size is a widened wire u32 and fits both supported widths. + let payload_size = block.add_size.unwrap_or(0) as usize; + let total_size = 8usize + .checked_add(encrypted_header_size) + .and_then(|size| size.checked_add(payload_size)) + .ok_or(Error::InvalidHeader("RAR 1.5 block size overflows usize"))?; + Ok(EncryptedHeader { + block, + header, + total_size, + }) + } +} + +fn read_encrypted_header_at( + file: &mut (impl Read + std::io::Seek), + file_len: u64, + archive_offset: usize, + offset: usize, + password: &[u8], + cipher_cache: &mut EncryptedHeaderCipherCache, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + #[cfg(not(feature = "encryption"))] + { + let _ = ( + file, + file_len, + archive_offset, + offset, + password, + cipher_cache, + budget, + ); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + let absolute = archive_offset + .checked_add(offset) + .ok_or(Error::InvalidHeader("RAR 1.5 block offset overflows usize"))?; + let remaining = file_len + .checked_sub(absolute as u64) + .ok_or(Error::TooShort)?; + if remaining < 24 { + return Err(Error::TooShort); + } + let first = read_exact_at(file, absolute, 24)?; + let salt = read_header_salt(&first, 0)?; + budget.check_count(offset)?; + let mut cipher = cipher_cache.cipher(password, salt)?; + let mut first_block = [0u8; 16]; + first_block.copy_from_slice(&first[8..24]); + cipher.decrypt_block(&mut first_block); + let head_size = u16::from_le_bytes([first_block[5], first_block[6]]) as usize; + if head_size < 7 { + return Err(Error::InvalidHeader("RAR 1.5 block header is too short")); + } + // A u16 wire length rounds to at most 65536, including on 32-bit hosts. + let encrypted_header_size = head_size.next_multiple_of(16); + let encrypted_start = absolute + .checked_add(8) + .ok_or(Error::InvalidHeader("RAR 1.5 block offset overflows usize"))?; + if encrypted_header_size as u64 > remaining - 8 { + return Err(Error::TooShort); + } + budget.admit(head_size, offset)?; + let encrypted_rest_start = encrypted_start + .checked_add(16) + .ok_or(Error::InvalidHeader("RAR 1.5 block offset overflows usize"))?; + let encrypted_rest = read_exact_at(file, encrypted_rest_start, encrypted_header_size - 16)?; + let mut header = Vec::with_capacity(encrypted_header_size); + header.extend_from_slice(&first_block); + header.extend_from_slice(&encrypted_rest); + // The rounded header minus its first block is a whole number of AES blocks. + for block in header[16..].as_chunks_mut::<16>().0.iter_mut() { + cipher.decrypt_block(block); + } + header.truncate(head_size); + + let mut block = parse_block_header(&header, 0)?; + block.offset = offset; + // Parsed add_size is a widened wire u32 and fits both supported widths. + let payload_size = block.add_size.unwrap_or(0) as usize; + let total_size = 8usize + .checked_add(encrypted_header_size) + .and_then(|size| size.checked_add(payload_size)) + .ok_or(Error::InvalidHeader("RAR 1.5 block size overflows usize"))?; + Ok(EncryptedHeader { + block, + header, + total_size, + }) + } +} + +#[cfg(feature = "encryption")] +fn read_header_salt(input: &[u8], offset: usize) -> Result<[u8; 8]> { + input + .get(offset..offset + 8) + .ok_or(Error::TooShort) + .map(|salt| salt.try_into().expect("RAR 3 salt size")) +} + +fn parse_file_like_header( + input: &[u8], + block: BlockHeader, + archive_offset: usize, +) -> Result { + if block.head_size < 32 { + return Err(Error::InvalidHeader("RAR 1.5 file header is too short")); + } + if block.flags & LONG_BLOCK == 0 { + return Err(Error::InvalidHeader( + "RAR 1.5 file header is missing packed data size", + )); + } + + let start = block.offset; + let head_end = start + block.head_size as usize; + + // Complete block admission plus the minimum-size guard bounds base fields. + let fixed = &input[start..start + 32]; + let pack_low = u32::from_le_bytes([fixed[7], fixed[8], fixed[9], fixed[10]]) as u64; + let unp_low = u32::from_le_bytes([fixed[11], fixed[12], fixed[13], fixed[14]]) as u64; + let host_os = fixed[15]; + let file_crc = u32::from_le_bytes([fixed[16], fixed[17], fixed[18], fixed[19]]); + let file_time = u32::from_le_bytes([fixed[20], fixed[21], fixed[22], fixed[23]]); + let unp_ver = fixed[24]; + let method = fixed[25]; + let name_size = u16::from_le_bytes([fixed[26], fixed[27]]) as usize; + let attr = u32::from_le_bytes([fixed[28], fixed[29], fixed[30], fixed[31]]); + let mut pos = start + 32; + + let (pack_size, unp_size) = if block.flags & FHD_LARGE != 0 { + let high_pack = read_u32(input, pos)? as u64; + let high_unp = read_u32(input, pos + 4)? as u64; + pos += 8; + ((high_pack << 32) | pack_low, (high_unp << 32) | unp_low) + } else { + (pack_low, unp_low) + }; + + // Relative headers start at zero: base+LARGE prefix <=40, name is a u16. + let name_end = pos + name_size; + if name_end > head_end { + return Err(Error::InvalidHeader( + "RAR 1.5 file name extends beyond header", + )); + } + let unicode_name = (block.flags & FHD_UNICODE != 0).then(|| input[pos..name_end].to_vec()); + let name = decode_file_name(&input[pos..name_end], block.flags); + pos = name_end; + + let salt = if block.flags & FHD_SALT != 0 { + // name_end was bounded by the complete u16-sized header above. + let salt_end = pos + 8; + if salt_end > head_end { + return Err(Error::InvalidHeader( + "RAR 1.5 salt extends beyond file header", + )); + } + let salt_bytes = &input[pos..salt_end]; + pos = salt_end; + Some( + salt_bytes + .try_into() + .expect("RAR 1.5 salt slice has fixed length"), + ) + } else { + None + }; + + let file_comment = if block.flags & FHD_COMMENT != 0 { + match comment_block_end(input, pos, head_end) { + Some(comment_end) => { + let comment = input[pos..comment_end].to_vec(); + pos = comment_end; + comment + } + None => Vec::new(), + } + } else { + Vec::new() + }; + + let ext_time = if block.flags & FHD_EXTTIME != 0 { + input[pos..head_end].to_vec() + } else { + Vec::new() + }; + let data_start = head_end; + let data_len = usize::try_from(pack_size) + .map_err(|_| Error::InvalidHeader("RAR 1.5 packed file size overflows usize"))?; + let data_end = data_start + .checked_add(data_len) + .ok_or(Error::InvalidHeader( + "RAR 1.5 packed file size overflows usize", + ))?; + Ok(FileHeader { + block, + pack_size, + unp_size, + host_os, + file_crc, + file_time, + unp_ver, + method, + name, + unicode_name, + attr, + salt, + file_comment, + ext_time, + packed_range: archive_offset + data_start..archive_offset + data_end, + }) +} + +#[cfg(any(test, feature = "write"))] +pub(crate) fn validate_unicode_name(raw: &[u8], decoded: &[u8]) -> Result<()> { + if decode_file_name(raw, FHD_UNICODE) != decoded || std::str::from_utf8(decoded).is_err() { + return Err(Error::InvalidArgument("invalid legacy Unicode filename")); + } + crate::filename::validate_relative(decoded)?; + if let Some(zero) = raw.iter().position(|byte| *byte == 0) { + crate::filename::validate_relative(&raw[..zero])?; + } + Ok(()) +} + +pub(crate) fn decode_file_name(raw: &[u8], flags: u16) -> Vec { + if flags & FHD_UNICODE == 0 { + return raw.to_vec(); + } + + let Some(zero_pos) = raw.iter().position(|byte| *byte == 0) else { + return raw.to_vec(); + }; + if zero_pos + 1 >= raw.len() { + return raw[..zero_pos].to_vec(); + } + + let fallback = &raw[..zero_pos]; + let high_byte = raw[zero_pos + 1]; + let encoded = &raw[zero_pos + 2..]; + let mut pos = 0usize; + let mut flag_byte = 0u8; + let mut flag_bits = 0u8; + let mut dst_pos = 0usize; + let mut units = Vec::new(); + + while pos < encoded.len() { + if flag_bits == 0 { + flag_byte = encoded[pos]; + pos += 1; + flag_bits = 8; + } + let mode = flag_byte >> 6; + flag_byte <<= 2; + flag_bits -= 2; + + match mode { + 0 => { + let Some(&low) = encoded.get(pos) else { + return raw.to_vec(); + }; + pos += 1; + units.push(u16::from(low)); + dst_pos += 1; + } + 1 => { + let Some(&low) = encoded.get(pos) else { + return raw.to_vec(); + }; + pos += 1; + units.push((u16::from(high_byte) << 8) | u16::from(low)); + dst_pos += 1; + } + 2 => { + let Some((&low, &high)) = encoded.get(pos).zip(encoded.get(pos + 1)) else { + return raw.to_vec(); + }; + pos += 2; + units.push((u16::from(high) << 8) | u16::from(low)); + dst_pos += 1; + } + // The mode is the top two bits of a byte, so this is mode 3. + _ => { + let Some(&length_byte) = encoded.get(pos) else { + return raw.to_vec(); + }; + pos += 1; + let (count, correction, high) = if length_byte & 0x80 != 0 { + let Some(&correction) = encoded.get(pos) else { + return raw.to_vec(); + }; + pos += 1; + ((length_byte & 0x7f) as usize + 2, correction, high_byte) + } else { + (length_byte as usize + 2, 0, 0) + }; + for _ in 0..count { + let Some(&low) = fallback.get(dst_pos) else { + return raw.to_vec(); + }; + let low = low.wrapping_add(correction); + units.push((u16::from(high) << 8) | u16::from(low)); + dst_pos += 1; + } + } + } + } + + // An invalid Unicode field must not collapse distinct archive identities + // to the same replacement-character name. Keep the undecodable field, as + // for malformed encoding commands above; filesystem extraction can refuse it. + String::from_utf16(&units) + .map(String::into_bytes) + .unwrap_or_else(|_| raw.to_vec()) +} + +fn admit_plain_header( + input: &[u8], + offset: usize, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result<()> { + budget.control.check()?; + if !budget.is_limited() { + return Ok(()); + } + // Both parser call sites bound offset by the physical archive extent. + let prefix = &input[offset..]; + if prefix.len() < 7 { + return Err(Error::TooShort); + } + let size = u16::from_le_bytes([prefix[5], prefix[6]]) as usize; + if size < 7 { + return Err(Error::InvalidHeader("RAR 1.5 block header is too short")); + } + if size > prefix.len() { + return Err(Error::TooShort); + } + budget.admit(size, offset) +} + +fn read_block_header_at( + file: &mut (impl Read + std::io::Seek), + file_len: u64, + archive_offset: usize, + offset: usize, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + let absolute = archive_offset + .checked_add(offset) + .ok_or(Error::InvalidHeader("RAR 1.5 block offset overflows usize"))?; + let remaining = file_len + .checked_sub(absolute as u64) + .ok_or(Error::TooShort)?; + if remaining < 7 { + return Err(Error::TooShort); + } + let base = read_exact_at(file, absolute, 7)?; + // A successful exact read has admitted the entire fixed prefix. + let head_size = u16::from_le_bytes([base[5], base[6]]) as usize; + if head_size < 7 { + return Err(Error::InvalidHeader("RAR 1.5 block header is too short")); + } + if head_size as u64 > remaining { + return Err(Error::TooShort); + } + if offset != 0 { + budget.admit(head_size, offset)?; + } + let header = if head_size == 7 { + base + } else { + read_exact_at(file, absolute, head_size)? + }; + let mut block = parse_block_header(&header, 0)?; + block.offset = offset; + Ok(block) +} + +fn relative_block(block: &BlockHeader) -> BlockHeader { + let mut relative = block.clone(); + relative.offset = 0; + relative +} + +struct CrcWriter<'a, W: Write + ?Sized> { + inner: &'a mut W, + crc: &'a mut Crc32, +} + +impl Write for CrcWriter<'_, W> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.crc.update(&buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +fn parse_block_header(input: &[u8], offset: usize) -> Result { + if input.len() < offset + 7 { + return Err(Error::TooShort); + } + let prefix = &input[offset..offset + 7]; + let head_crc = u16::from_le_bytes([prefix[0], prefix[1]]); + let head_type = prefix[2]; + let flags = u16::from_le_bytes([prefix[3], prefix[4]]); + let head_size = u16::from_le_bytes([prefix[5], prefix[6]]); + if head_size < 7 { + return Err(Error::InvalidHeader("RAR 1.5 block header is too short")); + } + let add_size = if flags & LONG_BLOCK != 0 { + Some(read_u32(input, offset + 7)? as u64) + } else { + None + }; + if offset + head_size as usize > input.len() { + return Err(Error::TooShort); + } + if head_type != MARK_HEAD && should_validate_header_crc(head_type) { + let header_end = header_crc_end(input, offset, head_type, flags, head_size)?; + let actual = (crc32(&input[offset + 2..header_end]) & 0xffff) as u16; + if actual != head_crc { + return Err(Error::CrcMismatch { + expected: head_crc, + actual, + }); + } + } + validate_legacy_auth_block_size(head_type, head_size)?; + + Ok(BlockHeader { + head_crc, + head_type, + flags, + head_size, + add_size, + offset, + }) +} + +fn header_crc_end( + input: &[u8], + offset: usize, + head_type: u8, + flags: u16, + head_size: u16, +) -> Result { + let full_end = offset + head_size as usize; + let fixed_end = match head_type { + MAIN_HEAD if flags & MHD_COMMENT != 0 => Some(offset + 13), + COMM_HEAD => Some(offset + 13), + FILE_HEAD if flags & FHD_COMMENT != 0 => Some(file_header_comment_crc_end(input, offset)?), + _ => None, + }; + Ok(fixed_end.unwrap_or(full_end).min(full_end)) +} + +fn file_header_comment_crc_end(input: &[u8], offset: usize) -> Result { + if input.len() < offset + 32 { + return Err(Error::TooShort); + } + let fixed = &input[offset..offset + 32]; + let flags = u16::from_le_bytes([fixed[3], fixed[4]]); + let name_size = u16::from_le_bytes([fixed[26], fixed[27]]) as usize; + let mut end = offset + 32; + // Offset is within a physical slice; the remaining fields add at most + // a u16 name length plus two eight-byte fields, so usize cannot overflow. + if flags & FHD_LARGE != 0 { + end += 8; + } + end += name_size; + if flags & FHD_SALT != 0 { + end += 8; + } + Ok(end) +} + +fn should_validate_header_crc(head_type: u8) -> bool { + // Historical AV/SIGN blocks are documented with inconsistent CRC fields in + // real archives, so readers must not reject them solely on HEAD_CRC. + !matches!(head_type, 0x76 | 0x79) +} + +fn validate_legacy_auth_block_size(head_type: u8, head_size: u16) -> Result<()> { + let minimum = match head_type { + 0x76 => 21, + 0x79 => 182, + _ => return Ok(()), + }; + if head_size < minimum { + return Err(Error::InvalidHeader( + "RAR legacy authenticity block is too short", + )); + } + Ok(()) +} + +fn block_total_size(block: &BlockHeader) -> Result { + let total = block.head_size as u64 + block.add_size.unwrap_or(0); + usize::try_from(total).map_err(|_| Error::InvalidHeader("RAR 1.5 block size overflows usize")) +} + +fn file_block_total_size( + block: &BlockHeader, + default_total: usize, + pack_size: u64, +) -> Result { + // Parsed add_size is a widened u32, and default_total already includes it. + let low_payload_size = block.add_size.unwrap_or(0) as usize; + let header_prefix = default_total - low_payload_size; + // Every caller obtained pack_size from parse_file_like_header, which + // admitted its conversion and relative payload end before returning. + let pack_size = pack_size as usize; + header_prefix + .checked_add(pack_size) + .ok_or(Error::InvalidHeader("RAR 1.5 block size overflows usize")) +} + +fn checked_block_next(block: &BlockHeader, total: usize, archive_len: usize) -> Result { + let next = block + .offset + .checked_add(total) + .ok_or(Error::InvalidHeader("RAR 1.5 block size overflows usize"))?; + if next > archive_len { + return Err(Error::TooShort); + } + Ok(next) +} + +fn checked_file_block_next( + sfx_offset: usize, + block: &BlockHeader, + total: usize, + file_len: u64, +) -> Result { + let next = block + .offset + .checked_add(total) + .ok_or(Error::InvalidHeader("RAR 1.5 block size overflows usize"))?; + let absolute_next = sfx_offset + .checked_add(next) + .ok_or(Error::InvalidHeader("RAR 1.5 block size overflows usize"))?; + if absolute_next as u64 > file_len { + return Err(Error::TooShort); + } + Ok(next) +} + +/// Called only after full packed-size conversion and the next block extent have +/// been admitted. The checked total includes the payload, so subtraction fits. +fn packed_range(archive_offset: usize, next: usize, pack_size: u64) -> Range { + let block_end = archive_offset + next; + block_end - pack_size as usize..block_end +} + +#[cfg(all(test, feature = "write"))] +mod tests { + use crate::FeatureSet; + #[test] + fn header_budget_refuses_full_reads_after_plain_and_encrypted_prefixes() { + use crate::parse_budget::{ParseBudget, PrefixReader}; + let options = crate::ArchiveReadOptions::new().with_max_header_bytes(63); + let mut plain = vec![0u8; 16]; + plain[5] = 64; + let mut prefix = vec![0; 7]; // skipped marker bytes + prefix.extend_from_slice(&plain[..7]); + let mut reader = PrefixReader::new(prefix); + let e = super::read_block_header_at(&mut reader, 128, 0, 7, &mut ParseBudget::new(options)) + .map(|_| ()) + .expect_err("header budget must refuse"); + assert!(matches!( + e.root_cause(), + crate::Error::HeaderBytesLimitExceeded { required: 64, .. } + )); + assert_eq!(reader.reads, [7]); + + #[cfg(feature = "encryption")] + { + let mut cipher = + crate::crypto::rar30::Rar30Cipher::new(b"secret", Some([0; 8])).unwrap(); + cipher.encrypt_in_place(&mut plain).unwrap(); + let mut prefix = vec![0; 8]; + prefix.extend_from_slice(&plain); + let mut reader = PrefixReader::new(prefix); + let e = super::read_encrypted_header_at( + &mut reader, + 128, + 0, + 0, + b"secret", + &mut super::EncryptedHeaderCipherCache::default(), + &mut ParseBudget::new(options), + ) + .map(|_| ()) + .expect_err("header budget must refuse"); + assert!(matches!( + e.root_cause(), + crate::Error::HeaderBytesLimitExceeded { required: 64, .. } + )); + assert_eq!(reader.reads, [24]); + } + } + use super::*; + use crate::io_util::read_u16; + + #[test] + fn declared_payload_extents_are_checked_for_each_block_kind() { + for head_type in [ + FILE_HEAD, + NEWSUB_HEAD, + COMM_HEAD, + PROTECT_HEAD, + ENDARC_HEAD, + 0x7e, + ] { + let size: u16 = match head_type { + FILE_HEAD | NEWSUB_HEAD => 33, + PROTECT_HEAD => 26, + COMM_HEAD => 13, + _ => 11, + }; + let mut header = vec![0; usize::from(size)]; + header[2] = head_type; + header[3..5].copy_from_slice(&LONG_BLOCK.to_le_bytes()); + header[5..7].copy_from_slice(&size.to_le_bytes()); + header[7..11].copy_from_slice(&4u32.to_le_bytes()); + if matches!(head_type, FILE_HEAD | NEWSUB_HEAD) { + header[24] = 29; + header[25] = 0x30; + header[26..28].copy_from_slice(&1u16.to_le_bytes()); + header[32] = b'x'; + } else if head_type == PROTECT_HEAD { + header[14..18].copy_from_slice(&2u32.to_le_bytes()); + header[18..26].copy_from_slice(b"Protect!"); + } + test_write_header_crc(&mut header, 0); + for available in 0..=4 { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, 0); + bytes.extend_from_slice(&header); + bytes.resize(bytes.len() + available, 0); + for result in [ + Archive::parse(&bytes), + Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ), + ] { + if available < 4 { + assert!( + matches!(result, Err(Error::TooShort)), + "type {head_type:x}, extent {available}: {result:?}" + ); + } else { + assert_eq!(result.unwrap().blocks.len(), 1); + } + } + } + } + } + + #[test] + fn large_file_totals_refuse_relative_and_encrypted_prefix_overflow() { + for head_type in [FILE_HEAD, NEWSUB_HEAD] { + for encrypted in crate::test_features::encryption_cases() { + let pack_size = u64::MAX - 41; + let mut header = vec![0; 41]; + header[2] = head_type; + header[3..5].copy_from_slice(&(LONG_BLOCK | FHD_LARGE).to_le_bytes()); + header[5..7].copy_from_slice(&41u16.to_le_bytes()); + header[7..11].copy_from_slice(&(pack_size as u32).to_le_bytes()); + header[24] = 29; + header[25] = 0x30; + header[26..28].copy_from_slice(&1u16.to_le_bytes()); + header[32..36].copy_from_slice(&((pack_size >> 32) as u32).to_le_bytes()); + header[40] = b'x'; + test_write_header_crc(&mut header, 0); + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, if encrypted { MHD_PASSWORD } else { 0 }); + if encrypted { + #[cfg(feature = "encryption")] + { + header.resize(48, 0); + crate::crypto::rar30::Rar30Cipher::new(b"pw", Some([0; 8])) + .unwrap() + .encrypt_in_place(&mut header) + .unwrap(); + bytes.extend_from_slice(&[0; 8]); + } + } + bytes.extend_from_slice(&header); + let options = crate::ArchiveReadOptions::with_password(b"pw"); + for result in [ + Archive::parse_with_options(&bytes, options), + Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + options, + ), + ] { + // On 64-bit, plain relative FILE size fits exactly, but the + // containing archive offset (or encrypted physical prefix) does not. + let expected = if usize::BITS == 32 && !encrypted { + "RAR 1.5 packed file size overflows usize" + } else { + "RAR 1.5 block size overflows usize" + }; + assert!( + matches!(result, Err(Error::InvalidHeader(message)) if message == expected), + "{result:?}" + ); + } + } + } + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_source_failures_are_preserved_after_successful_parsing() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct FailingSource { + bytes: std::io::Cursor>, + fail: Arc, + } + impl Read for FailingSource { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + if self.fail.load(Ordering::Relaxed) { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "recovery source denied", + )) + } else { + self.bytes.read(bytes) + } + } + } + impl std::io::Seek for FailingSource { + fn seek(&mut self, from: std::io::SeekFrom) -> std::io::Result { + self.bytes.seek(from) + } + } + for bytes in [ + &include_bytes!("../tests/fixtures/rar15_40/rar250_protect_head_rr1.rar")[..], + &include_bytes!("../tests/fixtures/rar15_40/rar300/with_recovery_rar300.rar")[..], + ] { + let fail = Arc::new(AtomicBool::new(false)); + let source = crate::source::ReaderSource::new(FailingSource { + bytes: std::io::Cursor::new(bytes.to_vec()), + fail: fail.clone(), + }) + .unwrap(); + let archive = Archive::parse_seekable( + source.cursor(), + bytes.len() as u64, + 0, + ArchiveSource::Reader(source), + crate::ArchiveReadOptions::new(), + ) + .unwrap(); + fail.store(true, Ordering::Relaxed); + assert!( + matches!(archive.repair_protect_head(), Err(Error::Io(error)) + if error.kind == std::io::ErrorKind::PermissionDenied && error.message == "recovery source denied") + ); + } + let archive = preservation_seed(ArchiveVersion::Rar29); + assert!(matches!( + archive.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x archive does not contain a PROTECT_HEAD recovery record" + )) + )); + } + + #[test] + fn complete_header_crc_failures_survive_plain_and_encrypted_adapters() { + for encrypted in crate::test_features::encryption_cases() { + for corrupt in [false, true] { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, if encrypted { MHD_PASSWORD } else { 0 }); + let mut header = vec![0; 7]; + header[2] = ENDARC_HEAD; + header[5..7].copy_from_slice(&7u16.to_le_bytes()); + test_write_header_crc(&mut header, 0); + if corrupt { + header[0] ^= 1; + } + if encrypted { + #[cfg(feature = "encryption")] + { + header.resize(16, 0); + crate::crypto::rar30::Rar30Cipher::new(b"pw", Some([0; 8])) + .unwrap() + .encrypt_in_place(&mut header) + .unwrap(); + bytes.extend_from_slice(&[0; 8]); + } + } + bytes.extend_from_slice(&header); + let options = crate::ArchiveReadOptions::with_password(b"pw"); + for result in [ + Archive::parse_with_options(&bytes, options), + Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + options, + ), + ] { + if corrupt { + assert!( + matches!(result, Err(Error::CrcMismatch { .. })), + "{result:?}" + ); + } else { + assert!(matches!(result.unwrap().blocks.last(), Some(Block::End(_)))); + } + } + } + } + } + + #[test] + #[cfg(feature = "recovery")] + fn rev3_repair_preserves_reconstruction_and_publication_failures() { + let mut calls = 0; + let error = repair_rev3_volumes_to(&[], 1, &[(0, b"data")], |_, _| { + calls += 1; + Ok(()) + }) + .unwrap_err(); + assert!(matches!( + error, + Error::Rar3Recovery(crate::recovery::rar3::Error::InvalidCodewordSize) + )); + assert_eq!(calls, 0); + + let volumes: &[Option<&[u8]>] = &[Some(b"data"), Some(b"next")]; + let error = repair_rev3_volumes_to(volumes, 1, &[(0, b"0000")], |index, bytes| { + calls += 1; + assert_eq!(index, 0); + assert_eq!(bytes, b"data"); + Err( + std::io::Error::new(std::io::ErrorKind::PermissionDenied, "publication denied") + .into(), + ) + }) + .unwrap_err(); + assert!( + matches!(error, Error::Io(ref source) if source.kind == std::io::ErrorKind::PermissionDenied && source.message == "publication denied") + ); + assert_eq!(calls, 1, "publication stops at the failing volume"); + let mut published = Vec::new(); + repair_rev3_volumes_to(volumes, 1, &[(0, b"0000")], |index, bytes| { + published.push((index, bytes.to_vec())); + Ok(()) + }) + .unwrap(); + assert_eq!(published, [(0, b"data".to_vec()), (1, b"next".to_vec())]); + } + + #[test] + fn preservation_refuses_unsafe_unicode_names_and_independent_fallbacks() { + for unsafe_decoded in [false, true] { + let mut archive = preservation_seed(ArchiveVersion::Rar29); + let file = preservation_file(&mut archive); + if unsafe_decoded { + file.name = b"../entry".to_vec(); + file.unicode_name = Some(b"../entry\0".to_vec()); + } else { + let mut raw = crate::filename::encode_legacy_unicode(b"entry").unwrap(); + // Full UTF-16 commands leave the decoded name independent of this fallback. + raw[..5].copy_from_slice(b"/evil"); + assert_eq!(decode_file_name(&raw, FHD_UNICODE), b"entry"); + file.unicode_name = Some(raw); + } + assert!(archive + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("malformed or unsupported legacy Unicode name"))); + } + } + + #[test] + fn nested_comment_crc_failure_survives_both_parser_adapters() { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, MHD_COMMENT); + let nested = bytes.len(); + bytes.extend_from_slice(&comment_block(b"note")); + let main = RAR15_SIGNATURE.len(); + let size = (bytes.len() - main) as u16; + bytes[main + 5..main + 7].copy_from_slice(&size.to_le_bytes()); + test_write_header_crc(&mut bytes[..main + MAIN_HEADER_SIZE], main); + for corrupt in [false, true] { + let mut input = bytes.clone(); + if corrupt { + input[nested] ^= 1; + } + let memory = Archive::parse(&input); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&input), + input.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(input.clone())), + crate::ArchiveReadOptions::new(), + ); + if corrupt { + assert!(matches!(memory, Err(Error::CrcMismatch { .. }))); + assert!(matches!(seekable, Err(Error::CrcMismatch { .. }))); + } else { + for archive in [memory.unwrap(), seekable.unwrap()] { + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"note"[..]) + ); + } + } + } + } + + #[test] + fn edited_archive_comment_range_preserves_source_refusal() { + let mut builder = crate::Builder::new(ArchiveVersion::Rar15) + .store(true) + .comment(Some(b"note".to_vec())); + builder + .add_bytes(b"entry".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let mut archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"note"[..]) + ); + let comment = archive + .blocks + .iter_mut() + .find_map(|block| match block { + Block::Comment(comment) => Some(comment), + _ => None, + }) + .unwrap(); + comment.packed_range = usize::MAX..usize::MAX; + assert!(matches!(archive.archive_comment(), Err(Error::TooShort))); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn seekable_offsets_above_native_address_size_are_refused() { + use std::io::{self, Read, Seek, SeekFrom}; + struct HighInput<'a> { + start: u64, + position: u64, + bytes: &'a [u8], + } + impl Read for HighInput<'_> { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if !(self.start..self.start + self.bytes.len() as u64).contains(&self.position) { + return Ok(0); + } + let offset = (self.position - self.start) as usize; + let length = output.len().min(self.bytes.len() - offset); + output[..length].copy_from_slice(&self.bytes[offset..offset + length]); + self.position += length as u64; + Ok(length) + } + } + impl Seek for HighInput<'_> { + fn seek(&mut self, position: SeekFrom) -> io::Result { + match position { + SeekFrom::Start(position) => { + self.position = position; + Ok(position) + } + _ => Err(io::ErrorKind::Unsupported.into()), + } + } + } + let mut main = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut main, 0); + for (start, bytes, expected) in [ + ( + u64::from(u32::MAX) - 3, + &RAR15_SIGNATURE[..], + "RAR 1.5 block offset overflows usize", + ), + ( + u64::from(u32::MAX) - 17, + &main[..], + "RAR 1.5 block size overflows usize", + ), + ] { + let result = Archive::parse_seekable( + HighInput { + start, + position: 0, + bytes, + }, + start + bytes.len() as u64, + start as usize, + ArchiveSource::Memory(Arc::from(&[][..])), + crate::ArchiveReadOptions::new(), + ); + assert!( + matches!(result, Err(Error::InvalidHeader(message)) if message == expected), + "{result:?}" + ); + } + // A complete salt/first AES block may fit the u64 file but leave the + // address of its (possibly empty) tail beyond native usize. + let mut header = vec![0; 16]; + header[2] = ENDARC_HEAD; + header[5..7].copy_from_slice(&7u16.to_le_bytes()); + test_write_header_crc(&mut header[..7], 0); + Rar30Cipher::new(b"pw", Some([0; 8])) + .unwrap() + .encrypt_in_place(&mut header) + .unwrap(); + let mut bytes = vec![0; 8]; + bytes.extend_from_slice(&header); + let start = u64::from(u32::MAX) - 10; + let result = read_encrypted_header_at( + &mut HighInput { + start, + position: 0, + bytes: &bytes, + }, + start + bytes.len() as u64, + start as usize, + 0, + b"pw", + &mut EncryptedHeaderCipherCache::default(), + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ); + assert!(matches!( + result, + Err(Error::InvalidHeader("RAR 1.5 block offset overflows usize")) + )); + } + + #[test] + fn protection_header_requires_a_declared_data_size() { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, 0); + let start = bytes.len(); + let mut header = vec![0; 26]; + header[2] = PROTECT_HEAD; + header[5..7].copy_from_slice(&26u16.to_le_bytes()); + bytes.extend_from_slice(&header); + test_write_header_crc(&mut bytes, start); + for result in [ + Archive::parse(&bytes), + Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ), + ] { + assert!(matches!( + result, + Err(Error::InvalidHeader( + "RAR 2.x recovery header is missing data size" + )) + )); + } + } + + #[test] + fn maximum_large_packed_size_is_refused_before_payload_access() { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, 0); + let start = bytes.len(); + let mut header = vec![0; 41]; + header[2] = FILE_HEAD; + header[3..5].copy_from_slice(&(LONG_BLOCK | FHD_LARGE).to_le_bytes()); + header[5..7].copy_from_slice(&41u16.to_le_bytes()); + header[7..11].copy_from_slice(&u32::MAX.to_le_bytes()); + header[24] = 29; + header[25] = 0x30; + header[26..28].copy_from_slice(&1u16.to_le_bytes()); + header[32..36].copy_from_slice(&u32::MAX.to_le_bytes()); + header[40] = b'x'; + bytes.extend_from_slice(&header); + test_write_header_crc(&mut bytes, start); + for result in [ + Archive::parse(&bytes), + Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ), + ] { + let expected = if usize::BITS == 32 { + // The low u32 payload plus its header already exceeds usize. + "RAR 1.5 block size overflows usize" + } else { + "RAR 1.5 packed file size overflows usize" + }; + assert!( + matches!( + result, + Err(Error::InvalidHeader(message)) if message == expected + ), + "expected {expected}, got {result:?}" + ); + } + } + + #[test] + fn sfx_payload_ranges_agree_for_plain_and_encrypted_headers() { + for encrypted in crate::test_features::encryption_cases() { + let mut builder = crate::Builder::new(if encrypted { + ArchiveVersion::Rar30 + } else { + ArchiveVersion::Rar29 + }) + .store(true) + .password(encrypted.then(|| b"secret".to_vec())) + .header_encryption(encrypted); + builder + .add_bytes( + b"payload".to_vec(), + b"sfx payload bytes".to_vec(), + None, + None, + ) + .unwrap(); + let mut bytes = vec![0; 19]; + bytes.extend_from_slice(&builder.to_bytes().unwrap()); + let options = crate::ArchiveReadOptions::with_optional_password( + encrypted.then_some(&b"secret"[..]), + ); + let memory = Archive::parse_with_options(&bytes, options).unwrap(); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 19, + ArchiveSource::Memory(Arc::from(bytes.clone())), + options, + ) + .unwrap(); + assert_eq!(memory.sfx_offset, 19); + assert_eq!(memory.blocks, seekable.blocks); + for archive in [&memory, &seekable] { + let mut output = Vec::new(); + archive + .files() + .next() + .unwrap() + .write_stored_to(archive, options.password, &mut output) + .unwrap(); + assert_eq!(output, b"sfx payload bytes"); + } + } + } + + #[test] + #[cfg(feature = "recovery")] + fn edited_public_recovery_ranges_keep_typed_refusals() { + let old = Archive::parse(include_bytes!( + "../tests/fixtures/rar15_40/rar250_protect_head_rr1.rar" + )) + .unwrap(); + let mut out_of_source = old.clone(); + let protect = out_of_source + .blocks + .iter_mut() + .find_map(|block| match block { + Block::Protect(protect) => Some(protect), + _ => None, + }) + .unwrap(); + protect.data_range = usize::MAX..usize::MAX; + assert!(matches!( + out_of_source.repair_protect_head(), + Err(Error::TooShort) + )); + let mut overflow = old; + overflow.sfx_offset = usize::MAX; + assert!(matches!( + overflow.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x protected sector range overflows" + )) + )); + + let mut modern = Archive::parse(include_bytes!( + "../tests/fixtures/rar15_40/rar300/with_recovery_rar300.rar" + )) + .unwrap(); + modern.sfx_offset = 1; + let recovery = modern + .blocks + .iter_mut() + .find_map(|block| match block { + Block::NewSub(sub) if sub.kind == NewSubKind::RecoveryRecord => Some(sub), + _ => None, + }) + .unwrap(); + recovery.file.block.offset = usize::MAX; + assert!(matches!( + modern.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery protected range overflows" + )) + )); + } + + #[test] + fn comment_decoders_preserve_limits_and_each_cancellation_checkpoint() { + let text = b"checkpoint comment"; + for (version, method, packed) in [ + (15, 0x30, text.to_vec()), + ( + 15, + 0x31, + crate::codec::rar13::unpack15_encode(text).unwrap(), + ), + ( + 20, + 0x31, + crate::codec::rar20::unpack20_encode_literals(text).unwrap(), + ), + ( + 26, + 0x31, + crate::codec::rar20::unpack20_encode_literals(text).unwrap(), + ), + ] { + let mut block = block_header_with(0); + block.head_type = COMM_HEAD; + block.head_size = COMMENT_HEADER_SIZE as u16; + let comment = CommentHeader { + block, + unp_size: text.len() as u16, + unp_ver: version, + method, + comment_crc: (crc32(text) & 0xffff) as u16, + packed_range: 0..packed.len(), + }; + assert_eq!(comment.decode(&packed).unwrap(), text); + let mut limited = crate::output_limit::OutputBudget::new( + crate::ArchiveReadOptions::new().with_max_member_output_bytes(0), + ); + assert!(matches!( + comment + .decode_with_budget(&packed, &mut limited) + .unwrap_err() + .root_cause(), + Error::MemberOutputLimitExceeded { .. } + )); + let mut completed = false; + for checks in 0..128 { + let token = crate::ReadCancellation::new(); + let mut budget = crate::output_limit::OutputBudget::new( + crate::ArchiveReadOptions::new().with_cancellation(&token), + ); + budget.control.cancel_after_checks(checks); + match comment.decode_with_budget(&packed, &mut budget) { + Ok(output) => { + assert_eq!(output, text); + assert!(checks > 2); + completed = true; + break; + } + Err(error) => assert!( + matches!(error.root_cause(), Error::Cancelled), + "version {version}, check {checks}: {error:?}" + ), + } + } + assert!( + completed, + "version {version}: checkpoint sweep did not finish" + ); + } + } + + #[test] + fn file_comment_rejects_a_complete_but_undersized_comment_header() { + let mut file = file_header_with(FHD_COMMENT); + file.file_comment = comment_block(b"note"); + file.file_comment[5..7].copy_from_slice(&12u16.to_le_bytes()); + test_write_header_crc(&mut file.file_comment[..12], 0); + assert!(matches!( + file.file_comment(), + Err(Error::InvalidHeader("RAR 1.5 comment header is too short")) + )); + } + + #[test] + fn directory_callbacks_preserve_failure_and_cancellation_in_both_extractors() { + let mut builder = crate::Builder::new(ArchiveVersion::Rar29); + builder + .add_directory(b"directory".to_vec(), None, None) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + for parallel in [false, true] { + for cancel in [false, true] { + let token = crate::ReadCancellation::new(); + let mut calls = 0; + let open = |meta: &ExtractedEntryMeta| -> Result> { + calls += 1; + assert!(meta.is_directory); + assert_eq!(meta.name, b"directory"); + if cancel { + token.cancel(); + Ok(Box::new(std::io::sink())) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "directory refused", + ) + .into()) + } + }; + let options = crate::ArchiveReadOptions::new().with_cancellation(&token); + let error = if parallel { + archive.extract_to_parallel_buffered(options, open) + } else { + archive.extract_to(options, open) + } + .unwrap_err(); + if cancel { + assert!(matches!(error.root_cause(), Error::Cancelled)); + } else { + assert!( + matches!(error.root_cause(), Error::Io(error) if error.kind == std::io::ErrorKind::PermissionDenied && error.message == "directory refused") + ); + } + assert_eq!(calls, 1); + } + } + } + + #[test] + fn block_prefix_truncation_returns_too_short_before_field_decoding() { + let bytes = stored_archive_bytes(b"member", b"payload"); + let file = Archive::parse(&bytes) + .unwrap() + .files() + .next() + .unwrap() + .clone(); + let header = + &bytes[file.block.offset..file.block.offset + usize::from(file.block.head_size)]; + for prefix in 0..header.len() { + assert!(matches!( + parse_block_header(&header[..prefix], 0), + Err(Error::TooShort) + )); + } + assert_eq!( + parse_block_header(header, 0).unwrap().head_size, + file.block.head_size + ); + } + + #[test] + fn incomplete_large_size_fields_have_matching_parser_refusals() { + for head_type in [FILE_HEAD, NEWSUB_HEAD] { + for size in 32u16..=40 { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, 0); + let start = bytes.len(); + let mut header = vec![0; usize::from(size)]; + header[2] = head_type; + header[3..5].copy_from_slice(&(LONG_BLOCK | FHD_LARGE).to_le_bytes()); + header[5..7].copy_from_slice(&size.to_le_bytes()); + header[24] = 29; + header[25] = 0x30; + bytes.extend_from_slice(&header); + test_write_header_crc(&mut bytes, start); + let memory = Archive::parse(&bytes); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ); + if size < 40 { + assert!(matches!(memory, Err(Error::TooShort)), "memory size {size}"); + assert!( + matches!(seekable, Err(Error::TooShort)), + "seekable size {size}" + ); + } else { + assert_eq!(memory.unwrap().blocks, seekable.unwrap().blocks); + } + } + } + } + + #[test] + fn seekable_parser_preserves_failures_at_each_input_operation() { + use std::io::{self, Cursor, Read, Seek, SeekFrom}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + struct FaultReader { + input: Cursor>, + operations: Arc, + fail_at: usize, + cancellation: Option, + } + impl FaultReader { + fn check(&self) -> io::Result<()> { + let operation = self.operations.fetch_add(1, Ordering::Relaxed); + if operation == self.fail_at { + if let Some(token) = &self.cancellation { + token.cancel(); + return Ok(()); + } + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "input fault", + )) + } else { + Ok(()) + } + } + } + impl Read for FaultReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + self.check()?; + self.input.read(output) + } + } + impl Seek for FaultReader { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.check()?; + self.input.seek(position) + } + } + + let mut cases = Vec::new(); + for (target, encrypted) in [ + (ArchiveVersion::Rar15, false), + (ArchiveVersion::Rar20, false), + (ArchiveVersion::Rar29, false), + #[cfg(feature = "encryption")] + (ArchiveVersion::Rar30, true), + #[cfg(feature = "encryption")] + (ArchiveVersion::Rar40, true), + ] { + let mut builder = crate::Builder::new(target) + .store(true) + .comment(Some(b"archive comment".to_vec())) + .password(encrypted.then(|| b"secret".to_vec())) + .header_encryption(encrypted); + builder + .add_bytes(b"member".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + cases.push((builder.to_bytes().unwrap(), encrypted)); + } + // Historical recovery records cannot be produced by the legacy writer. + for bytes in [ + &include_bytes!("../tests/fixtures/rar15_40/rar250_protect_head_rr1.rar")[..], + &include_bytes!("../tests/fixtures/rar15_40/rar300/with_recovery_rar300.rar")[..], + ] { + cases.push((bytes.to_vec(), false)); + } + for (bytes, encrypted) in cases { + let options = if encrypted { + crate::ArchiveReadOptions::with_password(b"secret") + } else { + crate::ArchiveReadOptions::new() + }; + let expected = Archive::parse_with_options(&bytes, options).unwrap(); + let operations = Arc::new(AtomicUsize::new(0)); + let parse = |fail_at, cancellation: Option<&crate::ReadCancellation>| { + operations.store(0, Ordering::Relaxed); + Archive::parse_seekable( + FaultReader { + input: Cursor::new(bytes.clone()), + operations: operations.clone(), + fail_at, + cancellation: cancellation.cloned(), + }, + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + cancellation.map_or_else(|| options, |token| options.with_cancellation(token)), + ) + }; + let archive = parse(usize::MAX, None).unwrap(); + assert_eq!(archive.blocks, expected.blocks); + assert_eq!( + archive.archive_comment().unwrap(), + expected.archive_comment().unwrap() + ); + let count = operations.load(Ordering::Relaxed); + let cancelled = crate::ReadCancellation::new(); + cancelled.cancel(); + assert!(matches!( + parse(usize::MAX, Some(&cancelled)) + .unwrap_err() + .root_cause(), + Error::Cancelled + )); + assert_eq!(operations.load(Ordering::Relaxed), 0); + assert!(count > 0); + for operation in 0..count { + let error = parse(operation, None).unwrap_err(); + match error.root_cause() { + Error::Io(error) => { + assert_eq!(error.kind, io::ErrorKind::PermissionDenied); + assert_eq!(error.message, "input fault"); + } + error => panic!("operation {operation}: {error:?}"), + } + assert_eq!(operations.load(Ordering::Relaxed), operation + 1); + let token = crate::ReadCancellation::new(); + let error = parse(operation, Some(&token)).unwrap_err(); + assert!(matches!(error.root_cause(), Error::Cancelled)); + assert_eq!(operations.load(Ordering::Relaxed), operation + 1); + } + } + } + + #[test] + fn audit_nested_comments_and_time_fields_preserve_legacy_admission() { + let mut time = file_header_with(0); + time.ext_time = 0x0800u16.to_le_bytes().to_vec(); + assert_eq!(time.mtime_refinement(), None); + for (kind, size) in [(COMM_HEAD, 13u16), (COMM_HEAD, 12), (0x7f, 13)] { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, MHD_COMMENT); + let nested_start = bytes.len(); + bytes.extend_from_slice(&comment_block(b"")); + bytes[nested_start + 2] = kind; + bytes[nested_start + 5..nested_start + 7].copy_from_slice(&size.to_le_bytes()); + let main_start = RAR15_SIGNATURE.len(); + let main_len = (bytes.len() - main_start) as u16; + bytes[main_start + 5..main_start + 7].copy_from_slice(&main_len.to_le_bytes()); + test_write_header_crc(&mut bytes[..main_start + 13], main_start); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(std::sync::Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ) + .unwrap(); + assert_eq!( + seekable.blocks.len(), + usize::from(kind == COMM_HEAD && size == 13) + ); + } + assert!(matches!( + Archive::parse_with_options( + RAR15_SIGNATURE, + crate::ArchiveReadOptions::new().with_max_header_bytes(100), + ), + Err(Error::TooShort) + )); + } + + #[test] + fn seekable_comment_unknown_and_truncated_payload_match_memory_parser() { + for (kind, size, add_size) in [ + (COMM_HEAD, 13u16, 0u32), + (0x7f, 11, 0), + (0x7f, 11, 1), + (COMM_HEAD, 12, 0), + ] { + let mut bytes = RAR15_SIGNATURE.to_vec(); + test_write_main_header(&mut bytes, 0); + let start = bytes.len(); + bytes.extend_from_slice(&0u16.to_le_bytes()); + bytes.push(kind); + bytes.extend_from_slice(&0x8000u16.to_le_bytes()); + bytes.extend_from_slice(&size.to_le_bytes()); + bytes.extend_from_slice(&add_size.to_le_bytes()); + bytes.resize(start + usize::from(size), 0); + test_write_header_crc(&mut bytes, start); + let memory = Archive::parse(&bytes); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(std::sync::Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ); + if add_size != 0 || size < 13 && kind == COMM_HEAD { + assert_eq!(memory.unwrap_err().kind(), seekable.unwrap_err().kind()); + } else { + let memory = memory.unwrap(); + let seekable = seekable.unwrap(); + assert_eq!(memory.blocks, seekable.blocks); + } + } + } + + fn test_write_main_header(out: &mut Vec, flags: u16) { + let start = out.len(); + out.extend_from_slice(&0u16.to_le_bytes()); + out.push(MAIN_HEAD); + out.extend_from_slice(&flags.to_le_bytes()); + out.extend_from_slice(&13u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); + test_write_header_crc(out, start); + } + + fn test_write_header_crc(out: &mut [u8], start: usize) { + let crc = (crc32(&out[start + 2..]) & 0xffff) as u16; + out[start..start + 2].copy_from_slice(&crc.to_le_bytes()); + } + + fn legacy_auth_block(head_type: u8, head_size: u16) -> Vec { + let mut block = vec![0; head_size as usize]; + block[2] = head_type; + block[5..7].copy_from_slice(&head_size.to_le_bytes()); + block + } + + #[test] + fn rejects_too_short_legacy_auth_blocks_even_without_crc_check() { + assert!(matches!( + parse_block_header(&legacy_auth_block(0x76, 20), 0), + Err(Error::InvalidHeader( + "RAR legacy authenticity block is too short" + )) + )); + assert!(matches!( + parse_block_header(&legacy_auth_block(0x79, 181), 0), + Err(Error::InvalidHeader( + "RAR legacy authenticity block is too short" + )) + )); + } + + #[test] + fn accepts_minimum_legacy_auth_block_sizes_with_bad_crc() { + assert_eq!( + parse_block_header(&legacy_auth_block(0x76, 21), 0) + .unwrap() + .head_size, + 21 + ); + assert_eq!( + parse_block_header(&legacy_auth_block(0x79, 182), 0) + .unwrap() + .head_size, + 182 + ); + } + + #[test] + fn parses_fhd_large_high_size_fields_from_file_header() { + let name = b"large.bin"; + let head_size = 32 + 8 + name.len(); + let mut header = Vec::new(); + header.extend_from_slice(&0u16.to_le_bytes()); + header.push(FILE_HEAD); + header.extend_from_slice(&(LONG_BLOCK | FHD_LARGE).to_le_bytes()); + header.extend_from_slice(&(head_size as u16).to_le_bytes()); + header.extend_from_slice(&0x89ab_cdefu32.to_le_bytes()); + header.extend_from_slice(&0x7654_3210u32.to_le_bytes()); + header.push(3); + header.extend_from_slice(&0x1234_5678u32.to_le_bytes()); + header.extend_from_slice(&0x5a21_0000u32.to_le_bytes()); + header.push(29); + header.push(0x35); + header.extend_from_slice(&(name.len() as u16).to_le_bytes()); + header.extend_from_slice(&0x20u32.to_le_bytes()); + header.extend_from_slice(&1u32.to_le_bytes()); + header.extend_from_slice(&2u32.to_le_bytes()); + header.extend_from_slice(name); + + let block = BlockHeader { + head_crc: 0, + head_type: FILE_HEAD, + flags: LONG_BLOCK | FHD_LARGE, + head_size: head_size as u16, + add_size: Some(0x89ab_cdef), + offset: 0, + }; + let parsed = parse_file_like_header(&header, block, 0); + if usize::BITS == 32 { + assert!(matches!( + parsed, + Err(Error::InvalidHeader( + "RAR 1.5 packed file size overflows usize" + )) + )); + return; + } + let file = parsed.unwrap(); + + assert_eq!(file.pack_size, 0x0000_0001_89ab_cdef); + assert_eq!(file.unp_size, 0x0000_0002_7654_3210); + assert_eq!(file.name, name); + assert_eq!( + file.packed_range, + head_size..head_size + usize::try_from(0x0000_0001_89ab_cdefu64).unwrap() + ); + } + + #[test] + fn file_header_rejects_missing_size_and_overlong_optional_fields() { + let header = [0u8; 32]; + let mut block = block_header_with(LONG_BLOCK); + block.head_size = 31; + assert!(matches!( + parse_file_like_header(&header, block.clone(), 0), + Err(Error::InvalidHeader("RAR 1.5 file header is too short")) + )); + + block.head_size = 32; + block.flags = 0; + assert!(matches!( + parse_file_like_header(&header, block.clone(), 0), + Err(Error::InvalidHeader( + "RAR 1.5 file header is missing packed data size" + )) + )); + + block.flags = LONG_BLOCK; + let mut with_name = header; + with_name[26..28].copy_from_slice(&1u16.to_le_bytes()); + assert!(matches!( + parse_file_like_header(&with_name, block.clone(), 0), + Err(Error::InvalidHeader( + "RAR 1.5 file name extends beyond header" + )) + )); + + block.flags |= FHD_SALT; + assert!(matches!( + parse_file_like_header(&header, block, 0), + Err(Error::InvalidHeader( + "RAR 1.5 salt extends beyond file header" + )) + )); + } + + #[test] + fn fhd_large_archive_extent_uses_high_packed_size_without_underflowing() { + let name = b"large-zero-low.bin"; + let head_size = 32 + 8 + name.len(); + let mut archive = Vec::from(RAR15_SIGNATURE); + test_write_main_header(&mut archive, 0); + + let start = archive.len(); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.push(FILE_HEAD); + archive.extend_from_slice(&(LONG_BLOCK | FHD_LARGE).to_le_bytes()); + archive.extend_from_slice(&(head_size as u16).to_le_bytes()); + archive.extend_from_slice(&0u32.to_le_bytes()); + archive.extend_from_slice(&0u32.to_le_bytes()); + archive.push(3); + archive.extend_from_slice(&0u32.to_le_bytes()); + archive.extend_from_slice(&0x5a21_0000u32.to_le_bytes()); + archive.push(29); + archive.push(0x35); + archive.extend_from_slice(&(name.len() as u16).to_le_bytes()); + archive.extend_from_slice(&0x20u32.to_le_bytes()); + archive.extend_from_slice(&1u32.to_le_bytes()); + archive.extend_from_slice(&1u32.to_le_bytes()); + archive.extend_from_slice(name); + test_write_header_crc(&mut archive, start); + + let result = Archive::parse(&archive); + if usize::BITS == 32 { + assert!( + matches!( + result, + Err(Error::InvalidHeader( + "RAR 1.5 packed file size overflows usize" + )) + ), + "{result:?}" + ); + } else { + assert!(matches!(result, Err(Error::TooShort)), "{result:?}"); + } + } + + fn block_header_with(flags: u16) -> BlockHeader { + BlockHeader { + head_crc: 0, + head_type: FILE_HEAD, + flags, + head_size: 32, + add_size: None, + offset: 0, + } + } + + fn file_header_with(flags: u16) -> FileHeader { + FileHeader { + block: block_header_with(flags), + pack_size: 0, + unp_size: 0, + host_os: 0, + file_crc: 0, + file_time: 0, + unp_ver: 29, + method: 0x30, + name: b"entry".to_vec(), + unicode_name: None, + attr: 0, + salt: None, + file_comment: Vec::new(), + ext_time: Vec::new(), + packed_range: 0..0, + } + } + + fn main_header_with(flags: u16) -> MainHeader { + MainHeader { + head_crc: 0, + flags, + head_size: 13, + reserved1: 0, + reserved2: 0, + encrypt_version: None, + } + } + + #[test] + fn main_header_predicates_match_each_flag_bit() { + let cases = [ + (MHD_VOLUME, MainHeader::is_volume as fn(&MainHeader) -> bool), + (MHD_COMMENT, MainHeader::has_archive_comment), + (MHD_SOLID, MainHeader::is_solid), + (MHD_NEWNUMBERING, MainHeader::uses_new_numbering), + (MHD_PROTECT, MainHeader::has_recovery_record), + (MHD_PASSWORD, MainHeader::has_encrypted_headers), + (MHD_FIRSTVOLUME, MainHeader::is_first_volume), + ]; + let zero = main_header_with(0); + for (bit, predicate) in cases { + assert!( + !predicate(&zero), + "expected false when bit {bit:#x} is clear" + ); + let one = main_header_with(bit); + assert!(predicate(&one), "expected true when bit {bit:#x} is set"); + } + } + + #[test] + fn file_header_flag_predicates_track_each_bit() { + let cases = [ + ( + FHD_SPLIT_BEFORE, + FileHeader::is_split_before as fn(&FileHeader) -> bool, + ), + (FHD_SPLIT_AFTER, FileHeader::is_split_after), + (FHD_PASSWORD, FileHeader::is_encrypted), + (FHD_SOLID, FileHeader::is_solid), + (FHD_EXTTIME, FileHeader::has_ext_time), + ]; + let zero = file_header_with(0); + for (bit, predicate) in cases { + assert!( + !predicate(&zero), + "expected false on FileHeader for bit {bit:#x}" + ); + let one = file_header_with(bit); + assert!( + predicate(&one), + "expected true on FileHeader for bit {bit:#x}" + ); + } + + let directory = file_header_with(FHD_DIRECTORY_MASK); + assert!(directory.is_directory()); + assert!(!file_header_with(0).is_directory()); + + let stored = file_header_with(0); + assert!(stored.is_stored()); + let mut packed = file_header_with(0); + packed.method = 0x33; + assert!(!packed.is_stored()); + } + + /// A stored comment block holding `text`, as it sits inside a file header. + fn comment_block(text: &[u8]) -> Vec { + let head_size = (COMMENT_HEADER_SIZE + text.len()) as u16; + let mut block = vec![0, 0, COMM_HEAD, 0, 0]; + block.extend_from_slice(&head_size.to_le_bytes()); + block.extend_from_slice(&(text.len() as u16).to_le_bytes()); + block.push(15); + block.push(0x30); + block.extend_from_slice(&((crc32(text) & 0xffff) as u16).to_le_bytes()); + block.extend_from_slice(text); + let crc = (crc32(&block[2..COMMENT_HEADER_SIZE]) & 0xffff) as u16; + block[..2].copy_from_slice(&crc.to_le_bytes()); + block + } + + #[test] + fn file_header_comment_decodes_the_nested_comment_block() { + let mut without_flag = file_header_with(0); + without_flag.file_comment = comment_block(b"abc"); + assert!(!without_flag.has_file_comment()); + assert_eq!(without_flag.file_comment().unwrap(), None); + + let mut with_flag = file_header_with(FHD_COMMENT); + with_flag.file_comment = comment_block(b"hey"); + assert!(with_flag.has_file_comment()); + assert_eq!(with_flag.file_comment().unwrap().unwrap(), b"hey"); + + let mut empty_flagged = file_header_with(FHD_COMMENT); + empty_flagged.file_comment.clear(); + assert!(!empty_flagged.has_file_comment()); + + let mut truncated = file_header_with(FHD_COMMENT); + truncated.file_comment = comment_block(b"hey"); + truncated.file_comment.pop(); + assert!(matches!(truncated.file_comment(), Err(Error::TooShort))); + + let mut corrupt = file_header_with(FHD_COMMENT); + corrupt.file_comment = comment_block(b"hey"); + *corrupt.file_comment.last_mut().unwrap() = b'!'; + assert!(matches!( + corrupt.file_comment(), + Err(Error::CrcMismatch { .. }) + )); + + // The bare size and text RAR 1.3 uses, which this writer emitted by + // mistake until 0.7. + let mut bare_size = file_header_with(FHD_COMMENT); + bare_size.file_comment = vec![3, 0, b'h', b'e', b'y']; + assert!(bare_size.file_comment().is_err()); + } + + #[test] + fn file_header_comment_rejects_wrong_block_type_and_unsupported_codec() { + let mut wrong_type = file_header_with(FHD_COMMENT); + wrong_type.file_comment = comment_block(b"note"); + wrong_type.file_comment[2] = FILE_HEAD; + test_write_header_crc(&mut wrong_type.file_comment, 0); + assert!(matches!( + wrong_type.file_comment(), + Err(Error::InvalidHeader( + "RAR 1.5 file comment is not a comment block" + )) + )); + + let mut unsupported = file_header_with(FHD_COMMENT); + unsupported.file_comment = comment_block(b"note"); + unsupported.file_comment[9] = 29; + unsupported.file_comment[10] = 0x33; + let crc = (crc32(&unsupported.file_comment[2..COMMENT_HEADER_SIZE]) & 0xffff) as u16; + unsupported.file_comment[..2].copy_from_slice(&crc.to_le_bytes()); + let result = unsupported.file_comment(); + assert!( + matches!( + result, + Err(Error::UnsupportedCompression { + unpack_version: 29, + method: 0x33, + .. + }) + ), + "{result:?}" + ); + } + + #[test] + fn stored_file_comment_rejects_declared_size_mismatch() { + let mut header = file_header_with(FHD_COMMENT); + header.file_comment = comment_block(b"note"); + header.file_comment[7..9].copy_from_slice(&3u16.to_le_bytes()); + let crc = (crc32(&header.file_comment[2..COMMENT_HEADER_SIZE]) & 0xffff) as u16; + header.file_comment[..2].copy_from_slice(&crc.to_le_bytes()); + + assert!(matches!( + header.file_comment(), + Err(Error::InvalidHeader( + "RAR 1.5 stored comment has mismatched packed and unpacked sizes" + )) + )); + } + + #[test] + fn file_header_name_metadata_and_crc_helpers_describe_entry() { + let mut header = file_header_with(0); + header.name = b"r\xc3\xa9sum\xc3\xa9.txt".to_vec(); + header.file_crc = crc32(b"hello"); + header.attr = 0x20; + header.host_os = 3; + header.file_time = 0x5a21_0000; + + assert_eq!(header.name_bytes(), b"r\xc3\xa9sum\xc3\xa9.txt"); + assert_eq!(header.name_lossy(), "résumé.txt"); + let meta = header.metadata(); + assert_eq!(meta.name, header.name); + assert_eq!(meta.attr, 0x20); + assert_eq!(meta.host_os, 3); + assert_eq!(meta.file_time, 0x5a21_0000); + assert!(!meta.is_directory); + + header.verify_crc32(b"hello").unwrap(); + match header.verify_crc32(b"different") { + Err(Error::Crc32Mismatch { expected, actual }) => { + assert_eq!(expected, header.file_crc); + assert_ne!(actual, expected); + } + other => panic!("expected Crc32Mismatch, got {other:?}"), + } + + let directory = file_header_with(FHD_DIRECTORY_MASK); + assert!(directory.metadata().is_directory); + + // Garbage bytes still produce a String through lossy decoding. + let mut garbage = file_header_with(0); + garbage.name = vec![0xff, 0xfe, b'/', 0x80, b'x']; + let lossy = garbage.name_lossy(); + assert!(lossy.ends_with("/\u{fffd}x"), "got {lossy:?}"); + } + + #[test] + fn newsub_header_name_lossy_delegates_to_inner_file_header() { + let mut file = file_header_with(0); + file.name = b"CMT".to_vec(); + let sub = NewSubHeader { + file, + kind: NewSubKind::ArchiveComment, + }; + assert_eq!(sub.name_lossy(), "CMT"); + assert_eq!(sub.name_bytes(), b"CMT"); + } + + #[test] + fn writer_options_constructor_and_default_match_documented_targets() { + let default = WriterOptions::default(); + assert_eq!(default.target, ArchiveVersion::Rar15); + assert_eq!(default.features, FeatureSet::store_only()); + + let explicit = WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()); + assert_eq!(explicit.target, ArchiveVersion::Rar20); + assert_eq!(explicit.features, FeatureSet::store_only()); + } + + fn stored_archive_bytes(name: &[u8], data: &[u8]) -> Vec { + write_stored_archive( + &[StoredEntry { + name, + data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }], + WriterOptions::default(), + ) + .unwrap() + } + + fn preservation_seed(target: ArchiveVersion) -> Archive { + let mut builder = crate::Builder::new(target).store(true); + builder + .add_bytes(b"entry".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + archive + } + + fn preservation_file(archive: &mut Archive) -> &mut FileHeader { + archive + .blocks + .iter_mut() + .find_map(|block| match block { + Block::File(file) => Some(file), + _ => None, + }) + .unwrap() + } + + fn preservation_comment_seed(target: ArchiveVersion) -> Archive { + let mut builder = crate::Builder::new(target) + .store(true) + .comment(Some(b"archive note".to_vec())); + builder + .add_bytes(b"entry".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"entry", Some(b"member note".to_vec())) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + archive + } + + #[test] + fn preservation_old_comment_envelopes_refuse_unsupported_metadata() { + let seed = preservation_comment_seed(ArchiveVersion::Rar20); + let comment = seed + .blocks + .iter() + .find_map(|block| match block { + Block::Comment(comment) => Some(comment), + _ => None, + }) + .unwrap(); + assert!(comment.supports_rewrite()); + type Edit = fn(&mut CommentHeader); + let edits: &[Edit] = &[ + |c| c.block.head_type = FILE_HEAD, + |c| c.block.flags = 1, + |c| c.block.add_size = Some(0), + |c| c.block.head_size = 12, + |c| c.unp_size = u16::MAX, + |c| c.method = 0x36, + |c| { + c.method = 0x31; + c.unp_ver = 29; + }, + ]; + for edit in edits { + let mut edited = comment.clone(); + edit(&mut edited); + assert!(!edited.supports_rewrite(), "{edited:?}"); + } + for version in [15, 20, 26] { + let mut supported = comment.clone(); + supported.method = 0x35; + supported.unp_ver = version; + assert!(supported.supports_rewrite()); + } + } + + #[test] + fn preservation_archive_and_member_comments_report_layout_and_envelope_refusals() { + let standalone = preservation_comment_seed(ArchiveVersion::Rar20); + // Standalone comments remain preservable even without MHD_COMMENT. + let mut unflagged = standalone.clone(); + unflagged.main.flags &= !MHD_COMMENT; + assert!(unflagged.rewrite_preservation_issues().is_empty()); + let Block::Comment(comment) = &standalone.blocks[0] else { + unreachable!() + }; + let mut bytes = standalone + .source + .read_range(0..standalone.source.len().unwrap()) + .unwrap(); + let start = RAR15_SIGNATURE.len(); + let size = MAIN_HEADER_SIZE as u16 + comment.block.head_size; + bytes[start + 3..start + 5] + .copy_from_slice(&(standalone.main.flags | MHD_COMMENT).to_le_bytes()); + bytes[start + 5..start + 7].copy_from_slice(&size.to_le_bytes()); + test_write_header_crc(&mut bytes[start..start + MAIN_HEADER_SIZE], 0); + let seed = Archive::parse_owned(bytes).unwrap(); + assert!(seed.rewrite_preservation_issues().is_empty()); + type Edit = fn(&mut Archive); + let cases: &[(&str, Edit)] = &[ + ("legacy main header settings", |a| { + a.main.flags &= !MHD_COMMENT + }), + ("legacy main header settings", |a| { + a.main.head_size = MAIN_HEADER_SIZE as u16 + 1 + }), + ("legacy main header settings", |a| { + if let Block::Comment(c) = &mut a.blocks[0] { + c.block.offset += 1; + } + }), + ("archive comment location", |a| { + let comment = a.blocks.remove(0); + a.blocks.push(comment); + }), + ("archive comment location", |a| { + if let Block::Comment(c) = &mut a.blocks[0] { + c.method = 0x36; + } + }), + ("duplicate legacy archive comments", |a| { + a.blocks.insert(1, a.blocks[0].clone()) + }), + ("file comments have unsupported", |a| { + preservation_file(a).file_comment.clear() + }), + ("file comments have unsupported", |a| { + let c = &mut preservation_file(a).file_comment; + c[10] = 0x36; + test_write_header_crc(&mut c[..COMMENT_HEADER_SIZE], 0); + }), + ("file comments have unsupported", |a| { + preservation_file(a).file_comment.push(0) + }), + ( + "embedded legacy file comments with encrypted headers", + |a| a.main.flags |= MHD_PASSWORD, + ), + ]; + for &(expected, edit) in cases { + let mut archive = seed.clone(); + edit(&mut archive); + let issues = archive.rewrite_preservation_issues(); + assert!( + issues.iter().any(|issue| issue.contains(expected)), + "{expected}: {issues:?}" + ); + } + } + + #[test] + fn preservation_cmt_services_refuse_every_unsupported_metadata_field() { + let seed = preservation_comment_seed(ArchiveVersion::Rar30); + let index = seed + .blocks + .iter() + .position(|block| { + matches!(block, + Block::NewSub(sub) if sub.kind == NewSubKind::ArchiveComment) + }) + .unwrap(); + type Edit = fn(&mut FileHeader); + let edits: &[Edit] = &[ + |f| f.block.flags |= FHD_DIRECTORY_MASK, + |f| f.block.flags |= FHD_PASSWORD, + |f| f.block.flags |= 0x0800, + |f| f.block.head_size += 1, + |f| f.unp_ver = 20, + |f| f.method = 0x36, + |f| f.attr = 1, + |f| f.host_os = 4, + |f| f.unp_size = u64::from(u32::MAX) + 1, + ]; + for edit in edits { + let mut archive = seed.clone(); + let Block::NewSub(sub) = &mut archive.blocks[index] else { + unreachable!() + }; + edit(&mut sub.file); + assert!(archive + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("legacy CMT service metadata or encryption"))); + } + let mut late = seed.clone(); + let comment = late.blocks.remove(index); + late.blocks.push(comment); + assert!(late + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("legacy CMT service metadata"))); + let mut duplicate = seed.clone(); + duplicate + .blocks + .insert(index, duplicate.blocks[index].clone()); + assert!(duplicate + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("duplicate legacy archive comments"))); + let mut unknown = seed; + let Block::NewSub(sub) = &mut unknown.blocks[index] else { + unreachable!() + }; + sub.kind = NewSubKind::Unknown(b"future".to_vec()); + assert!(unknown + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("legacy recovery or other service records"))); + } + + #[test] + #[cfg(feature = "encryption")] + fn preservation_accepts_encrypted_cmt_with_its_declared_salt() { + let mut builder = crate::Builder::new(ArchiveVersion::Rar30) + .store(true) + .comment(Some(b"archive note".to_vec())) + .archive_comment_password(Some(b"secret".to_vec())); + builder + .add_bytes(b"entry".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + let comment = archive + .new_subs() + .find(|sub| sub.kind == NewSubKind::ArchiveComment) + .unwrap(); + assert!(comment.file.is_encrypted()); + assert!(comment.file.salt.is_some()); + assert!(archive.rewrite_preservation_issues().is_empty()); + } + + #[test] + fn preservation_accepts_unicode_names_and_consistent_compressed_solid_members() { + let mut builder = crate::Builder::new(ArchiveVersion::Rar29) + .solid(true) + .compression_level(Some(1)); + for name in [b"entry".as_slice(), b"second"] { + builder + .add_bytes(name.to_vec(), b"payload".repeat(50), None, None) + .unwrap(); + } + builder + .set_legacy_unicode_name( + b"entry", + crate::filename::encode_legacy_unicode(b"entry").unwrap(), + ) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.main.is_solid()); + assert!(archive.files().next().unwrap().unicode_name.is_some()); + assert!(archive.files().nth(1).unwrap().is_solid()); + assert!(archive.rewrite_preservation_issues().is_empty()); + } + + #[test] + fn preservation_end_records_require_complete_supported_layout() { + let seed = preservation_seed(ArchiveVersion::Rar29); + let mut bytes = seed + .source + .read_range(0..seed.source.len().unwrap()) + .unwrap(); + let start = bytes.len(); + bytes.extend([0, 0, ENDARC_HEAD, 0, 0, 7, 0]); + test_write_header_crc(&mut bytes[start..], 0); + let seed = Archive::parse_owned(bytes).unwrap(); + assert!(seed.rewrite_preservation_issues().is_empty()); + type Edit = fn(&mut BlockHeader); + let edits: &[Edit] = &[ + |e| e.flags = 1, + |e| e.head_size = 8, + |e| e.add_size = Some(0), + |e| e.offset += 1, + |e| e.offset = usize::MAX, + ]; + for edit in edits { + let mut archive = seed.clone(); + let Some(Block::End(end)) = archive.blocks.last_mut() else { + unreachable!() + }; + edit(end); + assert!(archive + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("legacy end header metadata or trailing bytes"))); + } + let mut encrypted_header = seed.clone(); + encrypted_header.main.flags |= MHD_PASSWORD; + let Some(Block::End(end)) = encrypted_header.blocks.last_mut() else { + unreachable!() + }; + // Header encryption requires salt plus a padded AES header block. + assert_eq!(end.offset + 7, encrypted_header.source.len().unwrap()); + assert!(encrypted_header + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("legacy end header metadata or trailing bytes"))); + let mut overflow = seed; + overflow.sfx_offset = usize::MAX; + assert!(overflow + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("legacy end header metadata or trailing bytes"))); + } + + #[test] + fn preservation_accepts_empty_non_solid_directories() { + for target in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let mut builder = crate::Builder::new(target); + builder + .add_directory(b"empty".to_vec(), None, None) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.is_directory()); + assert_eq!((file.pack_size, file.unp_size), (0, 0)); + assert!(!file.is_solid()); + let issues = archive.rewrite_preservation_issues(); + assert!(issues.is_empty(), "{target:?}: {issues:?}"); + } + } + + #[test] + fn preservation_preflight_reports_public_main_header_inconsistencies() { + let seed = preservation_seed(ArchiveVersion::Rar29); + type Edit = fn(&mut Archive); + let cases: &[(&str, Edit)] = &[ + ("legacy volume layout", |a| a.main.flags |= MHD_VOLUME), + ("legacy recovery records", |a| a.main.flags |= MHD_PROTECT), + ("legacy main header settings", |a| { + a.main.flags |= MHD_NEWNUMBERING + }), + ("legacy main header settings", |a| a.main.head_size += 1), + ("legacy main header settings", |a| a.main.reserved1 = 1), + ("legacy main header settings", |a| a.main.reserved2 = 1), + ("empty legacy archive", |a| { + a.blocks.retain(|b| !matches!(b, Block::File(_))) + }), + ("missing or malformed legacy archive comment", |a| { + a.main.flags |= MHD_COMMENT + }), + ]; + // These fields are public: a caller can edit a parsed archive before + // preflight. Every such refusal must remain observable and specific. + for &(expected, edit) in cases { + let mut archive = seed.clone(); + edit(&mut archive); + let issues = archive.rewrite_preservation_issues(); + assert!( + issues.iter().any(|issue| issue.contains(expected)), + "{expected}: {issues:?}" + ); + } + } + + #[test] + fn preservation_preflight_reports_public_member_metadata_inconsistencies() { + let seed = preservation_seed(ArchiveVersion::Rar29); + type Edit = fn(&mut FileHeader); + let cases: &[(&str, Edit)] = &[ + ("requires unpacker 21", |f| f.unp_ver = 21), + ("encryption salt settings", |f| { + f.unp_ver = 20; + f.salt = Some([0; 8]); + }), + ("encryption salt settings", |f| { + f.block.flags |= FHD_PASSWORD + }), + ("Pre-RAR2.9 extended timestamps", |f| { + f.unp_ver = 20; + f.block.flags |= FHD_EXTTIME; + f.ext_time = vec![0, 0]; + }), + ("extended timestamps are incomplete", |f| { + f.block.flags |= FHD_EXTTIME; + f.ext_time = vec![0]; + }), + ("legacy host metadata", |f| f.host_os = 4), + ("solid dependency without archive solid flag", |f| { + f.block.flags |= FHD_SOLID + }), + ("legacy file flags or extra metadata", |f| { + f.block.flags |= 0x0800 + }), + ("legacy file flags or extra metadata", |f| { + f.block.head_size += 1 + }), + ("unsupported legacy method or size", |f| f.method = 0x36), + ("unsupported legacy method or size", |f| { + f.unp_size = u64::from(u32::MAX) + 1 + }), + ("unsupported legacy directory payload", |f| { + f.block.flags |= FHD_DIRECTORY_MASK + }), + ("unsupported legacy directory payload", |f| { + f.block.flags |= FHD_DIRECTORY_MASK; + f.pack_size = 0; + }), + ("unsupported legacy directory payload", |f| { + f.block.flags |= FHD_DIRECTORY_MASK | FHD_SOLID; + f.pack_size = 0; + f.unp_size = 0; + }), + ("malformed or unsupported legacy Unicode name", |f| { + f.unicode_name = Some(b"bad".to_vec()) + }), + ]; + for &(expected, edit) in cases { + let mut archive = seed.clone(); + edit(preservation_file(&mut archive)); + let issues = archive.rewrite_preservation_issues(); + assert!( + issues.iter().any(|issue| issue.contains(expected)), + "{expected}: {issues:?}" + ); + } + let mut rar15 = preservation_seed(ArchiveVersion::Rar15); + preservation_file(&mut rar15).host_os = 3; + assert!(rar15 + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("RAR1.5 Unix metadata"))); + let mut rar15 = preservation_seed(ArchiveVersion::Rar15); + preservation_file(&mut rar15).unicode_name = + Some(crate::filename::encode_legacy_unicode(b"entry").unwrap()); + assert!(rar15 + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("malformed or unsupported legacy Unicode name"))); + } + + #[test] + fn flagged_but_missing_file_comment_does_not_hide_the_member() { + let mut bytes = stored_archive_bytes(b"entry", b"payload"); + let file_start = RAR15_SIGNATURE.len() + MAIN_HEADER_SIZE; + let head_size = usize::from(u16::from_le_bytes( + bytes[file_start + 5..file_start + 7].try_into().unwrap(), + )); + let flags = u16::from_le_bytes(bytes[file_start + 3..file_start + 5].try_into().unwrap()) + | FHD_COMMENT; + bytes[file_start + 3..file_start + 5].copy_from_slice(&flags.to_le_bytes()); + test_write_header_crc(&mut bytes[file_start..file_start + head_size], 0); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.name, b"entry"); + assert!(!file.has_file_comment()); + assert_eq!(file.packed_data(&archive).unwrap(), b"payload"); + } + + #[test] + #[cfg(feature = "recovery")] + fn repaired_legacy_volume_keeps_missing_end_block_and_nonzero_trailer() { + let mut bytes = stored_archive_bytes(b"entry", b"payload"); + let end_start = bytes.len(); + bytes.extend_from_slice(&[0, 0, ENDARC_HEAD, 0, 0, 7, 0]); + test_write_header_crc(&mut bytes[end_start..], 0); + let archive = Archive::parse(&bytes).unwrap(); + let end = archive + .blocks + .iter() + .find_map(|block| match block { + Block::End(end) => Some(end), + _ => None, + }) + .unwrap(); + let mut without_end = bytes[..end.offset].to_vec(); + assert!(Archive::parse(&without_end) + .unwrap() + .files() + .next() + .is_some()); + assert_eq!( + truncate_repaired_rev3_volume(without_end.clone()), + without_end + ); + + without_end = bytes; + without_end.extend_from_slice(b"not padding"); + assert_eq!( + truncate_repaired_rev3_volume(without_end.clone()), + without_end + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_header_readers_reject_short_prefixes_and_declared_headers() { + let mut cache = EncryptedHeaderCipherCache::default(); + for size in [0u16, 6, 17, 65] { + let mut plain = [0; 16]; + plain[5..7].copy_from_slice(&size.to_le_bytes()); + cache + .cipher(b"pw", [0; 8]) + .unwrap() + .encrypt_in_place(&mut plain) + .unwrap(); + let mut bytes = vec![0; 8]; + bytes.extend_from_slice(&plain); + let options = crate::ArchiveReadOptions::new(); + let memory = decrypt_encrypted_header_at( + &bytes, + 0, + b"pw", + &mut cache, + &mut crate::parse_budget::ParseBudget::new(options), + ) + .err() + .unwrap(); + let seekable = read_encrypted_header_at( + &mut std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + 0, + b"pw", + &mut cache, + &mut crate::parse_budget::ParseBudget::new(options), + ) + .err() + .unwrap(); + if size < 7 { + assert!(matches!( + memory, + Error::InvalidHeader("RAR 1.5 block header is too short") + )); + assert!(matches!( + seekable, + Error::InvalidHeader("RAR 1.5 block header is too short") + )); + } else { + assert!(matches!(memory, Error::TooShort)); + assert!(matches!(seekable, Error::TooShort)); + } + } + for available in [0, 7, 8, 23] { + let bytes = vec![0; available]; + assert!(matches!( + decrypt_encrypted_header_at( + &bytes, + 0, + b"pw", + &mut cache, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()) + ), + Err(Error::TooShort) + )); + assert!(matches!( + read_encrypted_header_at( + &mut std::io::Cursor::new(&bytes), + available as u64, + 0, + 0, + b"pw", + &mut cache, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()) + ), + Err(Error::TooShort) + )); + } + } + + #[test] + fn file_comment_crc_boundary_includes_large_sizes_name_and_salt() { + let mut short = vec![0; 11]; + short[2] = FILE_HEAD; + short[3..5].copy_from_slice(&(LONG_BLOCK | FHD_COMMENT).to_le_bytes()); + short[5..7].copy_from_slice(&11u16.to_le_bytes()); + assert!(matches!( + parse_block_header(&short, 0), + Err(Error::TooShort) + )); + for flags in [0, FHD_LARGE, FHD_SALT, FHD_LARGE | FHD_SALT] { + let name_start = 32 + if flags & FHD_LARGE != 0 { 8 } else { 0 }; + let crc_end = name_start + 4 + if flags & FHD_SALT != 0 { 8 } else { 0 }; + let mut header = vec![0; crc_end + 13]; + header[2] = FILE_HEAD; + header[3..5].copy_from_slice(&(LONG_BLOCK | FHD_COMMENT | flags).to_le_bytes()); + let size = header.len() as u16; + header[5..7].copy_from_slice(&size.to_le_bytes()); + header[24] = 29; + header[25] = 0x30; + header[26..28].copy_from_slice(&4u16.to_le_bytes()); + header[name_start..name_start + 4].copy_from_slice(b"name"); + header[crc_end + 2] = COMM_HEAD; + header[crc_end + 5..crc_end + 7].copy_from_slice(&13u16.to_le_bytes()); + assert_eq!(file_header_comment_crc_end(&header, 0).unwrap(), crc_end); + let crc = (crc32(&header[2..crc_end]) & 0xffff) as u16; + header[..2].copy_from_slice(&crc.to_le_bytes()); + let block = parse_block_header(&header, 0).unwrap(); + let file = parse_file_like_header(&header, block, 0).unwrap(); + assert_eq!(file.name, b"name"); + assert_eq!(file.file_comment.len(), 13); + } + } + + #[test] + fn main_encrypt_version_cannot_borrow_a_byte_from_the_next_header() { + let mut bytes = stored_archive_bytes(b"file", b"payload"); + let main_start = RAR15_SIGNATURE.len(); + let main_end = main_start + MAIN_HEADER_SIZE; + assert_eq!( + read_u16(&bytes, main_start + 5).unwrap(), + MAIN_HEADER_SIZE as u16 + ); + let flags = read_u16(&bytes, main_start + 3).unwrap() | MHD_ENCRYPTVER; + bytes[main_start + 3..main_start + 5].copy_from_slice(&flags.to_le_bytes()); + let crc = (crc32(&bytes[main_start + 2..main_end]) & 0xffff) as u16; + bytes[main_start..main_start + 2].copy_from_slice(&crc.to_le_bytes()); + let options = crate::ArchiveReadOptions::new(); + assert!(matches!( + Archive::parse_with_options(&bytes, options), + Err(Error::TooShort) + )); + assert!(matches!( + Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + options + ), + Err(Error::TooShort) + )); + } + + #[test] + fn memory_and_seekable_parsers_reject_invalid_file_header_lengths() { + let original = stored_archive_bytes(b"lengths.txt", b"payload"); + let file_offset = Archive::parse(&original) + .unwrap() + .files() + .next() + .unwrap() + .block + .offset; + + for (head_size, truncated) in [(6u16, false), (u16::MAX, true)] { + let mut bytes = original.clone(); + bytes[file_offset + 5..file_offset + 7].copy_from_slice(&head_size.to_le_bytes()); + + for options in [ + crate::ArchiveReadOptions::default(), + crate::ArchiveReadOptions::default().with_max_header_bytes(u64::MAX), + ] { + let memory = Archive::parse_with_options(&bytes, options).unwrap_err(); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone().into_boxed_slice())), + options, + ) + .unwrap_err(); + + if truncated { + assert!(matches!(memory, Error::TooShort)); + assert!(matches!(seekable, Error::TooShort)); + } else { + assert!(matches!(memory, Error::InvalidHeader(_))); + assert!(matches!(seekable, Error::InvalidHeader(_))); + } + } + } + } + + #[test] + fn parsers_reject_a_validly_checksummed_non_main_header() { + let mut bytes = stored_archive_bytes(b"entry.txt", b"payload"); + let main_offset = RAR15_SIGNATURE.len(); + let main_size = + u16::from_le_bytes([bytes[main_offset + 5], bytes[main_offset + 6]]) as usize; + bytes[main_offset + 2] = FILE_HEAD; + test_write_header_crc(&mut bytes[main_offset..main_offset + main_size], 0); + + let memory = Archive::parse(&bytes).unwrap_err(); + let seekable = Archive::parse_seekable( + std::io::Cursor::new(&bytes), + bytes.len() as u64, + 0, + ArchiveSource::Memory(Arc::from(bytes.clone().into_boxed_slice())), + crate::ArchiveReadOptions::default(), + ) + .unwrap_err(); + assert!(matches!( + memory, + Error::InvalidHeader("RAR 1.5 main header is missing") + )); + assert!(matches!( + seekable, + Error::InvalidHeader("RAR 1.5 main header is missing") + )); + } + + #[test] + fn archive_parse_owned_consumes_buffer_without_changing_dispatch() { + let bytes = stored_archive_bytes(b"owned.txt", b"hello rar15 owned"); + let archive = Archive::parse_owned(bytes.clone()).unwrap(); + assert_eq!(archive.files().count(), 1); + let file = archive.files().next().unwrap(); + assert_eq!(file.name, b"owned.txt"); + + // Default ArchiveReadOptions delegate also drives the same code path. + let with_options = + Archive::parse_owned_with_options(bytes.clone(), crate::ArchiveReadOptions::default()) + .unwrap(); + assert_eq!(with_options.files().count(), 1); + + let no_password = Archive::parse_owned_with_password(bytes, None).unwrap(); + assert_eq!(no_password.files().count(), 1); + } + + #[test] + #[cfg(feature = "encryption")] + fn archive_parse_owned_with_password_unlocks_encrypted_archive() { + let features = FeatureSet::store_only(); + let bytes = write_stored_archive( + &[StoredEntry { + name: b"locked.txt", + data: b"encrypted owned payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"pw"), + file_comment: None, + }], + WriterOptions { + target: ArchiveVersion::Rar20, + features, + compression_level: None, + dictionary_size: None, + method: Rar29Method::Auto, + archive_comment_metadata: None, + }, + ) + .unwrap(); + + let archive = Archive::parse_owned_with_password(bytes, Some(b"pw")).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.is_encrypted()); + } + + #[test] + fn file_header_write_packed_data_streams_through_writer() { + let payload = b"write_packed_data direct dump"; + let bytes = stored_archive_bytes(b"dump.bin", payload); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + let mut sink = Vec::new(); + file.write_packed_data(&archive, &mut sink).unwrap(); + assert_eq!(sink, payload); + } + + #[test] + fn direct_stored_writer_rejects_inconsistent_and_short_payloads() { + let bytes = stored_archive_bytes(b"entry", b"payload"); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + let mut inconsistent = file.clone(); + inconsistent.unp_size += 1; + let mut output = Vec::new(); + assert!(matches!( + inconsistent.write_to(&archive, None, &mut output), + Err(Error::InvalidHeader( + "RAR 1.5 stored file has mismatched packed and unpacked sizes" + )) + )); + assert!(output.is_empty()); + + // A valid header can still name a short range when its source is damaged. + let mut short = file.clone(); + short.packed_range.end -= 1; + assert!(matches!( + short.write_to(&archive, None, &mut output), + Err(Error::InvalidHeader( + "RAR 1.5 stored file ended before unpacked size" + )) + )); + assert_eq!(output, b"payloa"); + + let mut directory = file.clone(); + directory.attr |= 0x10; + let mut output = Vec::new(); + directory.write_to(&archive, None, &mut output).unwrap(); + assert!(output.is_empty()); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_stored_workspace_refusal_keeps_resource_error() { + let bytes = write_stored_archive( + &[StoredEntry { + name: b"entry", + data: b"payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"secret"), + file_comment: None, + }], + WriterOptions::new(ArchiveVersion::Rar30, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let mut output = Vec::new(); + let error = file + .write_stored_with_allowance( + &archive, + Some(b"secret"), + &mut output, + &Allowance::limited(0), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert!(output.is_empty()); + } + + #[test] + fn crc_writer_flush_propagates_to_inner_writer() { + struct FlushSpy { + data: Vec, + flushed: usize, + } + impl Write for FlushSpy { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.data.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.flushed += 1; + Ok(()) + } + } + let mut inner = FlushSpy { + data: Vec::new(), + flushed: 0, + }; + let mut crc = Crc32::new(); + let mut writer = CrcWriter { + inner: &mut inner, + crc: &mut crc, + }; + writer.write_all(b"hi").unwrap(); + writer.flush().unwrap(); + assert_eq!(inner.data, b"hi"); + assert_eq!(inner.flushed, 1); + } + + #[test] + fn parse_main_header_rejects_block_size_below_minimum() { + let block = BlockHeader { + head_crc: 0, + head_type: MAIN_HEAD, + flags: 0, + head_size: 12, + add_size: None, + offset: 0, + }; + let err = parse_main_header(&[0u8; 32], &block).unwrap_err(); + assert_eq!( + err, + Error::InvalidHeader("RAR 1.5 main header is too short") + ); + } + + #[test] + fn parse_main_header_reads_encrypt_version_when_flag_is_set() { + // Build a synthetic main header at offset 0: 13 bytes of base + 1 byte + // for the encrypt_version field at position 13. + let mut input = vec![0u8; 14]; + input[13] = 0x29; + let block = BlockHeader { + head_crc: 0, + head_type: MAIN_HEAD, + flags: MHD_ENCRYPTVER, + head_size: 14, + add_size: None, + offset: 0, + }; + let main = parse_main_header(&input, &block).unwrap(); + assert_eq!(main.encrypt_version, Some(0x29)); + } + + #[test] + fn malformed_unicode_filename_keeps_its_identity() { + let first = b"fallback\0\0\x80\0\xd8"; + let second = b"fallback\0\0\x80\x01\xd8"; + assert_eq!(decode_file_name(first, FHD_UNICODE), first); + assert_eq!(decode_file_name(second, FHD_UNICODE), second); + let invalid_copy = b"fallback\0\0\xc0\x7f"; + assert_eq!(decode_file_name(invalid_copy, FHD_UNICODE), invalid_copy); + } + + #[test] + fn decode_file_name_returns_raw_when_unicode_flag_is_clear() { + let raw = b"plain.txt"; + let decoded = decode_file_name(raw, 0); + assert_eq!(decoded, raw); + } + + #[test] + fn decode_file_name_returns_raw_when_unicode_marker_is_missing() { + // FHD_UNICODE set but no zero byte to split fallback from encoded + // payload — the decoder falls back to the raw bytes. + let raw = b"no-zero-marker"; + let decoded = decode_file_name(raw, FHD_UNICODE); + assert_eq!(decoded, raw); + } + + #[test] + fn decode_file_name_returns_fallback_when_no_encoded_payload_follows_zero() { + // Zero byte present but nothing after it — the decoder returns just + // the fallback prefix. + let mut raw = b"fallback".to_vec(); + raw.push(0); + let decoded = decode_file_name(&raw, FHD_UNICODE); + assert_eq!(decoded, b"fallback"); + } + + #[test] + fn decode_file_name_decodes_mode_zero_low_byte_only_units() { + // Mode 0 emits one ASCII byte per code unit. Encoded payload format: + // high_byte, flag_byte, byte0, byte1, ... + // flag_byte = 0b00_00_00_00 → four mode-0 emits. + let mut raw = b"orig".to_vec(); + raw.push(0); // separator + raw.push(0); // high_byte (unused for mode 0) + raw.push(0b00_00_00_00); // flag_byte: four mode-0 codes + raw.extend_from_slice(b"abcd"); + let decoded = decode_file_name(&raw, FHD_UNICODE); + assert_eq!(decoded, b"abcd"); + } + + #[test] + fn decode_file_name_decodes_mode_two_full_two_byte_units() { + // Mode 2 reads (low, high) bytes for a full UTF-16 code unit. + // Encode "Hi" via mode 2 twice: flag_byte = 0b10_10_00_00. + let mut raw = b"".to_vec(); + raw.push(0); // separator (zero_pos = 0) + raw.push(0); // high_byte placeholder + // Two mode-2 units, then two mode-0 units which we will not reach. + raw.push(0b10_10_00_00); + // First unit 'H' = U+0048: low=0x48, high=0x00 + raw.extend_from_slice(&[0x48, 0x00]); + // Second unit 'i' = U+0069 + raw.extend_from_slice(&[0x69, 0x00]); + let decoded = decode_file_name(&raw, FHD_UNICODE); + assert_eq!(decoded, b"Hi"); + } + + #[test] + fn decode_file_name_decodes_high_byte_and_copy_commands() { + assert_eq!( + decode_file_name(b"old\0\x04\x40\xe9", FHD_UNICODE), + "\u{4e9}".as_bytes() + ); + assert_eq!(decode_file_name(b"abc\0\0\xc0\x01", FHD_UNICODE), b"abc"); + assert_eq!( + decode_file_name(b"ABC\0\x01\xc0\x81\x01", FHD_UNICODE), + "\u{142}\u{143}\u{144}".as_bytes() + ); + } + + #[test] + fn truncated_unicode_commands_keep_the_raw_name() { + for raw in [ + &b"x\0\0\0"[..], // mode 0: missing low byte + &b"x\0\x01\x40"[..], // mode 1: missing low byte + &b"x\0\0\x80A"[..], // mode 2: missing high byte + &b"x\0\0\xc0"[..], // mode 3: missing run length + &b"x\0\0\xc0\x80"[..], // mode 3: missing correction + &b"x\0\0\xc0\0"[..], // mode 3: run exceeds fallback + ] { + assert_eq!(decode_file_name(raw, FHD_UNICODE), raw); + } + } +} diff --git a/arca-rar/vendor/rars/src/rar15_40/extract.rs b/arca-rar/vendor/rars/src/rar15_40/extract.rs new file mode 100644 index 00000000..94fb5fb4 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar15_40/extract.rs @@ -0,0 +1,2258 @@ +use super::*; +use crate::codec::rar13::Reader15State; +use crate::codec::rar20::Reader20State; +use crate::codec::rar29::Reader29State; +use crate::codec::workspace::{Allowance, Boxed, Budget, Buffer}; +use crate::volume_extract::{ChainedReader, SplitVolumeState, SplitVolumeStep}; +use std::io::{Read, Write}; + +enum CodecState { + Unpack15(Boxed, B>), + Unpack20(Boxed, B>), + Unpack29(Boxed, B>), +} + +impl CodecState { + fn with_allowance(file: &FileHeader, allowance: &B) -> Result { + if file.unp_ver >= 29 { + return Ok(Self::Unpack29(Boxed::try_new( + || Ok::<_, crate::codec::Error>(Reader29State::with_allowance(allowance)), + allowance, + )?)); + } + if file.unp_ver == 20 || file.unp_ver == 26 { + return Ok(Self::Unpack20(Boxed::try_new( + || Ok::<_, crate::codec::Error>(Reader20State::with_allowance(allowance)), + allowance, + )?)); + } + if file.unp_ver == 15 { + return Ok(Self::Unpack15(Boxed::try_new( + || Reader15State::with_allowance(allowance), + allowance, + )?)); + } + Err(Error::UnsupportedCompression { + family: "RAR 1.5-4.x", + unpack_version: file.unp_ver, + method: file.method, + }) + } + + fn supports(&self, file: &FileHeader) -> bool { + match self { + Self::Unpack15(_) => file.unp_ver == 15, + Self::Unpack20(_) => file.unp_ver == 20 || file.unp_ver == 26, + Self::Unpack29(_) => file.unp_ver >= 29, + } + } + + fn write_file_to( + &mut self, + archive: &Archive, + file: &FileHeader, + solid: bool, + password: Option<&[u8]>, + allowance: &B, + out: &mut impl Write, + ) -> Result<()> { + let mut packed = file + .packed_reader_with_allowance(archive, password, allowance) + .map_err(|error| file.map_encrypted_payload_error(password, error))?; + self.write_split_to(&mut packed, file, solid, password, out) + } + + fn decode_to( + &mut self, + input: &mut impl Read, + file: &FileHeader, + solid: bool, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + let target = usize::try_from(file.unp_size) + .map_err(|_| Error::InvalidHeader("RAR 1.5 split unpacked size overflows usize"))?; + match self { + Self::Unpack15(decoder) => decoder + .decode_member_from_reader(input, target, solid, out) + .map_err(Error::from) + .map_err(|error| file.map_encrypted_payload_error(password, error))?, + Self::Unpack20(decoder) => decoder + .decode_member_from_reader(input, target, out) + .map_err(Error::from) + .map_err(|error| file.map_encrypted_payload_error(password, error))?, + Self::Unpack29(decoder) => if solid { + decoder.decode_member_from_reader(input, target, out) + } else { + decoder.decode_non_solid_member_from_reader(input, target, out) + } + .map_err(Error::from) + .map_err(|error| file.map_encrypted_payload_error(password, error))?, + } + Ok(()) + } + fn write_split_to( + &mut self, + input: &mut impl Read, + file: &FileHeader, + solid: bool, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + let mut crc = Crc32::new(); + let mut crc_writer = CrcWriter { + inner: out, + crc: &mut crc, + }; + self.decode_to(input, file, solid, password, &mut crc_writer)?; + file.crc_result(crc.finish(), password) + } +} + +#[cfg(all(test, feature = "write"))] +impl CodecState { + fn new_for(file: &FileHeader) -> Result { + Self::with_allowance(file, &Allowance::default()) + } +} + +pub(super) struct DecoderSession<'a, B: Budget = Allowance> { + pub(super) read_control: crate::read_control::ReadControl, + codec: Option>, + solid: bool, + decoded_files: usize, + password: Option<&'a [u8]>, + allowance: B, +} + +impl<'a, B: Budget> DecoderSession<'a, B> { + pub(super) fn with_allowance(solid: bool, password: Option<&'a [u8]>, allowance: &B) -> Self { + Self { + read_control: crate::read_control::ReadControl::default(), + codec: None, + solid, + decoded_files: 0, + password, + allowance: allowance.clone(), + } + } + + /// Stream a compressed member; callers handle stored members separately. + pub(super) fn write_file_to( + &mut self, + archive: &Archive, + file: &FileHeader, + out: &mut impl Write, + ) -> Result<()> { + debug_assert!(!file.is_stored()); + if file.is_empty_compressed_payload() { + file.crc_result(0, self.password)?; + return Ok(()); + } + let solid = self.file_is_solid(file); + let password = self.password; + let allowance = self.allowance.clone(); + self.codec_for(file)? + .write_file_to(archive, file, solid, password, &allowance, out)?; + self.decoded_files += 1; + Ok(()) + } + + fn write_split_to( + &mut self, + input: &mut impl Read, + final_file: &FileHeader, + out: &mut impl Write, + ) -> Result<()> { + let solid = self.file_is_solid(final_file); + let password = self.password; + self.codec_for(final_file)? + .write_split_to(input, final_file, solid, password, out)?; + self.decoded_files += 1; + Ok(()) + } + + /// Decode compressed recovery data into a charged buffer. + #[cfg(any(all(test, feature = "write"), feature = "recovery"))] + pub(super) fn decode_file_owned( + &mut self, + archive: &Archive, + file: &FileHeader, + ) -> Result> { + debug_assert!(!file.is_stored()); + let mut out = Buffer::new(&self.allowance); + if file.is_empty_compressed_payload() { + file.crc_result(0, self.password)?; + return Ok(out); + } + let password = self.password; + let mut input = file.packed_reader_with_allowance(archive, password, &self.allowance)?; + let solid = self.file_is_solid(file); + self.codec_for(file)? + .decode_to(&mut input, file, solid, password, &mut out)?; + Ok(out) + } + + fn file_is_solid(&self, file: &FileHeader) -> bool { + if !self.solid || self.decoded_files == 0 { + return false; + } + // FHD_SOLID is not meaningful for unpack version < 20; rely on the + // archive-level MHD_SOLID flag in that case. + file.unp_ver < 20 || file.is_solid() + } + + fn codec_for(&mut self, file: &FileHeader) -> Result<&mut CodecState> { + self.read_control.check()?; + let reset = !self.file_is_solid(file) + || self + .codec + .as_ref() + .is_none_or(|codec| !codec.supports(file)); + if reset { + self.codec = Some(CodecState::with_allowance(file, &self.allowance)?); + } + // A missing codec always sets `reset`; successful construction then + // installs it before reaching this point. + let codec = self.codec.as_mut().expect("codec is initialized above"); + match codec { + CodecState::Unpack15(d) => d.read_control = self.read_control.clone(), + CodecState::Unpack20(d) => d.read_control = self.read_control.clone(), + CodecState::Unpack29(d) => d.read_control = self.read_control.clone(), + } + Ok(codec) + } +} + +impl<'a> DecoderSession<'a, Allowance> { + #[cfg(any(all(test, feature = "write"), feature = "recovery"))] + pub(super) fn new(solid: bool) -> Self { + Self::new_with_password(solid, None) + } + pub(super) fn new_with_password(solid: bool, password: Option<&'a [u8]>) -> Self { + Self::with_allowance(solid, password, &Allowance::default()) + } + #[cfg(any(all(test, feature = "write"), feature = "recovery"))] + pub(super) fn decode_file_data( + &mut self, + archive: &Archive, + file: &FileHeader, + ) -> Result> { + self.decode_file_owned(archive, file).map(Buffer::into_vec) + } +} + +impl FileHeader { + fn is_empty_compressed_payload(&self) -> bool { + !self.is_stored() && self.pack_size == 0 && self.unp_size == 0 + } +} + +/// Streams a multivolume archive set to caller-provided writers. +pub fn extract_volumes_to( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + open: F, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + if let Some(limit) = options.max_reader_workspace_bytes { + return extract_volumes_with_allowance(volumes, options, open, &Allowance::limited(limit)); + } + extract_volumes_with_allowance(volumes, options, open, &Allowance::default()) +} + +fn extract_volumes_with_allowance( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + mut open: F, + allowance: &B, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + options.check_cancelled()?; + if volumes.is_empty() { + return Err(Error::InvalidHeader("RAR 1.5 volume set is empty")); + } + + let password = options.password; + let mut budget = crate::output_limit::OutputBudget::new(options); + let mut split = SplitVolumeState::new(); + let mut session = DecoderSession::with_allowance( + volumes + .first() + .is_some_and(|archive| archive.main.is_solid()), + password, + allowance, + ); + session.read_control = budget.control.clone(); + for (volume_index, archive) in volumes.iter().enumerate() { + for (file_index, file) in archive.files().enumerate() { + options.check_cancelled()?; + match split.advance(file.is_split_before(), file.is_split_after()) { + SplitVolumeStep::Regular => { + let meta = file.metadata(); + if meta.is_directory { + options.check_cancelled()?; + let _ = open(&meta)?; + options.check_cancelled()?; + } else { + budget.check(file.unp_size, &file.name)?; + options.check_cancelled()?; + let mut writer = open(&meta)?; + options.check_cancelled()?; + budget.run(&file.name, &mut writer, |mut writer| { + if file.is_stored() { + file.write_stored_with_allowance( + archive, + password, + &mut writer, + allowance, + ) + .map_err(|error| file.entry_error("extracting", error))?; + } else { + session + .write_file_to(archive, file, &mut writer) + .map_err(|error| file.entry_error("extracting", error))?; + } + options.check_cancelled()?; + Ok(()) + })?; + } + } + SplitVolumeStep::Start => { + validate_split_fragment(file, password)?; + split.begin(PendingSplitRefs::with_allowance( + file, + volume_index, + file_index, + allowance, + )?); + } + SplitVolumeStep::Continue(current) => { + validate_split_continuation_refs(current, file, password)?; + current.append(file, volume_index, file_index)?; + } + SplitVolumeStep::Finish(mut completed) => { + validate_split_continuation_refs(&completed, file, password)?; + completed.append(file, volume_index, file_index)?; + completed.write_to( + volumes, + file, + password, + &mut session, + &mut budget, + &mut open, + )?; + } + SplitVolumeStep::MissingFirst => { + return Err(Error::InvalidHeader( + "RAR 1.5 split entry is missing its first part", + )); + } + SplitVolumeStep::Interrupted => { + return Err(Error::InvalidHeader( + "RAR 1.5 split entry is interrupted by a regular entry", + )); + } + } + } + } + + if split.is_pending() { + return Err(Error::InvalidHeader("RAR 1.5 split entry is incomplete")); + } + + options.check_cancelled()?; + Ok(()) +} + +fn validate_split_fragment(file: &FileHeader, password: Option<&[u8]>) -> Result<()> { + if file.is_directory() { + return Err(Error::InvalidHeader( + "RAR 1.5 split directory entry is invalid", + )); + } + if file.is_encrypted() { + crate::crypto::require_encryption()?; + } + if file.is_encrypted() && password.is_none() { + return Err(Error::NeedPassword); + } + Ok(()) +} + +fn validate_split_continuation_refs( + pending: &PendingSplitRefs, + file: &FileHeader, + password: Option<&[u8]>, +) -> Result<()> { + validate_split_fragment(file, password)?; + if file.name != pending.name { + return Err(Error::InvalidHeader("RAR 1.5 split entry name changed")); + } + if file.method != pending.method { + return Err(Error::InvalidHeader( + "RAR 1.5 split entry compression method changed", + )); + } + if file.unp_ver != pending.unp_ver { + return Err(Error::InvalidHeader( + "RAR 1.5 split entry unpack version changed", + )); + } + if file.is_encrypted() != pending.encrypted { + return Err(Error::InvalidHeader( + "RAR 1.5 split entry encryption flag changed", + )); + } + if pending.encrypted && pending.unp_ver >= 29 && file.salt != pending.salt { + return Err(Error::InvalidHeader("RAR 3.x split entry salt changed")); + } + Ok(()) +} + +struct PendingSplitRefs { + name: Vec, + fragments: Buffer<(usize, usize), B>, + file_time: u32, + mtime_refinement: Option, + attr: u32, + host_os: u8, + method: u8, + unp_ver: u8, + encrypted: bool, + salt: Option<[u8; 8]>, +} + +impl PendingSplitRefs { + fn with_allowance( + file: &FileHeader, + volume_index: usize, + file_index: usize, + allowance: &B, + ) -> Result { + Ok(Self { + name: file.name.clone(), + fragments: Buffer::copied(&[(volume_index, file_index)], allowance)?, + file_time: file.file_time, + mtime_refinement: file.mtime_refinement(), + attr: file.attr, + host_os: file.host_os, + method: file.method, + unp_ver: file.unp_ver, + encrypted: file.is_encrypted(), + salt: file.salt, + }) + } + + fn append(&mut self, _file: &FileHeader, volume_index: usize, file_index: usize) -> Result<()> { + self.fragments.try_push((volume_index, file_index))?; + Ok(()) + } + + fn write_to( + self, + volumes: &[Archive], + final_file: &FileHeader, + password: Option<&[u8]>, + session: &mut DecoderSession<'_, B>, + budget: &mut crate::output_limit::OutputBudget, + open: &mut F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + budget.check(final_file.unp_size, &final_file.name)?; + let meta = ExtractedEntryMeta { + name_is_unicode: final_file.unicode_name.is_some(), + name: self.name.clone(), + file_time: self.file_time, + mtime_refinement: self.mtime_refinement, + attr: self.attr, + host_os: self.host_os, + is_directory: false, + }; + let mut writer = open(&meta)?; + budget.run(&final_file.name, &mut writer, |mut writer| { + let mut reader = self.fragment_reader_with_allowance( + volumes, + password, + &self.fragments.allowance(), + )?; + + if final_file.is_stored() { + let expected_len = usize::try_from(final_file.unp_size).map_err(|_| { + Error::InvalidHeader("RAR 1.5 split unpacked size overflows usize") + })?; + let actual_len = self.packed_size(volumes)?; + let expected_packed_len = if self.encrypted && self.unp_ver >= 20 { + expected_len.checked_add(15).map(|len| len & !15).ok_or( + Error::InvalidHeader("RAR 2.x encrypted split stored size overflows"), + )? + } else { + expected_len + }; + if actual_len != expected_packed_len { + return Err(Error::InvalidHeader( + "RAR 1.5 split stored file has wrong reassembled size", + )); + } + + let mut crc = Crc32::new(); + let mut crc_writer = CrcWriter { + inner: &mut writer, + crc: &mut crc, + }; + let copied = std::io::copy(&mut reader.take(expected_len as u64), &mut crc_writer)?; + if copied != expected_len as u64 { + return Err(Error::InvalidHeader( + "RAR 1.5 split stored file ended before unpacked size", + )); + } + let actual = crc.finish(); + final_file + .crc_result(actual, password) + .map_err(|error| final_file.entry_error("extracting", error)) + } else { + session + .write_split_to(&mut reader, final_file, &mut writer) + .map_err(|error| final_file.entry_error("extracting", error)) + } + }) + } + + fn packed_size(&self, volumes: &[Archive]) -> Result { + // Fragments index the same immutable volume slice enumerated by + // extract_volumes_to; their member indices cannot disappear. + self.fragments + .iter() + .try_fold(0usize, |total, &(volume_index, file_index)| { + let archive = &volumes[volume_index]; + let file = archive + .files() + .nth(file_index) + .expect("split fragment index comes from archive enumeration"); + total + .checked_add(usize::try_from(file.pack_size).map_err(|_| { + Error::InvalidHeader("RAR 1.5 split packed size overflows usize") + })?) + .ok_or(Error::InvalidHeader( + "RAR 1.5 split packed size overflows usize", + )) + }) + } + + fn fragment_reader_with_allowance<'a>( + &self, + volumes: &'a [Archive], + password: Option<&[u8]>, + allowance: &B, + ) -> Result, B>, B>> { + let mut readers = Buffer::with_capacity(self.fragments.len(), allowance)?; + for &(volume_index, file_index) in &self.fragments { + let archive = &volumes[volume_index]; + let file = archive + .files() + .nth(file_index) + .expect("split fragment index comes from archive enumeration"); + readers.push_admitted(archive.range_reader(file.packed_range.clone())?); + } + let reader = ChainedReader::with_readers(readers); + if !self.encrypted { + return Ok(PackedReader::Plain(reader)); + } + + let password = password.expect("encrypted split fragments were admitted with a password"); + Ok(PackedReader::Encrypted(Boxed::try_new( + || { + DecryptingReader::with_allowance( + reader, + self.unp_ver, + password, + self.salt, + allowance, + ) + }, + allowance, + )?)) + } +} + +#[cfg(all(test, feature = "write"))] +impl PendingSplitRefs { + fn new(file: &FileHeader, volume_index: usize, file_index: usize) -> Self { + Self::with_allowance(file, volume_index, file_index, &Allowance::default()) + .expect("unlimited split bookkeeping") + } + #[cfg(all(test, feature = "write"))] + fn fragment_reader<'a>( + &self, + volumes: &'a [Archive], + password: Option<&[u8]>, + ) -> Result>>> { + self.fragment_reader_with_allowance(volumes, password, &Allowance::default()) + } +} + +#[cfg(feature = "encryption")] +enum SplitCipher { + Rar15(Rar15Cipher), + Rar20(Boxed), + Rar30(Boxed), +} + +#[cfg(feature = "encryption")] +impl SplitCipher { + fn with_allowance( + unp_ver: u8, + password: &[u8], + salt: Option<[u8; 8]>, + allowance: &B, + ) -> Result { + if unp_ver == 15 { + return Ok(Self::Rar15(Rar15Cipher::new(password))); + } + if unp_ver == 20 || unp_ver == 26 { + return Ok(Self::Rar20(Boxed::try_new( + || Ok::<_, crate::codec::Error>(Rar20Cipher::new(password)), + allowance, + )?)); + } + if unp_ver >= 29 { + return Ok(Self::Rar30(Boxed::try_new( + || Rar30Cipher::new(password, salt).map_err(super::map_rar30_crypto_error), + allowance, + )?)); + } + Err(Error::UnsupportedEncryption { + family: "RAR 1.5-4.x split volume", + unpack_version: unp_ver, + }) + } +} + +pub(super) enum PackedReader { + Plain(R), + Encrypted(Boxed, B>), +} +impl Read for PackedReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self { + Self::Plain(reader) => reader.read(out), + Self::Encrypted(reader) => reader.read(out), + } + } +} +#[cfg(not(feature = "encryption"))] +pub(super) type DecryptingReader = crate::crypto::unavailable::Reader; + +#[cfg(feature = "encryption")] +pub(super) struct DecryptingReader { + inner: R, + cipher: SplitCipher, + encrypted_block: Buffer, + decrypted: Buffer, + read_buffer: Option>, + decrypted_pos: usize, + eof: bool, +} + +#[cfg(feature = "encryption")] +impl DecryptingReader { + pub(super) fn with_allowance( + inner: R, + unp_ver: u8, + password: &[u8], + salt: Option<[u8; 8]>, + allowance: &B, + ) -> Result { + let cipher = SplitCipher::with_allowance(unp_ver, password, salt, allowance)?; + let read_buffer = matches!(cipher, SplitCipher::Rar15(_)) + .then(|| Buffer::filled(64 * 1024, 0, allowance)) + .transpose()?; + Ok(Self { + inner, + cipher, + encrypted_block: Buffer::new(allowance), + decrypted: Buffer::new(allowance), + read_buffer, + decrypted_pos: 0, + eof: false, + }) + } + + fn fill_decrypted(&mut self) -> std::io::Result<()> { + if self.decrypted_pos < self.decrypted.len() || self.eof { + return Ok(()); + } + self.decrypted.clear(); + self.decrypted_pos = 0; + + match &mut self.cipher { + SplitCipher::Rar15(cipher) => { + let read_buffer = self + .read_buffer + .as_mut() + .expect("RAR 1.5 decrypting reader has a reusable buffer"); + let count = self.inner.read(read_buffer)?; + if count == 0 { + self.eof = true; + return Ok(()); + } + self.decrypted + .extend_from_slice(&read_buffer[..count]) + .map_err(Into::::into) + .map_err(Error::from) + .map_err(std::io::Error::other)?; + cipher.crypt_in_place(&mut self.decrypted); + } + SplitCipher::Rar20(cipher) => Self::fill_block_decrypted( + &mut self.inner, + &mut self.encrypted_block, + &mut self.decrypted, + &mut self.eof, + |block| cipher.decrypt_block(block), + )?, + SplitCipher::Rar30(cipher) => Self::fill_block_decrypted( + &mut self.inner, + &mut self.encrypted_block, + &mut self.decrypted, + &mut self.eof, + |block| cipher.decrypt_block(block), + )?, + } + Ok(()) + } + + fn fill_block_decrypted( + inner: &mut R, + encrypted_block: &mut Buffer, + decrypted: &mut Buffer, + eof: &mut bool, + mut decrypt_block: impl FnMut(&mut [u8; 16]), + ) -> std::io::Result<()> { + // fill_decrypted returns before calling us once EOF is known. The only + // EOF transition below breaks the loop immediately. + while encrypted_block.len() < 16 { + let mut buf = [0u8; 64 * 1024]; + let count = inner.read(&mut buf)?; + if count == 0 { + *eof = true; + break; + } + encrypted_block + .extend_from_slice(&buf[..count]) + .map_err(Into::::into) + .map_err(Error::from) + .map_err(std::io::Error::other)?; + } + + let full_len = (encrypted_block.len() / 16) * 16; + if full_len != 0 { + let tail = Buffer::copied(&encrypted_block[full_len..], &encrypted_block.allowance()) + .map_err(Error::from) + .map_err(std::io::Error::other)?; + let mut data = std::mem::replace(encrypted_block, tail); + data.truncate(full_len); + for block in data.as_chunks_mut::<16>().0.iter_mut() { + decrypt_block(block); + } + *decrypted = data; + } else if !encrypted_block.is_empty() { + // With no full block, the loop could only have ended at EOF. + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "RAR encrypted payload is not block aligned", + )); + } + Ok(()) + } +} + +#[cfg(feature = "encryption")] +impl Read for DecryptingReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if out.is_empty() { + return Ok(0); + } + self.fill_decrypted()?; + if self.decrypted_pos == self.decrypted.len() { + return Ok(0); + } + let count = out.len().min(self.decrypted.len() - self.decrypted_pos); + out[..count] + .copy_from_slice(&self.decrypted[self.decrypted_pos..self.decrypted_pos + count]); + self.decrypted_pos += count; + Ok(count) + } +} + +#[cfg(all(test, feature = "write"))] +#[cfg(feature = "encryption")] +impl SplitCipher { + fn new(unp_ver: u8, password: &[u8], salt: Option<[u8; 8]>) -> Result { + Self::with_allowance(unp_ver, password, salt, &Allowance::default()) + } +} +#[cfg(all(test, feature = "write"))] +#[cfg(feature = "encryption")] +impl DecryptingReader { + pub(super) fn new( + inner: R, + unp_ver: u8, + password: &[u8], + salt: Option<[u8; 8]>, + ) -> Result { + Self::with_allowance(inner, unp_ver, password, salt, &Allowance::default()) + } +} +#[cfg(all(test, feature = "write"))] +mod tests { + #[test] + fn reader_workspace_split_descriptor_growth_is_admitted_before_mutation() { + let first = file(b"a.txt", FHD_SPLIT_AFTER); + let bytes = std::mem::size_of::<(usize, usize)>() as u64; + let quota = Allowance::limited(bytes); + let mut pending = PendingSplitRefs::with_allowance(&first, 0, 0, "a).unwrap(); + assert_eq!(quota.used(), bytes); + let error = pending.append(&first, 1, 0).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(&pending.fragments[..], &[(0, 0)]); + assert_eq!(quota.used(), bytes); + drop(pending); + assert_eq!(quota.used(), 0); + } + + #[test] + fn reader_session_workspace_refusals_charge_every_legacy_decoder_and_encrypted_payload() { + use crate::codec::workspace::RefusingBudget; + let plain = b"abcabcabc"; + for version in [15, 20, 29] { + let packed = match version { + 15 => crate::codec::rar13::unpack15_encode(plain).unwrap(), + 20 => crate::codec::rar20::unpack20_encode_literals(plain).unwrap(), + _ => crate::codec::rar29::unpack29_encode_literals(plain).unwrap(), + }; + for encrypted in crate::test_features::encryption_cases() { + let payload = packed.clone(); + #[cfg(feature = "encryption")] + let mut payload = payload; + if encrypted { + #[cfg(feature = "encryption")] + { + match version { + 15 => Rar15Cipher::new(b"pw").crypt_in_place(&mut payload), + 20 => { + payload.resize(payload.len().div_ceil(16) * 16, 0); + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut payload) + .unwrap(); + } + _ => { + payload.resize(payload.len().div_ceil(16) * 16, 0); + Rar30Cipher::new(b"pw", None) + .unwrap() + .encrypt_in_place(&mut payload) + .unwrap(); + } + } + } + } + let mut entry = file(b"charged.bin", if encrypted { FHD_PASSWORD } else { 0 }); + entry.method = 0x33; + entry.unp_ver = version; + entry.unp_size = plain.len() as u64; + entry.pack_size = payload.len() as u64; + entry.packed_range = 0..payload.len(); + entry.file_crc = crate::crc32::crc32(plain); + let archive = archive_with_source(vec![Block::File(entry.clone())], payload); + let run = |budget: &RefusingBudget| -> Result<()> { + let mut session = DecoderSession::with_allowance(false, Some(b"pw"), budget); + let mut out = Buffer::new(budget); + session.write_file_to(&archive, &entry, &mut out)?; + assert_eq!(&out[..], plain); + let decoded = session.decode_file_owned(&archive, &entry)?; + assert_eq!(&decoded[..], plain); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + let error = run(&budget).unwrap_err(); + assert_eq!( + error.kind(), + crate::ErrorKind::Cancelled, + "version {version}, encrypted {encrypted}, allocation {index}: {error}" + ); + assert_eq!(budget.used(), 0); + } + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn reader_session_workspace_refusal_keeps_its_resource_kind_for_encrypted_members() { + let mut entry = file(b"quota.bin", FHD_PASSWORD); + entry.method = 0x33; + entry.unp_ver = 29; + entry.unp_size = 1; + entry.pack_size = 16; + entry.packed_range = 0..16; + let archive = archive_with_source(vec![Block::File(entry.clone())], vec![0; 16]); + let budget = Allowance::limited(1); + let mut session = DecoderSession::with_allowance(false, Some(b"pw"), &budget); + let error = session + .write_file_to(&archive, &entry, &mut std::io::sink()) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(budget.used(), 0); + let error = entry + .packed_reader_with_allowance(&archive, Some(b"pw"), &budget) + .err() + .unwrap(); + assert_eq!( + entry.map_encrypted_payload_error(Some(b"pw"), error).kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(budget.used(), 0); + } + + #[test] + #[cfg(feature = "encryption")] + fn reader_workspace_refusals_release_legacy_cipher_and_decryption_buffers() { + use crate::codec::workspace::RefusingBudget; + for version in [15, 20, 29] { + let plain = *b"0123456789abcdefRAR AES CBC data"; + let mut encrypted = plain; + match version { + 15 => Rar15Cipher::new(b"pw").crypt_in_place(&mut encrypted), + 20 => Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(), + _ => Rar30Cipher::new(b"pw", None) + .unwrap() + .encrypt_in_place(&mut encrypted) + .unwrap(), + } + let run = |budget: &RefusingBudget| -> Result<()> { + let input = ChunkedReader::new(Cursor::new(&encrypted), 7); + let mut reader = + DecryptingReader::with_allowance(input, version, b"pw", None, budget)?; + let mut out = Buffer::new(budget); + out.read_to_end(&mut reader)?; + assert_eq!(&out[..], &plain); + Ok(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + let error = run(&budget).unwrap_err(); + assert_eq!( + error.kind(), + crate::ErrorKind::Cancelled, + "version {version}, allocation {index}: {error}" + ); + assert_eq!(budget.used(), 0); + } + } + } + + use super::super::{ + ArchiveSource, Block, BlockHeader, MainHeader, FHD_DIRECTORY_MASK, FHD_PASSWORD, + FHD_SPLIT_AFTER, FHD_SPLIT_BEFORE, + }; + use super::*; + #[cfg(feature = "encryption")] + use std::io::Cursor; + use std::sync::Arc; + + fn block(flags: u16) -> BlockHeader { + BlockHeader { + head_crc: 0, + head_type: 0x74, + flags, + head_size: 0, + add_size: Some(0), + offset: 0, + } + } + + fn file(name: &[u8], flags: u16) -> FileHeader { + FileHeader { + block: block(flags), + pack_size: 0, + unp_size: 0, + host_os: 2, + file_crc: 0, + file_time: 0, + unp_ver: 29, + method: 0x30, + name: name.to_vec(), + unicode_name: None, + attr: 0x20, + salt: None, + file_comment: Vec::new(), + ext_time: Vec::new(), + packed_range: 0..0, + } + } + + #[cfg(feature = "encryption")] + struct ChunkedReader { + inner: R, + chunk: usize, + } + + #[cfg(feature = "encryption")] + impl ChunkedReader { + fn new(inner: R, chunk: usize) -> Self { + Self { inner, chunk } + } + } + + #[cfg(feature = "encryption")] + impl Read for ChunkedReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let take = out.len().min(self.chunk); + self.inner.read(&mut out[..take]) + } + } + + fn read_in_small_chunks(mut reader: impl Read) -> Vec { + let mut out = Vec::new(); + let mut buf = [0u8; 7]; + loop { + let count = reader.read(&mut buf).unwrap(); + if count == 0 { + break; + } + out.extend_from_slice(&buf[..count]); + } + out + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_streams_rar15_payload() { + let plain = b"RAR 1.5 encrypted payload read in pieces"; + let mut encrypted = plain.to_vec(); + Rar15Cipher::new(b"pw").crypt_in_place(&mut encrypted); + let mut reader = DecryptingReader::new(Cursor::new(encrypted), 15, b"pw", None).unwrap(); + let mut out = Vec::new(); + let mut buf = [0u8; 3]; + + loop { + let count = reader.read(&mut buf).unwrap(); + if count == 0 { + break; + } + out.extend_from_slice(&buf[..count]); + } + + assert_eq!(out, plain); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_streams_rar20_blocks_from_short_inner_reads() { + let plain = *b"0123456789abcdefRAR2 block two!!"; + let mut encrypted = plain; + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(); + for version in [20, 26] { + let reader = DecryptingReader::new( + ChunkedReader::new(Cursor::new(encrypted), 5), + version, + b"pw", + None, + ) + .unwrap(); + assert_eq!(read_in_small_chunks(reader), plain); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_streams_rar30_blocks_from_short_inner_reads() { + let salt = Some([7u8; 8]); + let plain = *b"0123456789abcdefRAR3 block two!!"; + let mut encrypted = plain; + Rar30Cipher::new(b"pw", salt) + .unwrap() + .encrypt_in_place(&mut encrypted) + .unwrap(); + let reader = DecryptingReader::new( + ChunkedReader::new(Cursor::new(encrypted), 5), + 29, + b"pw", + salt, + ) + .unwrap(); + let out = read_in_small_chunks(reader); + + assert_eq!(out, plain); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_preserves_partial_blocks_across_source_errors() { + let plain = *b"0123456789abcdefRAR3 block two!!"; + let salt = Some([7; 8]); + for version in [15, 20, 29] { + let mut encrypted = plain; + match version { + 15 => Rar15Cipher::new(b"pw").crypt_in_place(&mut encrypted), + 20 => Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(), + _ => Rar30Cipher::new(b"pw", salt) + .unwrap() + .encrypt_in_place(&mut encrypted) + .unwrap(), + } + for kind in [ + std::io::ErrorKind::Interrupted, + std::io::ErrorKind::PermissionDenied, + ] { + for fail_at in [0, 5, 16, 21] { + let inner = + crate::read_errors::ErrorOnceReader::new(encrypted.to_vec(), fail_at, kind); + let mut reader = DecryptingReader::new(inner, version, b"pw", salt).unwrap(); + let mut out = Vec::new(); + let result = reader.read_to_end(&mut out); + if kind == std::io::ErrorKind::Interrupted { + result.unwrap(); + } else { + let error = result.unwrap_err(); + assert_eq!(error.kind(), kind); + assert_eq!(error.to_string(), "source read failed"); + let emitted = if version == 15 { + fail_at as usize + } else { + fail_at as usize / 16 * 16 + }; + assert_eq!(out, plain[..emitted]); + reader.read_to_end(&mut out).unwrap(); + } + assert_eq!(out, plain, "version {version}, error {kind:?} at {fail_at}"); + } + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_preserves_complete_blocks_and_caches_eof() { + struct CountingReader { + inner: Cursor>, + chunk: usize, + reads: usize, + } + impl Read for CountingReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + self.reads += 1; + let count = out.len().min(self.chunk); + self.inner.read(&mut out[..count]) + } + } + + let plain = *b"0123456789abcdefRAR3 block two!!"; + let salt = Some([7; 8]); + for version in [15, 20, 29] { + let mut encrypted = plain; + match version { + 15 => Rar15Cipher::new(b"pw").crypt_in_place(&mut encrypted), + 20 => Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(), + _ => Rar30Cipher::new(b"pw", salt) + .unwrap() + .encrypt_in_place(&mut encrypted) + .unwrap(), + } + for chunk in [5, 64 * 1024] { + for length in 0..=encrypted.len() { + let inner = CountingReader { + inner: Cursor::new(encrypted[..length].to_vec()), + chunk, + reads: 0, + }; + let mut reader = DecryptingReader::new(inner, version, b"pw", salt).unwrap(); + assert_eq!(reader.read(&mut []).unwrap(), 0); + assert_eq!(reader.inner.reads, 0); + let mut out = Vec::new(); + let result = loop { + let mut buf = [0; 3]; + match reader.read(&mut buf) { + Ok(0) => break Ok(()), + Ok(count) => out.extend_from_slice(&buf[..count]), + Err(error) => break Err(error), + } + }; + let complete_length = if version == 15 { + length + } else { + length / 16 * 16 + }; + assert_eq!( + out, + plain[..complete_length], + "version {version}, length {length}, chunk {chunk}" + ); + if version != 15 && length % 16 != 0 { + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + } else { + result.unwrap(); + let reads = reader.inner.reads; + assert_eq!(reader.read(&mut [0]).unwrap(), 0); + assert_eq!(reader.read(&mut [0]).unwrap(), 0); + assert_eq!(reader.inner.reads, reads, "EOF must not reread the source"); + } + } + } + } + } + + #[test] + fn validate_split_fragment_rejects_directories_and_demands_password_for_encrypted() { + let dir = file(b"d", FHD_DIRECTORY_MASK | FHD_SPLIT_AFTER); + assert!(matches!( + validate_split_fragment(&dir, None), + Err(Error::InvalidHeader(_)) + )); + + let encrypted = file(b"a", FHD_PASSWORD | FHD_SPLIT_AFTER); + #[cfg(feature = "encryption")] + { + assert!(matches!( + validate_split_fragment(&encrypted, None), + Err(Error::NeedPassword) + )); + validate_split_fragment(&encrypted, Some(b"pw")).unwrap(); + } + #[cfg(not(feature = "encryption"))] + for password in [None, Some(b"pw".as_slice())] { + assert!(matches!( + validate_split_fragment(&encrypted, password), + Err(Error::FeatureDisabled { + feature: "encryption" + }) + )); + } + + let plain = file(b"a", FHD_SPLIT_AFTER); + validate_split_fragment(&plain, None).unwrap(); + } + + #[test] + fn validate_split_continuation_refs_rejects_property_drift_between_fragments() { + let first = file(b"a.txt", FHD_SPLIT_AFTER); + let pending = PendingSplitRefs::new(&first, 0, 0); + + let renamed = file(b"b.txt", FHD_SPLIT_BEFORE); + assert!(matches!( + validate_split_continuation_refs(&pending, &renamed, None), + Err(Error::InvalidHeader(_)) + )); + + let mut new_method = file(b"a.txt", FHD_SPLIT_BEFORE); + new_method.method = 0x35; + assert!(matches!( + validate_split_continuation_refs(&pending, &new_method, None), + Err(Error::InvalidHeader(_)) + )); + + let mut new_version = file(b"a.txt", FHD_SPLIT_BEFORE); + new_version.unp_ver = 20; + assert!(matches!( + validate_split_continuation_refs(&pending, &new_version, None), + Err(Error::InvalidHeader(_)) + )); + + let new_encryption = file(b"a.txt", FHD_PASSWORD | FHD_SPLIT_BEFORE); + #[cfg(feature = "encryption")] + assert!(matches!( + validate_split_continuation_refs(&pending, &new_encryption, Some(b"pw")), + Err(Error::InvalidHeader(_)) + )); + #[cfg(not(feature = "encryption"))] + assert!(matches!( + validate_split_continuation_refs(&pending, &new_encryption, Some(b"pw")), + Err(Error::FeatureDisabled { + feature: "encryption" + }) + )); + + let same = file(b"a.txt", FHD_SPLIT_BEFORE); + validate_split_continuation_refs(&pending, &same, None).unwrap(); + } + + #[test] + #[cfg(feature = "encryption")] + fn validate_split_continuation_refs_rejects_salt_drift_for_rar3_encrypted_entries() { + let mut first = file(b"a.txt", FHD_PASSWORD | FHD_SPLIT_AFTER); + first.salt = Some([1u8; 8]); + let pending = PendingSplitRefs::new(&first, 0, 0); + + let mut other_salt = file(b"a.txt", FHD_PASSWORD | FHD_SPLIT_BEFORE); + other_salt.salt = Some([2u8; 8]); + assert!(matches!( + validate_split_continuation_refs(&pending, &other_salt, Some(b"pw")), + Err(Error::InvalidHeader(_)) + )); + + let mut same_salt = file(b"a.txt", FHD_PASSWORD | FHD_SPLIT_BEFORE); + same_salt.salt = Some([1u8; 8]); + validate_split_continuation_refs(&pending, &same_salt, Some(b"pw")).unwrap(); + } + + fn empty_archive() -> Archive { + Archive { + sfx_offset: 0, + main: MainHeader { + head_crc: 0, + flags: 0, + head_size: 0, + reserved1: 0, + reserved2: 0, + encrypt_version: None, + }, + blocks: Vec::new(), + source: ArchiveSource::Memory(Arc::from(Vec::new().into_boxed_slice())), + } + } + + fn archive_with(blocks: Vec) -> Archive { + let mut archive = empty_archive(); + archive.blocks = blocks; + archive + } + + fn archive_with_source(blocks: Vec, source: Vec) -> Archive { + Archive { + sfx_offset: 0, + main: MainHeader { + head_crc: 0, + flags: 0, + head_size: 0, + reserved1: 0, + reserved2: 0, + encrypt_version: None, + }, + blocks, + source: ArchiveSource::Memory(Arc::from(source.into_boxed_slice())), + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_split_fragment_reader_decrypts_after_chaining_fragments() { + let plain = *b"0123456789abcdefRAR2 block two!!"; + let mut encrypted = plain; + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(); + let split = 7; + + let mut first = file(b"a.txt", FHD_PASSWORD | FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.pack_size = split as u64; + first.packed_range = 0..split; + + let mut second = file(b"a.txt", FHD_PASSWORD | FHD_SPLIT_BEFORE); + second.unp_ver = 20; + second.pack_size = (encrypted.len() - split) as u64; + second.packed_range = 0..(encrypted.len() - split); + + let mut pending = PendingSplitRefs::new(&first, 0, 0); + pending.append(&second, 1, 0).unwrap(); + let volumes = vec![ + archive_with_source(vec![Block::File(first)], encrypted[..split].to_vec()), + archive_with_source(vec![Block::File(second)], encrypted[split..].to_vec()), + ]; + + let reader = pending.fragment_reader(&volumes, Some(b"pw")).unwrap(); + let out = read_in_small_chunks(reader); + + assert_eq!(out, plain); + } + + fn never_open(_meta: &ExtractedEntryMeta) -> Result> { + panic!("open should not be invoked for this test"); + } + + #[test] + fn extract_volumes_to_rejects_split_state_violations() { + let empty: Vec = Vec::new(); + assert!(matches!( + extract_volumes_to(&empty, crate::ArchiveReadOptions::default(), never_open), + Err(Error::InvalidHeader(_)) + )); + + let only_continuation = vec![archive_with(vec![Block::File(file( + b"a.txt", + FHD_SPLIT_BEFORE, + ))])]; + assert!(matches!( + extract_volumes_to( + &only_continuation, + crate::ArchiveReadOptions::default(), + never_open, + ), + Err(Error::InvalidHeader(_)) + )); + + let interrupted = vec![archive_with(vec![ + Block::File(file(b"a.txt", FHD_SPLIT_AFTER)), + Block::File(file(b"unrelated", 0)), + ])]; + assert!(matches!( + extract_volumes_to( + &interrupted, + crate::ArchiveReadOptions::default(), + never_open, + ), + Err(Error::InvalidHeader(_)) + )); + + let incomplete = vec![archive_with(vec![Block::File(file( + b"a.txt", + FHD_SPLIT_AFTER, + ))])]; + assert!(matches!( + extract_volumes_to( + &incomplete, + crate::ArchiveReadOptions::default(), + never_open, + ), + Err(Error::InvalidHeader(_)) + )); + } + + #[test] + fn codec_state_new_for_chooses_codec_by_unpack_version() { + let mut f = file(b"a", 0); + f.unp_ver = 15; + assert!(matches!( + CodecState::new_for(&f).unwrap(), + CodecState::Unpack15(_) + )); + f.unp_ver = 20; + assert!(matches!( + CodecState::new_for(&f).unwrap(), + CodecState::Unpack20(_) + )); + f.unp_ver = 26; + assert!(matches!( + CodecState::new_for(&f).unwrap(), + CodecState::Unpack20(_) + )); + f.unp_ver = 29; + assert!(matches!( + CodecState::new_for(&f).unwrap(), + CodecState::Unpack29(_) + )); + f.unp_ver = 36; + assert!(matches!( + CodecState::new_for(&f).unwrap(), + CodecState::Unpack29(_) + )); + f.unp_ver = 14; + f.method = 0x35; + assert!(matches!( + CodecState::new_for(&f), + Err(Error::UnsupportedCompression { + unpack_version: 14, + method: 0x35, + .. + }) + )); + } + + #[test] + fn codec_state_supports_matches_codec_to_file_version() { + let mut f = file(b"a", 0); + + f.unp_ver = 15; + let unpack15 = CodecState::new_for(&f).unwrap(); + assert!(unpack15.supports(&f)); + f.unp_ver = 20; + assert!(!unpack15.supports(&f)); + f.unp_ver = 29; + assert!(!unpack15.supports(&f)); + + f.unp_ver = 20; + let unpack20 = CodecState::new_for(&f).unwrap(); + assert!(unpack20.supports(&f)); + f.unp_ver = 26; + assert!(unpack20.supports(&f)); + f.unp_ver = 15; + assert!(!unpack20.supports(&f)); + f.unp_ver = 29; + assert!(!unpack20.supports(&f)); + + f.unp_ver = 29; + let unpack29 = CodecState::new_for(&f).unwrap(); + assert!(unpack29.supports(&f)); + f.unp_ver = 36; + assert!(unpack29.supports(&f)); + f.unp_ver = 20; + assert!(!unpack29.supports(&f)); + } + + #[test] + fn decoder_session_empty_compressed_payload_does_not_reset_solid_codec() { + let mut session = DecoderSession::new(true); + let mut first = file(b"first.txt", 0); + first.unp_ver = 29; + first.method = 0x35; + session.codec = Some(CodecState::new_for(&first).unwrap()); + session.decoded_files = 4; + + let mut empty = file(b"empty.txt", super::super::FHD_SOLID); + empty.unp_ver = 20; + empty.method = 0x33; + empty.file_crc = 0; + let archive = Archive { + sfx_offset: 0, + main: MainHeader { + head_crc: 0, + flags: super::super::MHD_SOLID, + head_size: 13, + reserved1: 0, + reserved2: 0, + encrypt_version: None, + }, + blocks: vec![Block::File(empty.clone())], + source: ArchiveSource::Memory(Arc::from([])), + }; + + let mut out = Vec::new(); + session.write_file_to(&archive, &empty, &mut out).unwrap(); + + assert!(out.is_empty()); + assert_eq!(session.decoded_files, 4); + assert!(matches!(session.codec, Some(CodecState::Unpack29(_)))); + } + + #[test] + fn empty_compressed_data_decode_preserves_solid_state_and_checks_crc() { + let mut session = DecoderSession::new(true); + let mut entry = file(b"empty.txt", super::super::FHD_SOLID); + entry.method = 0x35; + session.codec = Some(CodecState::new_for(&entry).unwrap()); + session.decoded_files = 2; + let archive = archive_with(vec![Block::File(entry.clone())]); + + assert_eq!(session.decode_file_data(&archive, &entry).unwrap(), b""); + assert_eq!(session.decoded_files, 2); + assert!(matches!(session.codec, Some(CodecState::Unpack29(_)))); + + entry.file_crc = 1; + assert!(matches!( + session.decode_file_data(&archive, &entry), + Err(Error::Crc32Mismatch { + expected: 1, + actual: 0 + }) + )); + assert_eq!(session.decoded_files, 2); + } + + #[test] + #[cfg(feature = "encryption")] + fn split_cipher_new_rejects_unsupported_unpack_version() { + for ver in [14u8, 16, 19, 25, 27, 28] { + assert!( + matches!( + SplitCipher::new(ver, b"pw", None), + Err(Error::UnsupportedEncryption { unpack_version, .. }) if unpack_version == ver + ), + "unp_ver {ver} should be rejected" + ); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_rejects_non_utf8_rar30_password() { + let result = DecryptingReader::new(Cursor::new(Vec::::new()), 29, &[0xff], None); + assert!(matches!( + result, + Err(Error::Rar30Crypto(Rar30Error::NonUtf8Password)) + )); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_new_rejects_unsupported_unpack_version() { + let result = DecryptingReader::new(Cursor::new(Vec::::new()), 25, b"pw", None); + assert!(matches!( + result, + Err(Error::UnsupportedEncryption { + unpack_version: 25, + .. + }) + )); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_rejects_non_block_aligned_rar20_payload() { + let mut payload = vec![0u8; 23]; + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut payload[..16]) + .unwrap(); + let mut reader = DecryptingReader::new(Cursor::new(payload), 20, b"pw", None).unwrap(); + let mut buf = [0u8; 64]; + let err = loop { + match reader.read(&mut buf) { + Ok(0) => panic!("expected non-block-aligned data error"), + Ok(_) => continue, + Err(err) => break err, + } + }; + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_reports_rar30_body_crypto_errors_without_header_context() { + let error = super::map_rar30_crypto_error(Rar30Error::UnalignedInput); + assert!(matches!( + error, + Error::Rar30Crypto(Rar30Error::UnalignedInput) + )); + assert_eq!(error.to_string(), "RAR 3.x AES input is not block aligned"); + + let mapped = + file(b"encrypted.bin", FHD_PASSWORD).map_encrypted_payload_error(Some(b"pw"), error); + assert_eq!(mapped, Error::WrongPasswordOrCorruptData); + } + + #[test] + fn pending_split_refs_fragment_reader_chains_unencrypted_volumes() { + let plain: &[u8] = b"hello, this string is split across two volumes!"; + let split = 11usize; + + let mut first = file(b"a.txt", FHD_SPLIT_AFTER); + first.pack_size = split as u64; + first.packed_range = 0..split; + let mut second = file(b"a.txt", FHD_SPLIT_BEFORE); + second.pack_size = (plain.len() - split) as u64; + second.packed_range = 0..(plain.len() - split); + + let mut pending = PendingSplitRefs::new(&first, 0, 0); + pending.append(&second, 1, 0).unwrap(); + let volumes = vec![ + archive_with_source(vec![Block::File(first)], plain[..split].to_vec()), + archive_with_source(vec![Block::File(second)], plain[split..].to_vec()), + ]; + + let reader = pending.fragment_reader(&volumes, None).unwrap(); + let out = read_in_small_chunks(reader); + assert_eq!(out, plain); + } + + #[test] + fn pending_split_refs_packed_size_sums_fragment_pack_sizes() { + let mut first = file(b"a.txt", FHD_SPLIT_AFTER); + first.pack_size = 7; + let mut second = file(b"a.txt", FHD_SPLIT_BEFORE); + second.pack_size = 5; + + let mut pending = PendingSplitRefs::new(&first, 0, 0); + pending.append(&second, 1, 0).unwrap(); + let volumes = vec![ + archive_with(vec![Block::File(first)]), + archive_with(vec![Block::File(second)]), + ]; + assert_eq!(pending.packed_size(&volumes).unwrap(), 12); + } + + #[test] + fn pending_split_refs_rejects_advertised_packed_size_overflow() { + let mut first = file(b"a.txt", FHD_SPLIT_AFTER); + first.pack_size = u64::MAX; + let mut second = file(b"a.txt", FHD_SPLIT_BEFORE); + second.pack_size = 1; + + let mut pending = PendingSplitRefs::new(&first, 0, 0); + pending.append(&second, 1, 0).unwrap(); + let volumes = vec![ + archive_with(vec![Block::File(first)]), + archive_with(vec![Block::File(second)]), + ]; + assert_eq!( + pending.packed_size(&volumes).unwrap_err(), + Error::InvalidHeader("RAR 1.5 split packed size overflows usize") + ); + } + + #[derive(Default, Clone)] + struct Capture { + bytes: std::rc::Rc>>, + opened: std::rc::Rc>>, + } + + struct CaptureWriter(std::rc::Rc>>); + + impl Write for CaptureWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl Capture { + fn opener(&self) -> impl FnMut(&ExtractedEntryMeta) -> Result> + '_ { + let bytes = self.bytes.clone(); + let opened = self.opened.clone(); + move |meta| { + opened.borrow_mut().push(meta.clone()); + Ok(Box::new(CaptureWriter(bytes.clone()))) + } + } + } + + #[test] + fn parallel_extraction_keeps_directory_callbacks_and_split_fallback() { + let directory = file(b"dir", FHD_DIRECTORY_MASK); + let archive = archive_with(vec![Block::File(directory.clone())]); + let capture = Capture::default(); + archive + .extract_to_parallel_buffered(crate::ArchiveReadOptions::new(), capture.opener()) + .unwrap(); + assert!(capture.opened.borrow()[0].is_directory); + let mut bytes = vec![]; + directory.write_to(&archive, None, &mut bytes).unwrap(); + assert!(bytes.is_empty()); + for flag in [FHD_SPLIT_BEFORE, FHD_SPLIT_AFTER] { + let archive = archive_with(vec![Block::File(file(b"split", flag))]); + let error = archive + .extract_to_parallel_buffered(crate::ArchiveReadOptions::new(), |_| { + panic!("split cannot open output") + }) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::InvalidHeader("RAR 1.5 split entry requires multivolume extraction") + )); + } + } + + #[test] + fn extract_volumes_to_invokes_open_for_directory_entries() { + let dir = file(b"d", FHD_DIRECTORY_MASK); + let volumes = vec![archive_with(vec![Block::File(dir)])]; + + let capture = Capture::default(); + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default(), + capture.opener(), + ) + .unwrap(); + + let opened = capture.opened.borrow(); + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].name, b"d"); + assert!(opened[0].is_directory); + assert!(capture.bytes.borrow().is_empty()); + } + + #[test] + fn extract_volumes_to_writes_stored_file_payload_and_verifies_crc() { + let payload = b"hello stored payload!".to_vec(); + let mut entry = file(b"hello.txt", 0); + entry.unp_ver = 20; + entry.pack_size = payload.len() as u64; + entry.unp_size = payload.len() as u64; + entry.packed_range = 0..payload.len(); + entry.file_crc = super::super::crc32(&payload); + + let volumes = vec![archive_with_source( + vec![Block::File(entry)], + payload.clone(), + )]; + + let capture = Capture::default(); + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default(), + capture.opener(), + ) + .unwrap(); + + assert_eq!(capture.bytes.borrow().as_slice(), payload.as_slice()); + let opened = capture.opened.borrow(); + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].name, b"hello.txt"); + assert!(!opened[0].is_directory); + } + + #[test] + fn extract_volumes_to_reports_stored_crc_mismatch_with_entry_context() { + let payload = b"crc mismatch payload".to_vec(); + let mut entry = file(b"bad.txt", 0); + entry.unp_ver = 20; + entry.pack_size = payload.len() as u64; + entry.unp_size = payload.len() as u64; + entry.packed_range = 0..payload.len(); + entry.file_crc = super::super::crc32(&payload).wrapping_add(1); + + let volumes = vec![archive_with_source( + vec![Block::File(entry)], + payload.clone(), + )]; + + let capture = Capture::default(); + let err = extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default(), + capture.opener(), + ) + .unwrap_err(); + assert!( + matches!(err, Error::AtEntry { .. }), + "expected Error::AtEntry, got {err:?}" + ); + } + + #[test] + fn extract_volumes_to_writes_split_stored_file_across_volumes() { + let payload = b"this stored payload spans two volumes".to_vec(); + let split = 13usize; + + let mut first = file(b"split.txt", FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.pack_size = split as u64; + first.unp_size = payload.len() as u64; + first.packed_range = 0..split; + first.file_crc = super::super::crc32(&payload); + + let mut second = file(b"split.txt", FHD_SPLIT_BEFORE); + second.unp_ver = 20; + second.pack_size = (payload.len() - split) as u64; + second.unp_size = payload.len() as u64; + second.packed_range = 0..(payload.len() - split); + second.file_crc = super::super::crc32(&payload); + + let volumes = vec![ + archive_with_source(vec![Block::File(first)], payload[..split].to_vec()), + archive_with_source(vec![Block::File(second)], payload[split..].to_vec()), + ]; + + let capture = Capture::default(); + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default(), + capture.opener(), + ) + .unwrap(); + + assert_eq!(capture.bytes.borrow().as_slice(), payload.as_slice()); + let opened = capture.opened.borrow(); + assert_eq!(opened.len(), 1); + assert_eq!(opened[0].name, b"split.txt"); + } + + #[test] + fn extract_volumes_to_rejects_split_stored_size_mismatch() { + let payload = b"split stored mismatch".to_vec(); + let split = 10usize; + let truncated = payload.len() - 3; + + let mut first = file(b"a.txt", FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.pack_size = split as u64; + first.unp_size = payload.len() as u64; + first.packed_range = 0..split; + + let mut second = file(b"a.txt", FHD_SPLIT_BEFORE); + second.unp_ver = 20; + second.pack_size = (truncated - split) as u64; + second.unp_size = payload.len() as u64; + second.packed_range = 0..(truncated - split); + + let volumes = vec![ + archive_with_source(vec![Block::File(first)], payload[..split].to_vec()), + archive_with_source( + vec![Block::File(second)], + payload[split..truncated].to_vec(), + ), + ]; + + let capture = Capture::default(); + let err = extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default(), + capture.opener(), + ) + .unwrap_err(); + assert!( + matches!(err, Error::InvalidHeader(_)), + "expected Error::InvalidHeader, got {err:?}" + ); + } + + #[test] + fn split_stored_extraction_rejects_file_source_truncated_after_header_read() { + let payload = b"split file source can change"; + let split = 10; + let crc = super::super::crc32(payload); + let mut first = file(b"split.txt", FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.pack_size = split as u64; + first.unp_size = payload.len() as u64; + first.packed_range = 0..split; + first.file_crc = crc; + + let mut second = file(b"split.txt", FHD_SPLIT_BEFORE); + second.unp_ver = 20; + second.pack_size = (payload.len() - split) as u64; + second.unp_size = payload.len() as u64; + second.packed_range = 0..payload.len() - split; + second.file_crc = crc; + + let scratch = crate::scratch::case("legacy-split-truncated-source"); + let path = scratch.join("second.part"); + std::fs::write(&path, &payload[split..]).unwrap(); + let mut second_volume = archive_with(vec![Block::File(second)]); + second_volume.source = ArchiveSource::File(Arc::new(path.clone())); + std::fs::write(&path, &payload[split..payload.len() - 3]).unwrap(); + + let volumes = [ + archive_with_source(vec![Block::File(first)], payload[..split].to_vec()), + second_volume, + ]; + let err = extract_volumes_to(&volumes, crate::ArchiveReadOptions::default(), |_| { + Ok(Box::new(std::io::sink())) + }) + .unwrap_err(); + assert!(matches!( + err, + Error::InvalidHeader("RAR 1.5 split stored file ended before unpacked size") + )); + } + + #[test] + fn decoder_session_codec_for_resets_when_unpack_version_changes() { + let mut session = DecoderSession::new(true); + let mut f = file(b"a", 0); + f.unp_ver = 20; + assert!(matches!( + session.codec_for(&f).unwrap(), + CodecState::Unpack20(_) + )); + let mut g = file(b"b", 0); + g.unp_ver = 29; + assert!(matches!( + session.codec_for(&g).unwrap(), + CodecState::Unpack29(_) + )); + let mut h = file(b"c", 0); + h.unp_ver = 15; + assert!(matches!( + session.codec_for(&h).unwrap(), + CodecState::Unpack15(_) + )); + } + + #[test] + fn decoder_session_codec_for_propagates_unsupported_compression() { + let mut session = DecoderSession::new(false); + let mut f = file(b"a", 0); + f.unp_ver = 14; + assert!(matches!( + session.codec_for(&f), + Err(Error::UnsupportedCompression { + unpack_version: 14, + .. + }) + )); + } + + #[test] + fn decoder_session_codec_for_reuses_codec_in_solid_mode() { + let mut session = DecoderSession::new(true); + let mut f = file(b"a", 0); + f.unp_ver = 29; + let first = session.codec_for(&f).unwrap() as *const CodecState; + let second = session.codec_for(&f).unwrap() as *const CodecState; + assert_eq!(first, second); + } + + #[test] + fn public_file_writer_dispatches_stored_data_without_a_codec_session() { + let payload = b"decode_file_data stored dispatch".to_vec(); + let crc = super::super::crc32(&payload); + for unp_ver in [15u8, 20, 26, 29] { + let mut entry = file(b"a.txt", 0); + entry.unp_ver = unp_ver; + entry.pack_size = payload.len() as u64; + entry.unp_size = payload.len() as u64; + entry.packed_range = 0..payload.len(); + entry.file_crc = crc; + + let archive = archive_with_source(vec![Block::File(entry.clone())], payload.clone()); + let mut data = Vec::new(); + entry + .write_to(&archive, None, &mut data) + .unwrap_or_else(|err| panic!("write for unp_ver {unp_ver}: {err:?}")); + assert_eq!(data, payload, "unp_ver {unp_ver} payload mismatch"); + } + } + + #[test] + fn extraction_rejects_stored_members_with_mismatched_packed_size() { + let data = b"payload"; + for unpacked_size in [data.len() as u64 - 1, data.len() as u64 + 1] { + let mut entry = file(b"stored.txt", 0); + entry.pack_size = data.len() as u64; + entry.unp_size = unpacked_size; + entry.packed_range = 0..data.len(); + let archive = archive_with_source(vec![Block::File(entry)], data.to_vec()); + + for parallel in [false, true] { + let open = |_: &ExtractedEntryMeta| Ok(Box::new(std::io::sink()) as Box); + let result = if parallel { + archive.extract_to_parallel_buffered(crate::ArchiveReadOptions::default(), open) + } else { + archive.extract_to(crate::ArchiveReadOptions::default(), open) + }; + let error = result.expect_err("stored member size mismatch must fail"); + assert!(matches!(error.root_cause(), Error::InvalidHeader(_))); + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_works_through_boxed_inner_reader() { + let plain = *b"0123456789abcdefRAR2 block two!!"; + let mut encrypted = plain; + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(); + let inner: Box = Box::new(Cursor::new(encrypted.to_vec())); + let reader = DecryptingReader::new(inner, 20, b"pw", None).unwrap(); + let out = read_in_small_chunks(reader); + + assert_eq!(out, plain); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_boxed_inner_rejects_non_block_aligned_eof() { + let mut payload = vec![0u8; 32]; + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut payload[..16]) + .unwrap(); + // 23 bytes of trailing data (not a multiple of 16) — should error at EOF. + payload.truncate(23); + let inner: Box = Box::new(Cursor::new(payload)); + let mut reader = DecryptingReader::new(inner, 20, b"pw", None).unwrap(); + let mut buf = [0u8; 64]; + let err = loop { + match reader.read(&mut buf) { + Ok(0) => panic!("expected non-block-aligned data error"), + Ok(_) => continue, + Err(err) => break err, + } + }; + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + fn split_solid_rar29_member_keeps_history_from_regular_predecessor() { + let archive = Archive::parse(include_bytes!( + "../../tests/fixtures/rar15_40/rar300/solid_simple_rar300.rar" + )) + .unwrap(); + let files: Vec<_> = archive.files().cloned().collect(); + assert_eq!(files.len(), 2); + assert!(files[1].is_solid()); + let mut first_part = files[1].clone(); + let mut last_part = first_part.clone(); + let middle = first_part.packed_range.start + first_part.packed_range.len() / 2; + first_part.block.flags |= FHD_SPLIT_AFTER; + first_part.packed_range.end = middle; + first_part.pack_size = first_part.packed_range.len() as u64; + last_part.block.flags |= FHD_SPLIT_BEFORE; + last_part.packed_range.start = middle; + last_part.pack_size = last_part.packed_range.len() as u64; + let mut volume1 = archive.clone(); + volume1.blocks = vec![Block::File(files[0].clone()), Block::File(first_part)]; + let mut volume2 = archive; + volume2.blocks = vec![Block::File(last_part)]; + let capture = Capture::default(); + extract_volumes_to( + &[volume1, volume2], + crate::ArchiveReadOptions::default(), + capture.opener(), + ) + .unwrap(); + assert_eq!(capture.opened.borrow().len(), 2); + assert_eq!(capture.bytes.borrow().as_slice(), + b"shared prefix shared prefix shared prefix alpha\nshared prefix shared prefix shared prefix beta\n"); + } + + #[test] + #[cfg(feature = "encryption")] + fn split_rar15_encrypted_stored_member_has_no_block_padding() { + let payload = b"legacy byte cipher split"; + let mut encrypted = payload.to_vec(); + Rar15Cipher::new(b"pw").crypt_in_place(&mut encrypted); + let split = 7; + let mut first = file(b"split", FHD_PASSWORD | FHD_SPLIT_AFTER); + first.unp_ver = 15; + first.unp_size = payload.len() as u64; + first.pack_size = split as u64; + first.packed_range = 0..split; + first.file_crc = crc32(payload); + let mut last = first.clone(); + last.block.flags = FHD_PASSWORD | FHD_SPLIT_BEFORE; + last.pack_size = (payload.len() - split) as u64; + last.packed_range = 0..payload.len() - split; + let volumes = [ + archive_with_source(vec![Block::File(first)], encrypted[..split].to_vec()), + archive_with_source(vec![Block::File(last)], encrypted[split..].to_vec()), + ]; + let capture = Capture::default(); + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::with_password(b"pw"), + capture.opener(), + ) + .unwrap(); + assert_eq!(capture.bytes.borrow().as_slice(), payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn extract_volumes_to_assembles_encrypted_stored_split_across_two_volumes() { + let payload: &[u8] = b"twenty-byte payload!"; // exactly 20 bytes + let unpacked_len = payload.len(); + assert_eq!(unpacked_len, 20); + let padded_len = (unpacked_len + 15) & !15; // 32 + let mut encrypted = vec![0u8; padded_len]; + encrypted[..unpacked_len].copy_from_slice(payload); + Rar20Cipher::new(b"pw") + .encrypt_in_place(&mut encrypted) + .unwrap(); + let split = 13usize; + let crc = super::super::crc32(payload); + + let mut first = file(b"split.bin", FHD_PASSWORD | FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.pack_size = split as u64; + first.unp_size = unpacked_len as u64; + first.packed_range = 0..split; + first.file_crc = crc; + + let mut second = file(b"split.bin", FHD_PASSWORD | FHD_SPLIT_BEFORE); + second.unp_ver = 20; + second.pack_size = (padded_len - split) as u64; + second.unp_size = unpacked_len as u64; + second.packed_range = 0..(padded_len - split); + second.file_crc = crc; + + let volumes = vec![ + archive_with_source(vec![Block::File(first)], encrypted[..split].to_vec()), + archive_with_source(vec![Block::File(second)], encrypted[split..].to_vec()), + ]; + + let capture = Capture::default(); + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::with_password(b"pw"), + capture.opener(), + ) + .unwrap(); + + assert_eq!(capture.bytes.borrow().as_slice(), payload); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_stored_split_rejects_unpacked_size_rounding_overflow() { + let mut first = file(b"overflow", FHD_PASSWORD | FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.unp_size = u64::MAX; + first.pack_size = 16; + first.packed_range = 0..16; + let mut last = first.clone(); + last.block.flags = FHD_PASSWORD | FHD_SPLIT_BEFORE; + let volumes = [ + archive_with_source(vec![Block::File(first)], vec![0; 16]), + archive_with_source(vec![Block::File(last)], vec![0; 16]), + ]; + let capture = Capture::default(); + let error = extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::with_password(b"pw"), + capture.opener(), + ) + .unwrap_err(); + let expected = if usize::BITS == 64 { + "RAR 2.x encrypted split stored size overflows" + } else { + "RAR 1.5 split unpacked size overflows usize" + }; + assert!( + matches!(error.root_cause(), Error::InvalidHeader(message) if *message == expected) + ); + assert!(capture.bytes.borrow().is_empty()); + } + + #[test] + #[cfg(feature = "encryption")] + fn extract_volumes_to_rejects_encrypted_stored_split_when_padded_size_disagrees() { + let unpacked_len = 20usize; + // Two volumes total only 30 bytes, but expected_packed_len == 32. + let payload = [0u8; 30]; + + let mut first = file(b"split.bin", FHD_PASSWORD | FHD_SPLIT_AFTER); + first.unp_ver = 20; + first.pack_size = 13; + first.unp_size = unpacked_len as u64; + first.packed_range = 0..13; + + let mut second = file(b"split.bin", FHD_PASSWORD | FHD_SPLIT_BEFORE); + second.unp_ver = 20; + second.pack_size = 17; + second.unp_size = unpacked_len as u64; + second.packed_range = 0..17; + + let volumes = vec![ + archive_with_source(vec![Block::File(first)], payload[..13].to_vec()), + archive_with_source(vec![Block::File(second)], payload[13..].to_vec()), + ]; + + let capture = Capture::default(); + let err = extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::with_password(b"pw"), + capture.opener(), + ) + .unwrap_err(); + assert!( + matches!(err, Error::InvalidHeader(msg) if msg.contains("wrong reassembled size")), + "expected wrong reassembled size error, got {err:?}" + ); + } +} diff --git a/arca-rar/vendor/rars/src/rar15_40/write.rs b/arca-rar/vendor/rars/src/rar15_40/write.rs new file mode 100644 index 00000000..4dbae85f --- /dev/null +++ b/arca-rar/vendor/rars/src/rar15_40/write.rs @@ -0,0 +1,3829 @@ +use super::*; +use crate::codec::rar13::{ + unpack15_encode_with_options_and_progress, EncodeOptions as Rar15EncodeOptions, Unpack15Encoder, +}; +use crate::codec::rar20::{ + unpack20_encode_auto_with_options_and_progress, EncodeOptions as Rar20EncodeOptions, + Unpack20Encoder, +}; +use crate::codec::rar29::{ + unpack29_encode_literals, unpack29_encode_literals_with_options, + unpack29_encode_literals_with_options_and_progress, ChainEngine, + EncodeOptions as Rar29EncodeOptions, Unpack29Encoder, +}; +use crate::crc32::Crc32; +pub use crate::filter::{FilterKind, FilterPolicy, FilterSpec}; +pub use crate::streaming::EntrySource; +use crate::streaming::WriterResources; +use crate::write_plan::{MemberCoding, PlanShape, WriterOption}; +use crate::write_progress::{ProgressReporter, WorkTracker}; +use crate::write_stream::{MemberBytes, MemberPayload}; +use crate::FeatureSet; +use crate::{WriteOperation, WriteProgress, WriteProgressEvent}; +use std::io::Write; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct WriterOptions { + pub target: ArchiveVersion, + pub features: FeatureSet, + pub compression_level: Option, + pub dictionary_size: Option, + /// Which engine compresses a member. Only the RAR 2.9 family has more than + /// one, so the other targets ignore anything but the default. + pub method: Rar29Method, + /// Retained NewSub comment DOS timestamp and host ID. + pub(crate) archive_comment_metadata: Option<(u32, u8)>, +} + +/// Which compression engine the RAR 2.9 family writer uses for a member. +/// +/// PPMd is an alternative to LZ, not a filter, even though the writer used to +/// carry it as one. Keeping the two apart is what lets a filter be chosen +/// without also deciding the engine. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum Rar29Method { + /// Let the content decide. Text-shaped members are measured against PPMd; + /// anything else goes straight to LZ, so a binary member never pays for a + /// PPMd encode it was always going to lose. Compression levels 1 to 4 skip + /// the trial entirely, since those settings are asking for speed. + #[default] + Auto, + Lz, + Ppmd, +} + +impl WriterOptions { + pub const fn new(target: ArchiveVersion, features: FeatureSet) -> Self { + Self { + target, + features, + compression_level: None, + dictionary_size: None, + method: Rar29Method::Auto, + archive_comment_metadata: None, + } + } + + pub const fn with_method(mut self, method: Rar29Method) -> Self { + self.method = method; + self + } + + pub const fn with_compression_level(mut self, level: u8) -> Self { + self.compression_level = Some(level); + self + } + + pub const fn with_dictionary_size(mut self, size: usize) -> Self { + self.dictionary_size = Some(size); + self + } +} + +impl Default for WriterOptions { + fn default() -> Self { + Self { + target: ArchiveVersion::Rar15, + features: FeatureSet::store_only(), + compression_level: None, + dictionary_size: None, + method: Rar29Method::Auto, + archive_comment_metadata: None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StoredEntry<'a> { + pub name: &'a [u8], + pub data: &'a [u8], + pub file_time: u32, + pub file_attr: u32, + pub host_os: u8, + pub password: Option<&'a [u8]>, + pub file_comment: Option<&'a [u8]>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileEntry<'a> { + pub name: &'a [u8], + pub data: &'a [u8], + pub file_time: u32, + pub file_attr: u32, + pub host_os: u8, + pub password: Option<&'a [u8]>, + pub file_comment: Option<&'a [u8]>, +} + +/// One member of an archive written a member at a time. +/// +/// The bytes are opened when the member is coded and dropped once it is +/// written, so an archive of many files never holds more than the members +/// being worked on. A stored member is copied straight from its source and +/// never lands on the heap at all. +#[derive(Debug, Clone)] +pub struct StreamingEntry { + pub name: Vec, + pub source: EntrySource, + pub file_time: u32, + pub file_attr: u32, + pub host_os: u8, + pub password: Option>, + pub file_comment: Option>, +} + +impl StreamingEntry { + pub fn new(name: impl Into>, source: EntrySource) -> Self { + Self { + name: name.into(), + source, + file_time: 0, + file_attr: 0, + host_os: 3, + password: None, + file_comment: None, + } + } + + pub fn with_file_time(mut self, file_time: u32) -> Self { + self.file_time = file_time; + self + } + + pub fn with_file_attr(mut self, file_attr: u32) -> Self { + self.file_attr = file_attr; + self + } + + pub fn with_host_os(mut self, host_os: u8) -> Self { + self.host_os = host_os; + self + } + + pub fn with_password(mut self, password: impl Into>) -> Self { + self.password = Some(password.into()); + self + } + + pub fn with_file_comment(mut self, comment: impl Into>) -> Self { + self.file_comment = Some(comment.into()); + self + } +} + +#[cfg(any(test, feature = "encryption"))] +fn checked_align16(value: usize, message: &'static str) -> Result { + value + .checked_add(15) + .map(|size| size & !15) + .ok_or(Error::InvalidArgument(message)) +} + +const AUTO_RGB_WIDTHS: [usize; 4] = [24, 48, 96, 192]; +const MIN_STORE_FALLBACK_SIZE: usize = 1024; +const RAR29_TEXT_SAMPLE_SIZE: usize = 8192; + +/// How long a run of NUL has to be before the text screen reads it as padding +/// and stops counting it either way. +/// +/// Sixteen is well past what falls out of text (a run of NUL in a source file +/// is already unusual) and well short of the 512 byte blocks tar pads to. +/// Measured across 400 binaries over 200 KB in `/usr/bin` and +/// `/usr/lib/x86_64-linux-gnu`, 8, 16 and 32 all give the same answer, so the +/// exact number is not load-bearing. +const RAR29_NUL_RUN_IS_PADDING: usize = 16; + +/// The level an absent `--level` means for RAR 2.0, matching the method byte +/// `compression_method_for_level` writes for it. +const RAR20_DEFAULT_LEVEL: u8 = 3; +const RAR29_AUDIO_SAMPLE_SIZE: usize = 8192; +/// How much data one RAR 2.9 LZ block covers. +/// +/// Every block carries its own Huffman tables, so this is the distance over +/// which the tables have to fit the data. A megabyte was far too long a lease: +/// a binary's code and its data want different tables, and one set fitted to +/// both is worse than either. Measured on `--level 5`, packed bytes: +/// +/// ```text +/// block 16K 32K 64K 128K 1M +/// libc 686400 683942 684278 704689 797692 +/// python 632452 628926 627862 648835 738624 +/// mixed 136376 135530 135264 146831 149372 +/// lorem 16186 16186 16186 16186 16186 +/// ``` +/// +/// 64K is the best or within a rounding error of it on everything measured, +/// and text does not care either way. The curve turns back up below 32K, where +/// re-sending the tables starts to cost more than fitting them gains. +const RAR29_LZ_BLOCK_SIZE: usize = 64 * 1024; + +/// The largest dictionary the format encodes, used when a memory estimate +/// cannot resolve the real one. It has to stay an upper bound, so it does not +/// follow the block size. +const RAR29_MAX_DICTIONARY_SIZE: usize = 4 * 1024 * 1024; +#[cfg(feature = "encryption")] +const RAR15_ALIGN_OVERFLOW: &str = "RAR 1.5 block size overflows usize"; + +pub fn write_stored_archive( + entries: &[StoredEntry<'_>], + options: WriterOptions, +) -> Result> { + write_stored_archive_with_comment(entries, options, None) +} + +pub fn write_stored_archive_with_comment( + entries: &[StoredEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, +) -> Result> { + let members: Vec<_> = entries.iter().map(Member::from_stored).collect(); + collect_archive( + &members, + options, + MemberCoding::Stored, + archive_comment, + None, + ) +} + +pub(crate) struct RetainedMemberMetadata<'a> { + pub(crate) unicode_name: Option<&'a [u8]>, + pub(crate) unpack_version: Option, + pub(crate) extended_times: Option<&'a [u8]>, + pub(crate) is_directory: bool, + pub(crate) is_symlink: bool, +} + +pub(crate) struct RetainedFileEntry<'a> { + pub(crate) file: FileEntry<'a>, + pub(crate) source: Option<&'a EntrySource>, + pub(crate) metadata: RetainedMemberMetadata<'a>, +} + +/// Builder path for retained native metadata and entry kinds. Existing direct +/// entry structs keep their API. +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_archive_with_retained_metadata( + entries: &[RetainedFileEntry<'_>], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + header_password: Option<&[u8]>, + archive_comment_password: Option<&[u8]>, + output: &mut dyn Write, +) -> Result<()> { + // Builder validates retained records at their setters and constructs each + // file with its metadata, so neither record validity nor array length can + // change at this private handoff. + let members: Vec<_> = entries + .iter() + .map(|entry| { + let mut member = Member::from_file(&entry.file); + if let Some(source) = entry.source { + member.bytes = MemberBytes::NamedSource(source, entry.file.name); + } + member.unicode_name = entry.metadata.unicode_name; + member.unpack_version = entry.metadata.unpack_version; + member.extended_times = entry.metadata.extended_times; + member.is_directory = entry.metadata.is_directory; + member.is_symlink = entry.metadata.is_symlink; + member + }) + .collect(); + write_archive_to( + &members, + options, + coding, + archive_comment, + resources, + progress, + output, + header_password, + archive_comment_password, + ) +} + +pub fn write_compressed_archive( + entries: &[FileEntry<'_>], + options: WriterOptions, +) -> Result> { + write_compressed_archive_with_comment(entries, options, None) +} + +pub fn write_compressed_archive_with_comment( + entries: &[FileEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, +) -> Result> { + write_compressed_archive_with_comment_and_progress(entries, options, archive_comment, None) +} + +pub fn write_compressed_archive_with_comment_and_progress( + entries: &[FileEntry<'_>], + options: WriterOptions, + archive_comment: Option<&[u8]>, + progress: Option<&dyn WriteProgress>, +) -> Result> { + let members: Vec<_> = entries.iter().map(Member::from_file).collect(); + collect_archive( + &members, + options, + MemberCoding::Compressed, + archive_comment, + progress, + ) +} + +pub fn write_rar29_compressed_archive_with_filter_policy( + entries: &[FileEntry<'_>], + options: WriterOptions, + policy: FilterPolicy, +) -> Result> { + write_rar29_compressed_archive_with_filter_policy_and_progress(entries, options, policy, None) +} + +pub fn write_rar29_compressed_archive_with_filter_policy_and_progress( + entries: &[FileEntry<'_>], + options: WriterOptions, + policy: FilterPolicy, + progress: Option<&dyn WriteProgress>, +) -> Result> { + let members: Vec<_> = entries.iter().map(Member::from_file).collect(); + collect_archive( + &members, + options, + MemberCoding::Filtered(policy), + None, + progress, + ) +} + +/// Writes an archive straight to `output`, holding only the members being +/// compressed rather than every input at once. +/// +/// `resources` bounds how many members are in flight, which is what keeps the +/// peak flat across an archive of many files. One member still has to fit: +/// these codecs compress a member as a unit, so a member larger than the budget +/// is compressed on its own rather than refused. +pub fn write_streaming_archive_to( + entries: &[StreamingEntry], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + output: &mut dyn Write, +) -> Result<()> { + let members: Vec<_> = entries.iter().map(Member::from_streaming).collect(); + write_archive_to( + &members, + options, + coding, + archive_comment, + resources, + progress, + output, + None, + None, + ) +} + +/// Runs the streaming writer into a buffer, for the callers that want the +/// archive as bytes. The budget still applies, so a many-file archive holds one +/// window of members rather than all of them. +fn collect_archive( + members: &[Member<'_>], + options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + progress: Option<&dyn WriteProgress>, +) -> Result> { + let mut out = Vec::new(); + write_archive_to( + members, + options, + coding, + archive_comment, + &WriterResources::default(), + progress, + &mut out, + None, + None, + )?; + Ok(out) +} + +#[allow(clippy::too_many_arguments)] +fn write_archive_to( + members: &[Member<'_>], + mut options: WriterOptions, + coding: MemberCoding, + archive_comment: Option<&[u8]>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + output: &mut dyn Write, + explicit_header_password: Option<&[u8]>, + archive_comment_password: Option<&[u8]>, +) -> Result<()> { + if explicit_header_password.is_some() + || archive_comment_password.is_some() + || members.iter().any(|member| member.password.is_some()) + { + crate::crypto::require_encryption()?; + } + + if resources.max_preparation_bytes().is_some() || resources.max_memory_bytes().is_some() { + return Err(Error::UnsupportedFamilyFeature { + family: options.target.family(), + feature: if resources.max_memory_bytes().is_some() { + "aggregate managed-memory limit" + } else { + "preparation memory quota" + }, + }); + } + + if let Some(password) = archive_comment_password { + if password.is_empty() + || archive_comment.is_none() + || !matches!( + options.target, + ArchiveVersion::Rar30 | ArchiveVersion::Rar40 + ) + { + return Err(Error::InvalidArgument( + "encrypted legacy archive comments require a password and RAR3/4 CMT output", + )); + } + } + let control = + crate::write_progress::ResourceProgress::new(resources, progress.map(ProgressReporter)); + let reporting = &control as &dyn WriteProgress; + let progress = Some(reporting); + crate::write_progress::check_cancelled(progress.map(ProgressReporter))?; + let mut output = crate::write_progress::CancellableIo { + inner: output, + progress: progress.map(ProgressReporter), + }; + let output = &mut output as &mut dyn Write; + let has_file_comment = members.iter().any(|member| member.file_comment.is_some()); + if has_file_comment && options.features.header_encryption { + return Err(Error::UnsupportedWriterOption { + target: options.target, + option: WriterOption::FileComment, + because: Some("embedded legacy file comments with header encryption are incompatible with reference readers"), + }); + } + validate_plan( + options, + coding.shape(), + archive_comment.is_some(), + has_file_comment, + )?; + if let MemberCoding::Filtered(policy) = &coding { + validate_rar29_filter_policy(policy, options.method)?; + } + let header_password = if options.features.header_encryption { + validate_header_encrypted_archive_options( + options.target, + explicit_header_password.is_some() + || members.iter().any(|member| member.password.is_some()), + )?; + Some(if let Some(password) = explicit_header_password { + header_encryption_password( + std::iter::once(Some(password)).chain( + members + .iter() + .filter_map(|member| member.password) + .map(Some), + ), + )? + } else { + header_encryption_password(members.iter().map(|member| member.password))? + }) + } else { + None + }; + + let mut total_bytes = 0u64; + let mut largest_member = 0u64; + for member in members { + let unpacked = member.unpacked_size().map_err(|error| { + crate::write_stream::member_error(error, member.name, "reading source") + })? as u64; + total_bytes = total_bytes.saturating_add(unpacked); + largest_member = largest_member.max(unpacked); + } + if options.dictionary_size.is_none() { + // A solid run is one stream, so its window has to reach back across the + // members it already coded; independent members only ever look inside + // themselves. Validation has already rejected solid stored coding. + let reach = if options.features.solid { + total_bytes + } else { + largest_member + }; + options.dictionary_size = Some(fitted_dictionary_size(options.target, reach)); + } + // RAR 2.0 walks a non-solid member twice, once to pick a method and once to + // encode it, so its progress total counts every byte twice. + let total_work = if options.target == ArchiveVersion::Rar20 + && !options.features.solid + && coding.compresses() + { + total_bytes.saturating_mul(2) + } else { + total_bytes + }; + report_compression_operation(reporting, true, total_work, members.len()); + let work = WorkTracker::new( + Some(ProgressReporter(reporting)), + WriteOperation::Compression, + total_work, + ); + + let result = write_members_to( + members, + options, + &coding, + archive_comment, + header_password, + archive_comment_password, + resources, + &work, + output, + ); + result?; + if !work.finish() { + return Err(Error::Cancelled); + } + report_compression_operation(reporting, false, total_work, members.len()); + work.check() +} + +#[allow(clippy::too_many_arguments)] +fn write_members_to( + members: &[Member<'_>], + options: WriterOptions, + coding: &MemberCoding, + archive_comment: Option<&[u8]>, + header_password: Option<&[u8]>, + archive_comment_password: Option<&[u8]>, + resources: &WriterResources, + progress: &WorkTracker<'_>, + output: &mut dyn Write, +) -> Result<()> { + output.write_all(RAR15_SIGNATURE)?; + let mut main_flags = 0; + if options.features.solid { + main_flags |= MHD_SOLID; + } + if header_password.is_some() { + main_flags |= MHD_PASSWORD; + } + if archive_comment.is_some() && uses_old_style_archive_comment(options.target) { + main_flags |= MHD_COMMENT; + } + let mut head = Vec::new(); + write_main_header(&mut head, main_flags); + output.write_all(&head)?; + write_archive_comment( + output, + archive_comment, + options, + header_password, + archive_comment_password, + )?; + + // Solid members share one encoder, so they are coded in order. Independent + // ones are coded a window at a time and written as each window lands. + if options.features.solid { + let mut solid_encoder = Some(SolidEncoder::for_target(options)?); + let mut solid_run_has_member = false; + for member in members { + let encoded = encode_member( + member, + options, + coding, + &mut solid_encoder, + resources, + progress, + ) + .map_err(|error| crate::write_stream::member_error(error, member.name, "preparing"))?; + let solid_continuation = encoded.method != 0x30 && solid_run_has_member; + // Storing a member rebuilds the encoder, so the chain ends there. + // An empty compressed member leaves the chain exactly as it was: + // it feeds the encoder nothing, and readers pass over an empty + // payload without advancing their decoder. Counting one as a + // member left the next one flagged as continuing a chain that had + // been broken by a stored member two places back. + if member.is_directory { + // Directory headers do not participate in the solid stream. + } else if encoded.method == 0x30 { + solid_run_has_member = false; + } else if encoded.unpacked_size != 0 { + solid_run_has_member = true; + } + write_member( + output, + member, + encoded, + options, + solid_continuation, + header_password, + progress.reporter(), + ) + .map_err(|error| crate::write_stream::member_error(error, member.name, "writing"))?; + member.bytes.release(); + } + } else { + // Scoped rewrite sources retain plaintext until emission; their disk + // quota may admit fewer inputs than the CPU pool can prepare at once. + let window = members + .iter() + .filter_map(|member| member.bytes.source()) + .map(EntrySource::max_preparation_window) + .fold(crate::parallel::default_window(), usize::min); + crate::parallel::map_slice_windowed( + members, + window, + |member| { + encode_member(member, options, coding, &mut None, resources, progress).map_err( + |error| crate::write_stream::member_error(error, member.name, "preparing"), + ) + }, + |member, encoded| { + write_member( + output, + member, + encoded, + options, + false, + header_password, + progress.reporter(), + ) + .map_err(|error| { + crate::write_stream::member_error(error, member.name, "writing") + })?; + member.bytes.release(); + Ok(()) + }, + )?; + } + if let Some(password) = header_password { + write_encrypted_end_block(output, password)?; + } + Ok(()) +} + +/// Encodes one member, choosing an engine and a filter independently. +/// +/// The two are separate questions: which engine compresses this content best, +/// and which transform makes it easier to compress. They used to be one enum, +/// which meant some combinations simply could not be asked for. +fn encode_rar29_policy_filtered_payload( + data: &[u8], + policy: &FilterPolicy, + method: Rar29Method, + options: Rar29EncodeOptions, + lz_method: u8, + ppmd_trial: bool, + progress: Option<&WorkTracker<'_>>, +) -> Result { + // An empty member is stored whatever was asked for. A filter over it is a + // 0..0 range, which the codec refuses, so an archive holding one empty file + // used to compress every other member and then fail at the last step. Only + // the search path guarded this; every explicit policy walked into it. + if lz_method == 0x30 || data.is_empty() { + return Ok(EncodedPayload { + data: data.to_vec(), + method: 0x30, + }); + } + // The method byte carries the level that was asked for, not which engine + // answered. RAR 2.9 signals PPMd inside the stream (our own decoder takes + // no method byte at all), and WinRAR 3.00 stamps 0x34 on the PPMd archive + // it writes at -m4. Stamping 0x35 on every PPMd payload only looked right + // while the trial ran at level 5, and made a level 3 archive claim to be a + // level 5 one as soon as it ran anywhere else. + let ppmd = |data: &[u8]| -> Result { + Ok(EncodedPayload { + data: with_codec_progress(progress, |report| { + crate::codec::rar29::unpack29_encode_ppmd_with_progress( + data, + true, + None, + options.max_match_distance, + report, + ) + })?, + method: lz_method, + }) + }; + match (method, policy) { + (Rar29Method::Ppmd, FilterPolicy::None) => ppmd(data), + (Rar29Method::Ppmd, FilterPolicy::Explicit(filter)) => Ok(EncodedPayload { + data: with_codec_progress(progress, |report| { + crate::codec::rar29::unpack29_encode_ppmd_with_progress( + data, + true, + Some(filter.clone()), + options.max_match_distance, + report, + ) + })?, + method: lz_method, + }), + // Filtered writes are rejected before encoding; plain compressed writes + // use FilterPolicy::None when PPMd is forced. + (Rar29Method::Ppmd, FilterPolicy::Auto) => unreachable!(), + (Rar29Method::Lz, FilterPolicy::None) => { + encode_rar29_lz_member(data, options, lz_method, progress) + } + (Rar29Method::Lz, FilterPolicy::Auto) => encode_rar29_auto_filtered_member_with_progress( + data, options, lz_method, false, progress, + ), + (Rar29Method::Lz, FilterPolicy::Explicit(filter)) => Ok(EncodedPayload { + data: with_codec_progress(progress, |report| { + Unpack29Encoder::with_options(options).encode_member_with_filters_and_progress( + data, + std::slice::from_ref(filter), + Some(report), + ) + })?, + method: lz_method, + }), + (Rar29Method::Auto, FilterPolicy::Auto) => encode_rar29_auto_filtered_member_with_progress( + data, options, lz_method, ppmd_trial, progress, + ), + (Rar29Method::Auto, policy) => { + let mut best = match policy { + FilterPolicy::Explicit(filter) => EncodedPayload { + data: with_codec_progress(progress, |report| { + Unpack29Encoder::with_options(options) + .encode_member_with_filters_and_progress( + data, + std::slice::from_ref(filter), + Some(report), + ) + })?, + method: lz_method, + }, + _ => encode_rar29_lz_member(data, options, lz_method, progress)?, + }; + // Gated on the content, so a binary member never pays for a PPMd + // encode it was always going to lose. + if ppmd_trial && is_text_ppmd_candidate(data) { + let candidate = ppmd(data)?; + if candidate.data.len() < best.data.len() { + best = candidate; + } + } + Ok(best) + } + } +} + +/// Whether this level is willing to encode the member twice to choose an engine. +/// +/// Takes the resolved level, not the option. Reading `Option` meant an +/// absent level and `--level 3` disagreed, though both write method 0x33: the +/// default paid for the trial and the level that names the same number did not, +/// so the two produced archives 24% apart on the same 4 MiB of text. +/// +/// Levels 1 and 2 are the ones asking for speed and still skip it. Level 3 is +/// the default and where most archives are written, and it is worth about a +/// fifth of the size on text for roughly twice the encode. WinRAR turns PPMd on +/// one level later, at m4: on the bench text member WinRAR 3.00 packs 657,294 +/// bytes at m3 and 585,357 at m4, where we now write 540,517 at both. +fn ppmd_trial_pays(level: u8) -> bool { + level >= 3 +} + +fn validate_rar29_filter_policy(policy: &FilterPolicy, method: Rar29Method) -> Result<()> { + // Searching for a filter means measuring candidates against each other, + // and the search only knows how to measure them through LZ. + if matches!(policy, FilterPolicy::Auto) && method == Rar29Method::Ppmd { + return Err(Error::InvalidArgument( + "RAR 2.9 cannot search for a filter while PPMd is forced", + )); + } + let filter = match policy { + FilterPolicy::Explicit(filter) => filter, + FilterPolicy::None | FilterPolicy::Auto => return Ok(()), + }; + match filter.kind { + FilterKind::Delta { channels } => { + if channels == 0 || channels > 32 { + return Err(Error::InvalidArgument( + "RAR 2.9 DELTA filter channel count is invalid", + )); + } + } + FilterKind::Audio { channels } => { + if channels == 0 || channels > 32 { + return Err(Error::InvalidArgument( + "RAR 2.9 AUDIO filter channel count is invalid", + )); + } + } + FilterKind::Rgb { width, pos_r } => { + if width == 0 || !width.is_multiple_of(3) || pos_r > 2 { + return Err(Error::InvalidArgument( + "RAR 2.9 RGB filter parameters are invalid", + )); + } + } + FilterKind::E8 | FilterKind::E8E9 | FilterKind::Itanium => {} + // Refused here rather than in the codec, so nothing is compressed + // before the caller is told the filter cannot be written. + FilterKind::Arm => { + return Err(Error::UnsupportedFeature { + version: ArchiveVersion::Rar29, + feature: "ARM filter", + }) + } + } + Ok(()) +} + +fn encode_rar29_lz_member( + data: &[u8], + options: Rar29EncodeOptions, + method: u8, + progress: Option<&WorkTracker<'_>>, +) -> Result { + let compressed = with_codec_progress(progress, |report| { + unpack29_encode_literals_with_options_and_progress(data, options, report) + })?; + if compressed.len() >= data.len() { + return Ok(EncodedPayload { + data: data.to_vec(), + method: 0x30, + }); + } + Ok(EncodedPayload { + data: compressed, + method, + }) +} + +#[cfg(test)] +fn encode_rar29_filtered_member( + data: &[u8], + filter: FilterSpec, + options: Rar29EncodeOptions, +) -> Result> { + Unpack29Encoder::with_options(options) + .encode_member_with_filter(data, filter) + .map_err(Error::from) +} + +#[cfg(test)] +fn encode_rar29_filtered_members( + data: &[u8], + filters: &[FilterSpec], + options: Rar29EncodeOptions, +) -> Result> { + Unpack29Encoder::with_options(options) + .encode_member_with_filters(data, filters) + .map_err(Error::from) +} + +/// How the RAR 2.9 family measures a filter candidate, for the shared search. +#[derive(Clone, Copy)] +struct Rar29Search; + +impl crate::filter_search::FilterSearch for Rar29Search { + type Options = Rar29EncodeOptions; + + fn screened_kinds(&self, data: &[u8]) -> Vec { + let mut kinds = vec![FilterKind::Itanium]; + for channels in 1..=4 { + kinds.push(FilterKind::Delta { channels }); + // Free to ask, and it keeps the screened set from doubling. + if is_audio_filter_candidate(data, channels) { + kinds.push(FilterKind::Audio { channels }); + } + } + kinds.extend( + AUTO_RGB_WIDTHS + .into_iter() + .filter(|&width| data.len() >= width) + .map(|width| FilterKind::Rgb { width, pos_r: 0 }), + ); + kinds + } + + /// Unlike RAR 5, a greedy parse here really is cheaper. On a megabyte of + /// binary it measured 59ms against 91ms at full effort, and it ranks + /// candidates the same way, so the screens run reduced and only the + /// finalists pay full price. + fn screen_options(&self, options: Rar29EncodeOptions) -> Rar29EncodeOptions { + let mut ranking = Rar29EncodeOptions::new(options.max_match_candidates.min(8)) + .with_lazy_matching(false) + .with_max_match_distance(options.max_match_distance); + if let Some(block_size) = options.block_size { + ranking = ranking.with_block_size(block_size); + } + ranking + } + + /// The VM filter splits a range into blocks it can address, and a delta + /// block holds whole records, so the channel count is bounded by the block + /// rather than by a field width the way RAR 5's is. + fn max_delta_channels(&self) -> usize { + crate::codec::rar29::MAX_VM_DELTA_FILTER_BLOCK_SIZE + } + + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result> { + crate::codec::rar29::filtered_members(data, filters) + .map(|filtered| filtered.data) + .map_err(Error::from) + } + + fn encode_plain( + &self, + data: &[u8], + options: Rar29EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + match progress { + Some(progress) => { + unpack29_encode_literals_with_options_and_progress(data, options, progress) + .map_err(map_codec_cancel) + } + None => unpack29_encode_literals_with_options(data, options).map_err(Error::from), + } + } + + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: Rar29EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Unpack29Encoder::with_options(options) + .encode_member_with_filters_and_progress(data, filters, progress) + .map_err(map_codec_cancel) + } +} + +#[cfg(test)] +fn encode_rar29_auto_filtered_member( + data: &[u8], + options: Rar29EncodeOptions, + lz_method: u8, + include_ppmd: bool, +) -> Result { + encode_rar29_auto_filtered_member_with_progress(data, options, lz_method, include_ppmd, None) +} + +fn encode_rar29_auto_filtered_member_with_progress( + data: &[u8], + options: Rar29EncodeOptions, + lz_method: u8, + include_ppmd: bool, + progress: Option<&WorkTracker<'_>>, +) -> Result { + // The search measures the unfiltered member as one of its own candidates + // and returns the winner's bytes, so encoding the member plainly here as + // well was a second full pass over every binary member on the default + // settings. Text goes nowhere near the search, so it still needs one. + let text = is_text_ppmd_candidate(data); + let searching = !text && crate::filter_search::search_applies(data); + let mut best = EncodedPayload { + data: if searching { + let mut last = 0usize; + crate::filter_search::choose_filter( + &Rar29Search, + data, + options, + Some(&mut |event| match event { + crate::filter_search::EncodeProgress::PassStarted => { + last = 0; + !progress.is_some_and(WorkTracker::is_cancelled) + } + crate::filter_search::EncodeProgress::Advanced(position) => { + let delta = position.saturating_sub(last); + last = position; + progress.is_none_or(|work| work.advance(delta as u64)) + } + }), + )? + .1 + } else { + with_codec_progress(progress, |report| { + unpack29_encode_literals_with_options_and_progress(data, options, report) + })? + }, + method: lz_method, + }; + // Whichever engine wins, it wins because it was measured against the other + // over the whole member. Size used to decide it instead: under 1 MiB text + // was measured, over 16 MiB it went to PPMd unmeasured, and everything + // between kept the LZ bytes without ever encoding PPMd to compare. That + // middle band cost 24% on 2 MiB of man pages and 12% on 4 MiB, and the + // unmeasured end can lose just as badly the other way: 8 MiB of content + // that repeats every 4 MiB packs to 843 KB under LZ and 1.44 MB under PPMd. + // + // Screening on a sample first, the way the filter search earns its + // candidates, was tried and does not work here. The two engines are only + // comparable once there is enough output to compare: on 128 KiB of one + // repeating phrase LZ lands on 213 bytes and PPMd on 118, so the screen + // reads a 45% PPMd win off what is almost entirely archive overhead. And + // in the case worth catching, repetition further apart than the sample is + // wide, the sample cannot see the repetition at all. PPMd costs about + // three times LZ on the same bytes, so text at the top two levels pays for + // both engines. Levels 1 to 4 skip the trial as they always have. + if include_ppmd && text { + let ppmd = EncodedPayload { + data: with_codec_progress(progress, |report| { + crate::codec::rar29::unpack29_encode_ppmd_with_progress( + data, + true, + None, + options.max_match_distance, + report, + ) + })?, + method: lz_method, + }; + if ppmd.data.len() < best.data.len() { + best = ppmd; + } + // The escape tokens price themselves against costs measured off the + // stream they are shaping, so a member where every escape loses can + // still ratify a few before the measurements catch up. A pure-literal + // stream is the direct check. It costs a second PPMd pass over the + // member, paid only where PPMd is already being tried: text at the + // top two levels. + let ppmd_literals = EncodedPayload { + data: with_codec_progress(progress, |report| { + crate::codec::rar29::unpack29_encode_ppmd_with_progress( + data, false, None, 0, report, + ) + })?, + method: lz_method, + }; + if ppmd_literals.data.len() < best.data.len() { + best = ppmd_literals; + } + } + if best.data.len() >= data.len() { + return Ok(EncodedPayload { + data: data.to_vec(), + method: 0x30, + }); + } + Ok(best) +} + +fn is_text_ppmd_candidate(data: &[u8]) -> bool { + let mut printable = 0usize; + let mut nul = 0usize; + let mut total = 0usize; + for start in text_sample_offsets(data.len()) { + let end = start.saturating_add(RAR29_TEXT_SAMPLE_SIZE).min(data.len()); + let (window_printable, window_nul, window_total) = score_text_window(&data[start..end]); + printable += window_printable; + nul += window_nul; + total += window_total; + } + total != 0 && nul * 100 <= total && printable * 100 >= total * 85 +} + +/// Counts one sampled window as printable, NUL and total bytes. +/// +/// Counting only ASCII read every non-English member as binary. The bench's +/// text member is a set of translated man pages, 23% of it well-formed +/// multibyte UTF-8, and it scored 77% against the 85% bar and went to LZ. A +/// multibyte sequence is as much text as an ASCII letter. Accepting them costs +/// no precision either, because random binary almost never forms one: a lead +/// byte followed by a continuation byte is about 3% of positions, nowhere near +/// enough to carry a binary member over the bar. +fn score_text_window(window: &[u8]) -> (usize, usize, usize) { + let mut printable = 0usize; + let mut nul = 0usize; + let mut total = 0usize; + let mut index = 0usize; + while index < window.len() { + let byte = window[index]; + if byte == 0 { + let run = window[index..] + .iter() + .take_while(|&&byte| byte == 0) + .count(); + index += run; + // A run this long is padding rather than content, and padding says + // nothing about what surrounds it, so it is left out of the count + // instead of counted against the member. A tar pads every file to a + // 512 byte block and ends with a block of nothing, which put 27% NUL + // in front of a 1% bar and sent a tar of source files to LZ as if it + // were binary. What is left either side of the padding still decides + // it: a tar of text reads 99.8% text, and an ELF only climbs from + // 34% to 38%, nowhere near the bar. + if run < RAR29_NUL_RUN_IS_PADDING { + nul += run; + total += run; + } + } else if byte.is_ascii_graphic() || matches!(byte, b'\n' | b'\r' | b'\t' | b' ') { + printable += 1; + total += 1; + index += 1; + } else if let Some(len) = utf8_sequence_len(window, index) { + printable += len; + total += len; + index += len; + } else if utf8_lead_len(byte).is_some_and(|len| index + len > window.len()) { + // The window cut a sequence in half. That is an artefact of where + // the sample lands, not evidence of binary, so the window ends here. + break; + } else { + total += 1; + index += 1; + } + } + (printable, nul, total) +} + +/// How many bytes the sequence starting with this byte claims, if it can start +/// one. Overlong forms (`0xc0`, `0xc1`) and anything past U+10FFFF cannot. +fn utf8_lead_len(byte: u8) -> Option { + match byte { + 0xc2..=0xdf => Some(2), + 0xe0..=0xef => Some(3), + 0xf0..=0xf4 => Some(4), + _ => None, + } +} + +/// The length of the well-formed UTF-8 sequence at `index`, if there is one. +fn utf8_sequence_len(window: &[u8], index: usize) -> Option { + let len = utf8_lead_len(window[index])?; + if index + len > window.len() { + return None; + } + window[index + 1..index + len] + .iter() + .all(|byte| (0x80..=0xbf).contains(byte)) + .then_some(len) +} + +fn text_sample_offsets(len: usize) -> [usize; 3] { + [ + 0, + len.saturating_sub(RAR29_TEXT_SAMPLE_SIZE) / 2, + len.saturating_sub(RAR29_TEXT_SAMPLE_SIZE), + ] +} + +fn is_audio_filter_candidate(data: &[u8], channels: usize) -> bool { + if channels == 0 || channels > 4 || data.len() < channels * 64 { + return false; + } + + let mut total_delta = 0usize; + let mut small_delta = 0usize; + let mut compared = 0usize; + for start in text_sample_offsets(data.len()) { + // The admission check leaves at least 64 samples per channel, even + // for the shortest window, so every window has comparisons to make. + let end = start + .saturating_add(RAR29_AUDIO_SAMPLE_SIZE) + .min(data.len()); + let aligned_start = start + ((channels - start % channels) % channels); + for channel in 0..channels { + let mut previous = None; + let mut index = aligned_start + channel; + while index < end { + let byte = data[index]; + if let Some(previous) = previous { + let delta = usize::from(byte.abs_diff(previous)); + let delta = delta.min(256 - delta); + total_delta += delta; + small_delta += usize::from(delta <= 8); + compared += 1; + } + previous = Some(byte); + index += channels; + } + } + } + + total_delta <= compared * 24 && small_delta * 100 >= compared * 55 +} + +/// One member, however the caller supplied it. +struct Member<'a> { + unicode_name: Option<&'a [u8]>, + unpack_version: Option, + name: &'a [u8], + bytes: MemberBytes<'a>, + file_time: u32, + file_attr: u32, + host_os: u8, + password: Option<&'a [u8]>, + file_comment: Option<&'a [u8]>, + extended_times: Option<&'a [u8]>, + is_directory: bool, + is_symlink: bool, +} + +impl<'a> Member<'a> { + fn from_stored(entry: &'a StoredEntry<'a>) -> Self { + Self { + name: entry.name, + bytes: MemberBytes::Borrowed(entry.data), + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + password: entry.password, + file_comment: entry.file_comment, + unicode_name: None, + unpack_version: None, + extended_times: None, + is_directory: false, + is_symlink: false, + } + } + + fn from_file(entry: &'a FileEntry<'a>) -> Self { + Self { + name: entry.name, + bytes: MemberBytes::Borrowed(entry.data), + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + password: entry.password, + file_comment: entry.file_comment, + unicode_name: None, + unpack_version: None, + extended_times: None, + is_directory: false, + is_symlink: false, + } + } + + fn from_streaming(entry: &'a StreamingEntry) -> Self { + Self { + name: &entry.name, + bytes: MemberBytes::Source(&entry.source), + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + password: entry.password.as_deref(), + file_comment: entry.file_comment.as_deref(), + unicode_name: None, + unpack_version: None, + extended_times: None, + // Streaming callers have no retained metadata wrapper. The native + // attribute word already distinguishes directory sources. + is_directory: if entry.host_os == 3 { + entry.file_attr & 0o170000 == 0o040000 + } else { + entry.file_attr & 0x10 != 0 + }, + is_symlink: false, + } + } + + fn unpacked_size(&self) -> Result { + usize::try_from(self.bytes.len()?) + .map_err(|_| Error::InvalidArgument("RAR 1.5 writer does not support large files")) + } +} + +struct EncodedMember<'a> { + payload: MemberPayload<'a>, + method: u8, + unpacked_size: usize, + file_crc: u32, +} + +/// Working memory one member needs while it is coded. +/// +/// This is the admission weight that decides how many members run at once, not +/// a prediction: the encoder holds the member, its packed output, and a match +/// finder sized by the block it works in. +fn member_workspace(options: WriterOptions, unpacked: u64, compressing: bool) -> u64 { + if !compressing { + return 1024 * 1024; + } + let dictionary = + dictionary_size_for_options(options).unwrap_or(RAR29_MAX_DICTIONARY_SIZE) as u64; + unpacked + .saturating_mul(2) + .saturating_add(dictionary.saturating_mul(12)) + .saturating_add(2 * 1024 * 1024) +} + +/// Codes one member, holding its bytes only while it is being coded. +fn encode_member<'a>( + member: &Member<'a>, + options: WriterOptions, + coding: &MemberCoding, + solid_encoder: &mut Option, + resources: &WriterResources, + progress: &WorkTracker<'_>, +) -> Result> { + progress.check()?; + let unpacked_size = member.unpacked_size()?; + validate_member(member.name, unpacked_size)?; + if member.is_directory { + if unpacked_size != 0 { + return Err(Error::InvalidArgument("directory source must be empty")); + } + return Ok(EncodedMember { + payload: MemberPayload::Packed(Vec::new()), + method: 0x30, + unpacked_size: 0, + file_crc: 0, + }); + } + let _permit = resources.acquire_serialising_cancellable( + member_workspace(options, unpacked_size as u64, coding.compresses()), + &|| progress.is_cancelled(), + )?; + + // Legacy link targets must not become dependencies of later solid data: + // reference readers handle links separately from the solid unpacker. + let coding = if member.is_symlink { + if options.features.solid { + *solid_encoder = Some(SolidEncoder::for_target(options)?); + } + &MemberCoding::Stored + } else { + coding + }; + + // A stored member never needs to be resident: checksum it from its source + // and let the writer copy it straight through. + if let (MemberCoding::Stored, None, Some(source)) = + (coding, member.password, member.bytes.source()) + { + return Ok(EncodedMember { + payload: MemberPayload::Copied(source), + method: 0x30, + unpacked_size, + file_crc: { + let mut crc = Crc32::new(); + member.bytes.walk_exact_with_progress( + unpacked_size as u64, + progress.reporter(), + |chunk| { + crc.update(chunk); + progress.advance(chunk.len() as u64); + }, + )?; + crc.finish() + }, + }); + } + + let data = member + .bytes + .load_exact_with_progress(unpacked_size as u64, progress.reporter())?; + let mut crc = Crc32::new(); + for chunk in data.chunks(64 * 1024) { + progress.check()?; + crc.update(chunk); + if !coding.compresses() { + progress.advance(chunk.len() as u64); + } + } + let file_crc = crc.finish(); + progress.check()?; + let payload = match coding { + MemberCoding::Stored => EncodedPayload { + data: data.into_owned(), + method: 0x30, + }, + MemberCoding::Compressed => { + encode_or_store_payload(&data, options, solid_encoder, Some(progress))? + } + MemberCoding::Filtered(policy) => { + encode_filtered_payload(&data, policy, options, solid_encoder, Some(progress))? + } + }; + Ok(EncodedMember { + payload: MemberPayload::Packed(payload.data), + method: payload.method, + unpacked_size, + file_crc, + }) +} + +/// Writes one member's header and payload, encrypting either if asked. +fn write_member( + output: &mut dyn Write, + member: &Member<'_>, + encoded: EncodedMember<'_>, + options: WriterOptions, + solid_continuation: bool, + header_password: Option<&[u8]>, + progress: Option>, +) -> Result<()> { + let target = options.target; + crate::write_progress::check_cancelled(progress)?; + let (payload, salt) = match encoded.payload { + MemberPayload::Packed(mut packed) => { + let salt = + encrypt_packed_data_with_progress(&mut packed, target, member.password, progress)?; + (MemberPayload::Packed(packed), salt) + } + // Unencrypted by construction, so the stored bytes are their own + // payload and their size is the member's. + copied => (copied, None), + }; + let packed_size = payload.size(encoded.unpacked_size as u64); + let packed_size = usize::try_from(packed_size) + .map_err(|_| Error::InvalidArgument("RAR 1.5 packed size overflows u32"))?; + let mut flags = writer_file_flags(member.password, member.file_comment, solid_continuation); + if salt.is_some() { + flags |= FHD_SALT; + } + let wire_name = member.unicode_name.unwrap_or(member.name); + if member.unicode_name.is_some() { + flags |= super::FHD_UNICODE; + } + let mut extra = encode_file_comment(member.file_comment)?; + if let Some(raw) = member.extended_times { + flags |= FHD_EXTTIME; + extra.extend_from_slice(raw); + } + let mut header = Vec::new(); + write_file_header( + &mut header, + &FileRecord { + head_type: FILE_HEAD, + name: wire_name, + unpacked_size: encoded.unpacked_size, + file_crc: encoded.file_crc, + packed_size, + file_time: member.file_time, + file_attr: member.file_attr, + host_os: member.host_os, + target, + method: encoded.method, + dictionary_flags: if member.is_directory && target != ArchiveVersion::Rar15 { + FHD_DIRECTORY_MASK + } else { + dictionary_flags_for_options(options)? + }, + flags, + salt, + extra: &extra, + }, + )?; + if let Some(version) = member.unpack_version { + header[24] = version; + write_file_header_crc(&mut header, 0, wire_name.len(), flags); + } + match header_password { + Some(password) => write_encrypted_header(output, &header, password)?, + None => output.write_all(&header)?, + } + if matches!(payload, MemberPayload::Copied(_)) { + let mut checksum = Crc32::new(); + payload.write_to( + &mut CrcWriter { + inner: output, + crc: &mut checksum, + }, + packed_size as u64, + )?; + if checksum.finish() != encoded.file_crc { + return Err(Error::SourceChanged( + "entry source contents changed while writing", + )); + } + Ok(()) + } else { + payload.write_to(output, packed_size as u64) + } +} + +pub fn write_stored_volumes( + entry: StoredEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, +) -> Result>> { + write_stored_volumes_with_progress(entry, options, max_packed_per_volume, None) +} + +pub(crate) fn write_stored_volumes_with_progress( + entry: StoredEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, + progress: Option<&dyn WriteProgress>, +) -> Result>> { + let resources = WriterResources::default(); + let control = + crate::write_progress::ResourceProgress::new(&resources, progress.map(ProgressReporter)); + let progress = ProgressReporter(&control); + crate::write_progress::check_cancelled(Some(progress))?; + validate_plan(options, PlanShape::new().volumes(true), false, false)?; + validate_volume_writer_inputs(entry.name, entry.data, entry.file_comment, options)?; + if options.features.header_encryption { + return write_header_encrypted_split_volumes(SplitVolumeRecord { + progress, + name: entry.name, + unpacked: entry.data, + packed: entry.data, + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + target: options.target, + method: 0x30, + dictionary_flags: dictionary_flags_for_options(options)?, + base_flags: writer_file_flags(entry.password, None, false), + main_flags: 0, + password: entry.password, + max_packed_per_volume, + }); + } + + write_split_volumes(SplitVolumeRecord { + progress, + name: entry.name, + unpacked: entry.data, + packed: entry.data, + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + target: options.target, + method: 0x30, + dictionary_flags: dictionary_flags_for_options(options)?, + base_flags: writer_file_flags(entry.password, None, false), + main_flags: 0, + password: entry.password, + max_packed_per_volume, + }) +} + +pub fn write_compressed_volumes( + entry: FileEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, +) -> Result>> { + write_compressed_volumes_with_progress(entry, options, max_packed_per_volume, None) +} + +pub fn write_compressed_volumes_with_progress( + entry: FileEntry<'_>, + options: WriterOptions, + max_packed_per_volume: usize, + progress: Option<&dyn WriteProgress>, +) -> Result>> { + let resources = WriterResources::default(); + let control = + crate::write_progress::ResourceProgress::new(&resources, progress.map(ProgressReporter)); + let reporting = &control as &dyn WriteProgress; + let progress = Some(reporting); + crate::write_progress::check_cancelled(progress.map(ProgressReporter))?; + let total_work = if options.target == ArchiveVersion::Rar20 && !options.features.solid { + (entry.data.len() as u64).saturating_mul(2) + } else { + entry.data.len() as u64 + }; + report_compression_operation(reporting, true, total_work, 1); + let work = WorkTracker::new( + Some(ProgressReporter(reporting)), + WriteOperation::Compression, + total_work, + ); + let result = write_compressed_volumes_impl(entry, options, max_packed_per_volume, &work); + let result = result?; + if !work.finish() { + return Err(Error::Cancelled); + } + report_compression_operation(reporting, false, total_work, 1); + work.check()?; + Ok(result) +} + +fn write_compressed_volumes_impl( + entry: FileEntry<'_>, + mut options: WriterOptions, + max_packed_per_volume: usize, + progress: &WorkTracker<'_>, +) -> Result>> { + validate_plan( + options, + PlanShape::new().compressed(true).volumes(true), + false, + false, + )?; + validate_volume_writer_inputs(entry.name, entry.data, entry.file_comment, options)?; + // A volume set holds one member, and splitting it changes nothing about + // how far back the window has to reach. + if options.dictionary_size.is_none() { + options.dictionary_size = Some(fitted_dictionary_size( + options.target, + entry.data.len() as u64, + )); + } + + let mut solid_encoder = None; + let payload = encode_or_store_payload(entry.data, options, &mut solid_encoder, Some(progress)) + .map_err(|error| { + crate::write_stream::member_error(error, entry.name, "compressing volume member") + })?; + if options.features.header_encryption { + return write_header_encrypted_split_volumes(SplitVolumeRecord { + progress: progress + .reporter() + .expect("volume compression has a resource reporter"), + name: entry.name, + unpacked: entry.data, + packed: &payload.data, + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + target: options.target, + method: payload.method, + dictionary_flags: dictionary_flags_for_options(options)?, + base_flags: writer_file_flags(entry.password, None, false), + main_flags: if options.features.solid { MHD_SOLID } else { 0 }, + password: entry.password, + max_packed_per_volume, + }); + } + + write_split_volumes(SplitVolumeRecord { + progress: progress + .reporter() + .expect("volume compression has a resource reporter"), + name: entry.name, + unpacked: entry.data, + packed: &payload.data, + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + target: options.target, + method: payload.method, + dictionary_flags: dictionary_flags_for_options(options)?, + base_flags: writer_file_flags(entry.password, None, false), + main_flags: if options.features.solid { MHD_SOLID } else { 0 }, + password: entry.password, + max_packed_per_volume, + }) +} + +fn report_compression_operation( + progress: &dyn WriteProgress, + started: bool, + total_bytes: u64, + total_entries: usize, +) { + if started { + progress.report(WriteProgressEvent::OperationStarted { + operation: WriteOperation::Compression, + total_bytes: Some(total_bytes), + total_entries: Some(total_entries), + pass: 1, + }); + } else { + progress.report(WriteProgressEvent::OperationFinished { + operation: WriteOperation::Compression, + total_bytes: Some(total_bytes), + total_entries: Some(total_entries), + pass: 1, + }); + } +} + +/// Everything this writer refuses, in one place, before anything is written. +/// +/// The three near-identical checks this replaced each built a set of allowed +/// features out of the target and the request together, compared it with the +/// request, and on a mismatch said only that some feature was unsupported. +fn validate_plan( + options: WriterOptions, + shape: PlanShape, + has_archive_comment: bool, + has_file_comment: bool, +) -> Result<()> { + if options.archive_comment_metadata.is_some() + && (!has_archive_comment + || !matches!( + options.target, + ArchiveVersion::Rar30 | ArchiveVersion::Rar40 + )) + { + return Err(Error::InvalidArgument( + "retained archive comment metadata requires a RAR3/4 archive comment", + )); + } + if options.target.family() != crate::version::ArchiveFamily::Rar15To40 { + return Err(Error::UnsupportedVersion(options.target)); + } + crate::write_plan::validate_features(options.target, options.features, shape)?; + crate::write_plan::validate_compression_level(options.target, options.compression_level)?; + if options.method == Rar29Method::Ppmd { + crate::write_plan::validate_option(options.target, WriterOption::CompressionMethod, shape)?; + } + // A filter policy routes the member through the RAR 2.9 engine, so the + // formats without one have to be turned away here. This used to be + // computed into the shape and then never asked about, which let RAR 1.5 + // accept a policy and write an archive that failed its own checksum. + if shape.filtered { + crate::write_plan::validate_option(options.target, WriterOption::Filter, shape)?; + } + let _ = dictionary_flags_for_options(options)?; + if has_archive_comment { + crate::write_plan::validate_option(options.target, WriterOption::ArchiveComment, shape)?; + } + if has_file_comment { + crate::write_plan::validate_option(options.target, WriterOption::FileComment, shape)?; + } + Ok(()) +} + +/// Cross-flag rules: relations between options rather than per-option +/// capability, so they are checked after the capability table has had its say. +fn validate_header_encrypted_archive_options( + target: ArchiveVersion, + has_password: bool, +) -> Result<()> { + if !has_password { + return Err(Error::UnsupportedWriterOption { + target, + option: WriterOption::Feature(crate::Feature::HeaderEncryption), + because: Some("without a password"), + }); + } + Ok(()) +} + +fn rar29_encode_options_for_level(level: Option) -> Result { + let level = level.unwrap_or(5); + let candidates = match level { + 0 => 0, + 1 => 8, + 2 => 32, + 3 => 64, + 4 => 96, + 5 => 128, + _ => { + return Err(Error::InvalidArgument( + "RAR compression level must be in the range 0..5", + )) + } + }; + Ok(Rar29EncodeOptions::new(candidates) + .with_lazy_matching(level >= 4) + .with_lazy_lookahead(1) + .with_block_size(RAR29_LZ_BLOCK_SIZE)) +} + +fn rar29_encode_options_for_options(options: WriterOptions) -> Result { + Ok(rar29_encode_options_for_level(options.compression_level)? + .with_max_match_distance(dictionary_size_for_options(options)?)) +} + +/// RAR 2.0 levels, as candidates searched per position. +/// +/// The ladder used to run 16, 64, 256, 96, 128, so the top two levels searched +/// less than the middle one and asking for more compression got less of it: on +/// 4 MiB of man pages level 3 packed 848,854 bytes, level 5 packed 861,371 and +/// level 4 packed 867,857, which is the candidate order exactly. +/// +/// Lazy matching was off at every level, which cost more than the ladder did. +/// Measured on that member, packed bytes and seconds: +/// +/// ```text +/// candidates 16 64 256 512 1024 +/// lazy off 937,906 878,958 848,790 840,496 835,622 +/// lazy on 886,854 837,531 813,252 806,728 803,006 +/// seconds 2.37 4.41 8.67 12.78 17.43 +/// ``` +/// +/// It is worth 4% on text and 3% on a stripped binary, at every count, for +/// about half again the time. Candidates cost more and give less: the last +/// doubling buys 0.5%. So lazy matching is on wherever anything is compressed, +/// and the level chooses how far to search. +fn rar20_encode_options_for_level(level: Option) -> Result { + let level = level.unwrap_or(RAR20_DEFAULT_LEVEL); + let candidates = match level { + 0 => return Ok(Rar20EncodeOptions::new(0)), + 1 => 16, + 2 => 64, + 3 => 256, + 4 => 512, + 5 => 1024, + _ => { + return Err(Error::InvalidArgument( + "RAR compression level must be in the range 0..5", + )) + } + }; + Ok(Rar20EncodeOptions::new(candidates) + .with_lazy_matching(true) + // Look two positions ahead before committing to a match, not one. The + // class this moves is downloads: docx, jpeg, png, gif, a ZX Spectrum + // snapshot. High-entropy bytes with occasional short matches, where a + // three byte match often costs more than the literal it replaces and + // the better match starts two bytes later. Depth 1 cannot see that far. + // Measured on the small corpus, packed bytes: + // + // ```text + // depth 1 2 3 4 + // total 657,072 654,162 656,242 659,161 + // ``` + // + // Worth 0.44% at every level for about a third more encode time. Depth + // 3 and 4 give it back: on text, deferring that long skips matches + // worth taking. + .with_lazy_lookahead(2) + // The shortest-path parse is worth another 1.5% and costs four times + // the encode time, so it goes where the ladder already means "spend + // whatever it takes". rar29 gates its lazy matching at the same rung. + .with_optimal_parse(level >= 4) + // The audio trial is a second encode of the member, which is the one + // thing level 1 is trying not to pay for. + .with_try_audio(level > 1)) +} + +fn rar20_encode_options_for_options(options: WriterOptions) -> Result { + Ok(rar20_encode_options_for_level(options.compression_level)? + .with_max_match_distance(dictionary_size_for_options(options)?)) +} + +fn rar15_encode_options_for_level(level: Option) -> Result { + let level = level.unwrap_or(5); + match level { + 0 => Ok(Rar15EncodeOptions::new() + .with_old_distance_tokens(false) + .with_lazy_matching(false) + .with_stmode_literal_runs(false) + .with_max_long_match_distance(0)), + 1 => Ok(Rar15EncodeOptions::new() + .with_old_distance_tokens(false) + .with_lazy_matching(false) + .with_stmode_literal_runs(false) + .with_max_long_match_distance(4 * 1024)), + 2 => Ok(Rar15EncodeOptions::new() + .with_lazy_matching(false) + .with_stmode_literal_runs(false) + .with_max_long_match_distance(8 * 1024)), + 3 => Ok(Rar15EncodeOptions::new() + .with_lazy_matching(false) + .with_max_long_match_distance(16 * 1024)), + 4 => Ok(Rar15EncodeOptions::new() + .with_lazy_matching(false) + .with_max_long_match_distance(24 * 1024)), + 5 => Ok(Rar15EncodeOptions::new().with_lazy_matching(false)), + _ => Err(Error::InvalidArgument( + "RAR compression level must be in the range 0..5", + )), + } +} + +/// The smallest and largest dictionary worth declaring for a target. +/// +/// RAR 1.5 and older ignore the field: their decoders have a fixed window, and +/// measuring confirms the packed size does not move with it, so those stay +/// pinned. WinRAR 1.54 writes 64K in every archive we have from it. +fn dictionary_range(target: ArchiveVersion) -> (usize, usize) { + match target { + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 => { + (128 * 1024, RAR29_MAX_DICTIONARY_SIZE) + } + ArchiveVersion::Rar20 => (64 * 1024, RAR29_MAX_DICTIONARY_SIZE), + _ => (64 * 1024, 64 * 1024), + } +} + +/// The smallest dictionary the format encodes that reaches past `content`, so a +/// match can span everything the window is meant to cover. +/// +/// Picking one number for every archive is the wrong shape: too small loses +/// half the ratio on content whose repeats sit far apart, and too large costs a +/// decoder memory it never needs for a thirty byte file. WinRAR sizes it to the +/// data, and reading the dictionary bits out of its own archives against their +/// largest member shows exactly this rule: +/// +/// ```text +/// content 130048 196608 262144 705644 1048576 +/// declared 128K 256K 512K 1024K 2048K +/// ``` +fn fitted_dictionary_size(target: ArchiveVersion, content: u64) -> usize { + let (floor, cap) = dictionary_range(target); + let mut size = floor; + while size < cap && size as u64 <= content { + size *= 2; + } + size +} + +fn rar29_default_dictionary_size(target: ArchiveVersion) -> usize { + match target { + ArchiveVersion::Rar29 => 1024 * 1024, + ArchiveVersion::Rar30 | ArchiveVersion::Rar40 => 128 * 1024, + _ => 64 * 1024, + } +} + +fn dictionary_size_for_options(options: WriterOptions) -> Result { + options + .dictionary_size + .map(Ok) + .unwrap_or_else(|| Ok(rar29_default_dictionary_size(options.target))) +} + +fn dictionary_flags_for_options(options: WriterOptions) -> Result { + dictionary_flags_for_size(dictionary_size_for_options(options)?) +} + +fn dictionary_flags_for_size(size: usize) -> Result { + let bits = + match size { + 0x1_0000 => 0, + 0x2_0000 => 1, + 0x4_0000 => 2, + 0x8_0000 => 3, + 0x10_0000 => 4, + 0x20_0000 => 5, + 0x40_0000 => 6, + _ => return Err(Error::InvalidArgument( + "RAR 1.5-4.x dictionary size must be one of 64K, 128K, 256K, 512K, 1M, 2M, or 4M", + )), + }; + Ok((bits as u16) << 5) +} + +// Callers have passed `validate_plan`, including compression-level validation. +fn compression_method_for_level(options: WriterOptions) -> u8 { + let Some(level) = options.compression_level else { + return 0x33; + }; + 0x30 + level +} + +enum SolidEncoder { + Rar15(Box), + Rar20(Unpack20Encoder), + Rar29(Box), +} + +impl SolidEncoder { + fn for_target(options: WriterOptions) -> Result { + let encoder = match options.target { + ArchiveVersion::Rar15 => Self::Rar15(Box::new(Unpack15Encoder::with_options( + rar15_encode_options_for_level(options.compression_level)?, + ))), + ArchiveVersion::Rar20 => Self::Rar20(Unpack20Encoder::with_options( + rar20_encode_options_for_options(options)?, + )), + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 => { + Self::Rar29(Box::new(Unpack29Encoder::with_options( + rar29_encode_options_for_options(options)?, + ))) + } + _ => return Err(Error::UnsupportedVersion(options.target)), + }; + Ok(encoder) + } +} + +struct EncodedPayload { + data: Vec, + method: u8, +} + +/// Codes a filtered member, continuing the solid chain when there is one. +/// +/// A filtered member used to be coded on its own whatever the archive said, +/// while its header still claimed to continue the chain. That cost the whole +/// benefit of `--solid`: three 400 KB members went from 550 KB to 1.2 MB, and +/// the flag on the members was a plain lie about how they had been coded. +fn encode_filtered_payload( + data: &[u8], + policy: &FilterPolicy, + options: WriterOptions, + solid_encoder: &mut Option, + progress: Option<&WorkTracker<'_>>, +) -> Result { + let lz_method = compression_method_for_level(options); + // A named filter under forced PPMd is the one request the chain cannot + // take: its PPMd side codes no filters, and coding the member LZ instead + // would quietly overrule a flag the caller set. + let forced_ppmd_filter = + options.method == Rar29Method::Ppmd && matches!(policy, FilterPolicy::Explicit(_)); + let encoder = match ( + lz_method != 0x30 && !forced_ppmd_filter, + solid_encoder.as_mut(), + ) { + (true, Some(SolidEncoder::Rar29(encoder))) => encoder, + _ => { + return encode_rar29_policy_filtered_payload( + data, + policy, + options.method, + rar29_encode_options_for_options(options)?, + lz_method, + ppmd_trial_pays(lz_method.saturating_sub(0x30)), + progress, + ) + } + }; + + // An empty member has no range to filter, and `--no-filter` asked for none, + // but both still go through the chain's encoder like any other member. + // Coding either one on its own would leave the encoder's history in place + // while the headers said the chain had ended, so the member after it would + // be coded against a dictionary its own flags told the decoder to discard. + let mut last = 0usize; + let mut advance = |position: usize| { + let delta = position.saturating_sub(last); + last = last.max(position); + progress.is_none_or(|work| work.advance(delta as u64)) + }; + let packed = match policy { + FilterPolicy::Explicit(filter) if !data.is_empty() => encoder + .encode_member_with_filters_and_progress( + data, + std::slice::from_ref(filter), + Some(&mut advance), + ) + .map_err(map_codec_cancel)?, + // Naming a filter, or none, says nothing about which engine should code + // the member. `--no-filter` used to answer that question too, and answer + // it LZ, so asking a solid archive not to filter its binaries also took + // PPMd away from its text. + FilterPolicy::Explicit(_) | FilterPolicy::None => { + encode_rar29_chain_member(data, options, encoder, false, &mut advance)? + } + FilterPolicy::Auto => { + encode_rar29_chain_member(data, options, encoder, true, &mut advance)? + } + }; + + // A member the encoder could not shrink is stored, exactly as an unfiltered + // one is, which rebuilds the encoder and ends the chain here. + if should_store_fallback(options.target, true, data.len(), packed.len()) { + *solid_encoder = Some(SolidEncoder::for_target(options)?); + return Ok(EncodedPayload { + data: data.to_vec(), + method: 0x30, + }); + } + Ok(EncodedPayload { + data: packed, + method: lz_method, + }) +} + +fn encode_or_store_payload( + data: &[u8], + options: WriterOptions, + solid_encoder: &mut Option, + progress: Option<&WorkTracker<'_>>, +) -> Result { + let target = options.target; + if options.compression_level == Some(0) { + return Ok(EncodedPayload { + data: data.to_vec(), + method: 0x30, + }); + } + let solid = solid_encoder.is_some(); + if !solid + && matches!( + target, + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 + ) + { + let encode_options = rar29_encode_options_for_options(options)?; + let lz_method = compression_method_for_level(options); + // Forcing PPMd leaves nothing for a filter search to measure against. + let policy = if options.method == Rar29Method::Ppmd { + FilterPolicy::None + } else { + FilterPolicy::Auto + }; + return encode_rar29_policy_filtered_payload( + data, + &policy, + options.method, + encode_options, + lz_method, + ppmd_trial_pays(lz_method.saturating_sub(0x30)), + progress, + ); + } + let compressed = encode_compressed_payload(data, options, solid_encoder.as_mut(), progress)?; + if should_store_fallback(target, solid, data.len(), compressed.len()) { + if solid { + *solid_encoder = Some(SolidEncoder::for_target(options)?); + } + return Ok(EncodedPayload { + data: data.to_vec(), + method: 0x30, + }); + } + Ok(EncodedPayload { + data: compressed, + method: compression_method_for_level(options), + }) +} + +/// Codes one member into a RAR 2.9 solid chain, choosing both the engine and, +/// on the LZ side, the filter. +/// +/// The engine question and the filter question meet here because they are +/// asked of the same member and only one answer gets written. In practice they +/// barely overlap: the filter search declines text and the PPMd trial only +/// wants text, so a member that reaches one usually walks past the other. That +/// is a fact about the two screens rather than a rule, so the trials are +/// measured against each other rather than sequenced on the assumption. +/// +/// `search` is off for a caller who named a filter or asked for none, and for +/// PPMd, which leaves nothing for a filter to be measured through. +fn encode_rar29_chain_member( + data: &[u8], + options: WriterOptions, + encoder: &mut Unpack29Encoder, + search: bool, + progress: &mut dyn FnMut(usize) -> bool, +) -> Result> { + // A chain used to be LZ and nothing else, so a solid archive got no PPMd at + // all and an explicit --method ppmd was quietly ignored. It is the same + // content question here as in a non-solid archive, so it gets the same + // answer and the same gate: a level willing to encode the member twice, and + // content that looks like PPMd will win on it. + let level = compression_method_for_level(options).saturating_sub(0x30); + let engine = match options.method { + Rar29Method::Lz => ChainEngine::Lz, + Rar29Method::Ppmd => ChainEngine::Ppmd, + Rar29Method::Auto if ppmd_trial_pays(level) && is_text_ppmd_candidate(data) => { + ChainEngine::Smaller + } + Rar29Method::Auto => ChainEngine::Lz, + }; + let candidates = + if search && engine != ChainEngine::Ppmd && crate::filter_search::search_applies(data) { + crate::filter_search::filter_candidates( + &Rar29Search, + data, + rar29_encode_options_for_options(options)?, + )? + } else { + Vec::new() + }; + encoder + .encode_member_with_engine(data, engine, &candidates, progress) + .map_err(map_codec_cancel) +} + +fn encode_compressed_payload( + data: &[u8], + options: WriterOptions, + solid_encoder: Option<&mut SolidEncoder>, + progress: Option<&WorkTracker<'_>>, +) -> Result> { + let target = options.target; + let mut last = 0usize; + let mut advance = |position: usize| { + if position < last { + last = 0; + } + let delta = position.saturating_sub(last); + last = position; + progress.is_none_or(|progress| progress.advance(delta as u64)) + }; + // Non-solid RAR 2.9 members take the policy path in + // encode_or_store_payload before reaching this solid-chain encoder. + match (target, solid_encoder) { + (ArchiveVersion::Rar15, Some(SolidEncoder::Rar15(encoder))) => encoder + .encode_member_with_progress(data, &mut advance) + .map_err(map_codec_cancel), + (ArchiveVersion::Rar15, None) => unpack15_encode_with_options_and_progress( + data, + rar15_encode_options_for_level(options.compression_level)?, + &mut advance, + ) + .map_err(map_codec_cancel), + (ArchiveVersion::Rar20, None) => unpack20_encode_auto_with_options_and_progress( + data, + rar20_encode_options_for_options(options)?, + &mut advance, + ) + .map_err(map_codec_cancel), + (ArchiveVersion::Rar20, Some(SolidEncoder::Rar20(encoder))) => encoder + .encode_member_with_progress(data, &mut advance) + .map_err(map_codec_cancel), + ( + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40, + Some(SolidEncoder::Rar29(encoder)), + ) => encode_rar29_chain_member(data, options, encoder, true, &mut advance), + _ => Err(Error::UnsupportedVersion(target)), + } +} + +fn map_codec_cancel(error: crate::codec::Error) -> Error { + if error == crate::codec::Error::Cancelled { + Error::Cancelled + } else { + Error::from(error) + } +} + +fn with_codec_progress( + progress: Option<&WorkTracker<'_>>, + encode: impl FnOnce(&mut dyn FnMut(usize) -> bool) -> crate::codec::Result>, +) -> Result> { + if let Some(work) = progress { + work.check()?; + } + let mut last = 0usize; + let result = encode(&mut |position| { + if position < last { + last = 0; + } + let delta = position.saturating_sub(last); + last = position; + progress.is_none_or(|work| work.advance(delta as u64)) + }) + .map_err(map_codec_cancel)?; + if let Some(work) = progress { + work.check()?; + } + Ok(result) +} + +fn should_store_fallback( + target: ArchiveVersion, + solid: bool, + unpacked_len: usize, + packed_len: usize, +) -> bool { + // RAR 2.0 onwards store any member compression did not help. RAR 1.5 pays + // more header for a stored member, so a small one stays compressed. + let stores_any_size = !solid + && matches!( + target, + ArchiveVersion::Rar20 + | ArchiveVersion::Rar29 + | ArchiveVersion::Rar30 + | ArchiveVersion::Rar40 + ); + // Storing a member part way through a solid run rebuilds the encoder, so + // the member after it has to say it starts a fresh chain. Only RAR 2.0 + // onwards can: RAR 1.5 file headers have no solid bit, and readers take + // every member of a solid RAR 1.5 archive as a continuation whatever the + // header says. Breaking the chain there writes an archive nothing can read. + crate::write_plan::StoreFallback::new() + .allow_solid(target != ArchiveVersion::Rar15) + .min_size(if stores_any_size { + 0 + } else { + MIN_STORE_FALLBACK_SIZE + }) + .applies(solid, unpacked_len, packed_len) +} + +fn validate_volume_writer_inputs( + name: &[u8], + data: &[u8], + file_comment: Option<&[u8]>, + options: WriterOptions, +) -> Result<()> { + validate_file_entry(name, data).map_err(|error| { + crate::write_stream::member_error(error, name, "preparing volume member") + })?; + if file_comment.is_some() { + return Err(Error::UnsupportedFeature { + version: options.target, + feature: "volume_file_comment", + }); + } + Ok(()) +} + +fn writer_file_flags( + password: Option<&[u8]>, + file_comment: Option<&[u8]>, + solid_continuation: bool, +) -> u16 { + let mut flags = 0; + if password.is_some() { + flags |= FHD_PASSWORD; + } + if file_comment.is_some() { + flags |= FHD_COMMENT; + } + if solid_continuation { + flags |= FHD_SOLID; + } + flags +} + +/// Builds the comment block that goes inside a file header. +/// +/// RAR 1.3 and 1.4 wrote a bare size and the text, and this writer used to copy +/// that. From 1.5 on it is the same comment block the archive comment uses, +/// covered by the file header's size but not by its CRC. +fn encode_file_comment(comment: Option<&[u8]>) -> Result> { + let mut out = Vec::new(); + write_comment_header(&mut out, comment)?; + Ok(out) +} + +fn encrypt_split_packed_data( + data: &mut Vec, + target: ArchiveVersion, + password: &[u8], +) -> Result> { + #[cfg(not(feature = "encryption"))] + { + let _ = (data, target, password); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + match target { + ArchiveVersion::Rar15 => { + Rar15Cipher::new(password).crypt_in_place(data); + Ok(None) + } + ArchiveVersion::Rar20 => { + let padded_len = checked_align16(data.len(), RAR15_ALIGN_OVERFLOW) + .expect("byte vector length plus padding fits usize"); + data.resize(padded_len, 0); + Rar20Cipher::new(password) + .encrypt_in_place(data) + .expect("padded legacy ciphertext is block aligned"); + Ok(None) + } + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 => { + let salt = random_rar30_salt()?; + let padded_len = checked_align16(data.len(), RAR15_ALIGN_OVERFLOW) + .expect("byte vector length plus padding fits usize"); + data.resize(padded_len, 0); + Rar30Cipher::new(password, Some(salt)) + .map_err(super::map_rar30_crypto_error)? + .encrypt_in_place(data) + .expect("padded legacy ciphertext is block aligned"); + Ok(Some(salt)) + } + _ => Err(Error::UnsupportedVersion(target)), + } + } +} + +fn header_encryption_password<'a>( + mut passwords: impl Iterator>, +) -> Result<&'a [u8]> { + let first = passwords.next().flatten().ok_or(Error::NeedPassword)?; + for password in passwords { + if password != Some(first) { + return Err(Error::InvalidArgument( + "RAR 3.x header-encrypted writer needs one shared password", + )); + } + } + Ok(first) +} + +fn encrypt_packed_data_for_writer( + data: &mut Vec, + target: ArchiveVersion, + password: Option<&[u8]>, +) -> Result> { + encrypt_packed_data_with_progress(data, target, password, None) +} + +fn encrypt_packed_data_with_progress( + data: &mut Vec, + target: ArchiveVersion, + password: Option<&[u8]>, + progress: Option>, +) -> Result> { + #[cfg(not(feature = "encryption"))] + { + let _ = (data, target); + crate::write_progress::check_cancelled(progress)?; + if password.is_none() { + Ok(None) + } else { + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + } + #[cfg(feature = "encryption")] + { + crate::write_progress::check_cancelled(progress)?; + let Some(password) = password else { + return Ok(None); + }; + match target { + ArchiveVersion::Rar15 => { + let mut cipher = Rar15Cipher::new(password); + for chunk in data.chunks_mut(64 * 1024) { + crate::write_progress::check_cancelled(progress)?; + cipher.crypt_in_place(chunk); + } + Ok(None) + } + ArchiveVersion::Rar20 => { + let padded_len = checked_align16(data.len(), RAR15_ALIGN_OVERFLOW) + .expect("byte vector length plus padding fits usize"); + data.resize(padded_len, 0); + let mut cipher = Rar20Cipher::new(password); + for chunk in data.chunks_mut(64 * 1024) { + crate::write_progress::check_cancelled(progress)?; + cipher + .encrypt_in_place(chunk) + .expect("padded ciphertext and 64 KiB chunks are block aligned"); + } + Ok(None) + } + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 => { + let salt = random_rar30_salt()?; + // Byte vectors are bounded by isize::MAX, so adding 15 fits usize. + let padded_len = checked_align16(data.len(), RAR15_ALIGN_OVERFLOW) + .expect("byte vector length plus padding fits usize"); + data.resize(padded_len, 0); + let mut cipher = Rar30Cipher::new(password, Some(salt)) + .map_err(super::map_rar30_crypto_error)?; + for chunk in data.chunks_mut(64 * 1024) { + crate::write_progress::check_cancelled(progress)?; + cipher + .encrypt_in_place(chunk) + .expect("padded ciphertext and 64 KiB chunks are block aligned"); + } + Ok(Some(salt)) + } + _ => Err(Error::UnsupportedFeature { + version: target, + feature: "RAR writer file encryption", + }), + } + } +} + +#[cfg(feature = "encryption")] +fn random_rar30_salt() -> Result<[u8; 8]> { + let mut salt = [0; 8]; + crate::write_stream::fill_entropy( + &mut salt, + "RAR 3.x writer could not generate encryption salt", + )?; + Ok(salt) +} + +/// What WinRAR writes on an end-of-archive block. The bit marks the block as +/// one a reader may skip without understanding it. +const ENDARC_FLAGS: u16 = 0x4000; +const END_HEADER_SIZE: usize = 7; + +fn write_main_header(out: &mut Vec, flags: u16) { + let start = out.len(); + out.extend_from_slice(&0u16.to_le_bytes()); + out.push(MAIN_HEAD); + out.extend_from_slice(&flags.to_le_bytes()); + out.extend_from_slice(&13u16.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); + write_header_crc(out, start); +} + +fn write_comment_header(out: &mut Vec, comment: Option<&[u8]>) -> Result<()> { + let Some(comment) = comment else { + return Ok(()); + }; + let unp_size = u16::try_from(comment.len()) + .map_err(|_| Error::InvalidArgument("RAR 1.5 comment is longer than 65535 bytes"))?; + let head_size = 13usize + .checked_add(comment.len()) + .ok_or(Error::InvalidArgument( + "RAR 1.5 comment header size overflows", + ))?; + let head_size = u16::try_from(head_size) + .map_err(|_| Error::InvalidArgument("RAR 1.5 comment header size overflows"))?; + + let start = out.len(); + out.extend_from_slice(&0u16.to_le_bytes()); + out.push(COMM_HEAD); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&head_size.to_le_bytes()); + out.extend_from_slice(&unp_size.to_le_bytes()); + out.push(15); + out.push(0x30); + out.extend_from_slice(&((crc32(comment) & 0xffff) as u16).to_le_bytes()); + out.extend_from_slice(comment); + write_comment_header_crc(out, start); + Ok(()) +} + +fn uses_old_style_archive_comment(target: ArchiveVersion) -> bool { + matches!( + target, + ArchiveVersion::Rar15 | ArchiveVersion::Rar20 | ArchiveVersion::Rar29 + ) +} + +fn write_archive_comment( + out: &mut dyn Write, + comment: Option<&[u8]>, + options: WriterOptions, + header_password: Option<&[u8]>, + password: Option<&[u8]>, +) -> Result<()> { + let Some(comment) = comment else { + return Ok(()); + }; + if uses_old_style_archive_comment(options.target) { + let mut block = Vec::new(); + write_comment_header(&mut block, Some(comment))?; + out.write_all(&block)?; + return Ok(()); + } + let mut packed = unpack29_encode_literals(comment)?; + let salt = encrypt_packed_data_for_writer(&mut packed, options.target, password)?; + let metadata = options.archive_comment_metadata; + let mut header = Vec::new(); + write_file_header( + &mut header, + &FileRecord { + head_type: NEWSUB_HEAD, + name: b"CMT", + unpacked_size: comment.len(), + file_crc: crc32(comment), + packed_size: packed.len(), + file_time: metadata.map_or(0, |metadata| metadata.0), + file_attr: 0, + host_os: metadata.map_or(3, |metadata| metadata.1), + target: ArchiveVersion::Rar30, + method: 0x33, + dictionary_flags: dictionary_flags_for_target(ArchiveVersion::Rar30), + flags: if password.is_some() { + FHD_PASSWORD | FHD_SALT + } else { + 0 + }, + salt, + extra: &[], + }, + )?; + if let Some(password) = header_password { + write_encrypted_header(out, &header, password)?; + } else { + out.write_all(&header)?; + } + out.write_all(&packed)?; + Ok(()) +} + +/// Closes a header-encrypted archive with an end-of-archive block in a group of +/// its own. +/// +/// Every encrypted header carries its own salt, so a reader that reaches the end +/// of the last member's payload reads eight more bytes looking for the next +/// group's salt. With nothing there it stops on a short read rather than on a +/// clean end, and 7-Zip treats that as a decryption failure and refuses to open +/// the archive at all. WinRAR closes the same way. unrar stops at the last +/// member and never looks, which is why nothing caught this. +fn write_encrypted_end_block(out: &mut dyn Write, password: &[u8]) -> Result<()> { + let mut header = Vec::new(); + let start = header.len(); + header.extend_from_slice(&0u16.to_le_bytes()); + header.push(ENDARC_HEAD); + header.extend_from_slice(&ENDARC_FLAGS.to_le_bytes()); + header.extend_from_slice(&(END_HEADER_SIZE as u16).to_le_bytes()); + write_header_crc(&mut header, start); + debug_assert_eq!(header.len(), END_HEADER_SIZE); + write_encrypted_header(out, &header, password) +} + +fn write_encrypted_header(out: &mut dyn Write, header: &[u8], password: &[u8]) -> Result<()> { + #[cfg(not(feature = "encryption"))] + { + let _ = (out, header, password); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + let salt = random_rar30_salt()?; + let encrypted_size = checked_align16(header.len(), RAR15_ALIGN_OVERFLOW)?; + let mut encrypted_header = Vec::with_capacity(encrypted_size); + encrypted_header.extend_from_slice(header); + encrypted_header.resize(encrypted_size, 0); + Rar30Cipher::new(password, Some(salt)) + .map_err(super::map_rar30_crypto_error)? + .encrypt_in_place(&mut encrypted_header) + .map_err(super::map_rar30_crypto_error)?; + out.write_all(&salt)?; + out.write_all(&encrypted_header)?; + Ok(()) + } +} + +fn validate_member(name: &[u8], unpacked_size: usize) -> Result<()> { + if name.is_empty() { + return Err(Error::InvalidArgument("RAR 1.5 file name is empty")); + } + if name.len() > u16::MAX as usize { + return Err(Error::InvalidArgument("RAR 1.5 file name is too long")); + } + if unpacked_size > u32::MAX as usize { + return Err(Error::InvalidArgument( + "RAR 1.5 writer does not support large files", + )); + } + Ok(()) +} + +fn validate_file_entry(name: &[u8], data: &[u8]) -> Result<()> { + validate_member(name, data.len()) +} + +struct FileRecord<'a> { + head_type: u8, + name: &'a [u8], + unpacked_size: usize, + file_crc: u32, + packed_size: usize, + file_time: u32, + file_attr: u32, + host_os: u8, + target: ArchiveVersion, + method: u8, + dictionary_flags: u16, + flags: u16, + salt: Option<[u8; 8]>, + extra: &'a [u8], +} + +fn write_file_header_and_data( + out: &mut Vec, + record: FileRecord<'_>, + packed: &[u8], +) -> Result<()> { + write_file_header(out, &record)?; + out.extend_from_slice(packed); + Ok(()) +} + +fn write_file_header(out: &mut Vec, record: &FileRecord<'_>) -> Result<()> { + let start = out.len(); + let flags = record.flags | record.dictionary_flags; + let (host_os, file_attr) = + rar15_compatible_metadata(record.target, record.host_os, record.file_attr); + let packed_size = u32::try_from(record.packed_size) + .map_err(|_| Error::InvalidArgument("RAR 1.5 packed size overflows u32"))?; + let unpacked_size = u32::try_from(record.unpacked_size) + .map_err(|_| Error::InvalidArgument("RAR 1.5 unpacked size overflows u32"))?; + let head_size = 32usize + .checked_add(record.name.len()) + .and_then(|size| size.checked_add(if record.salt.is_some() { 8 } else { 0 })) + .and_then(|size| size.checked_add(record.extra.len())) + .ok_or(Error::InvalidArgument("RAR 1.5 file header size overflows"))?; + let head_size = u16::try_from(head_size) + .map_err(|_| Error::InvalidArgument("RAR 1.5 file header size overflows"))?; + let unp_ver = match record.target { + ArchiveVersion::Rar15 => 15, + ArchiveVersion::Rar20 => 20, + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 => 29, + _ => return Err(Error::UnsupportedVersion(record.target)), + }; + out.extend_from_slice(&0u16.to_le_bytes()); + out.push(record.head_type); + out.extend_from_slice(&(LONG_BLOCK | flags).to_le_bytes()); + out.extend_from_slice(&head_size.to_le_bytes()); + out.extend_from_slice(&packed_size.to_le_bytes()); + out.extend_from_slice(&unpacked_size.to_le_bytes()); + out.push(host_os); + out.extend_from_slice(&record.file_crc.to_le_bytes()); + out.extend_from_slice(&record.file_time.to_le_bytes()); + out.push(unp_ver); + out.push(record.method); + out.extend_from_slice(&(record.name.len() as u16).to_le_bytes()); + out.extend_from_slice(&file_attr.to_le_bytes()); + out.extend_from_slice(record.name); + if let Some(salt) = record.salt { + out.extend_from_slice(&salt); + } + out.extend_from_slice(record.extra); + write_file_header_crc(out, start, record.name.len(), flags); + Ok(()) +} + +fn rar15_compatible_metadata(target: ArchiveVersion, host_os: u8, file_attr: u32) -> (u8, u32) { + const HOST_DOS: u8 = 0; + const HOST_UNIX: u8 = 3; + const DOS_ARCHIVE: u32 = 0x20; + const DOS_DIRECTORY: u32 = 0x10; + const UNIX_FILE_TYPE_MASK: u32 = 0o170000; + const UNIX_DIRECTORY: u32 = 0o040000; + + if target == ArchiveVersion::Rar15 && host_os == HOST_UNIX { + let dos_attr = if file_attr & UNIX_FILE_TYPE_MASK == UNIX_DIRECTORY { + DOS_DIRECTORY + } else { + DOS_ARCHIVE + }; + return (HOST_DOS, dos_attr); + } + (host_os, file_attr) +} + +fn dictionary_flags_for_target(target: ArchiveVersion) -> u16 { + ((rar29_default_dictionary_size(target).trailing_zeros() - 16) as u16) << 5 +} + +struct SplitVolumeRecord<'a> { + progress: ProgressReporter<'a>, + name: &'a [u8], + unpacked: &'a [u8], + packed: &'a [u8], + file_time: u32, + file_attr: u32, + host_os: u8, + target: ArchiveVersion, + method: u8, + dictionary_flags: u16, + base_flags: u16, + main_flags: u16, + password: Option<&'a [u8]>, + max_packed_per_volume: usize, +} + +fn report_volume( + progress: ProgressReporter<'_>, + index: usize, + total: usize, + bytes: usize, +) -> Result<()> { + progress.report(WriteProgressEvent::VolumeFinished { + volume_number: index + 1, + total_volumes: Some(total), + bytes: bytes as u64, + }); + crate::write_progress::check_cancelled(Some(progress)) +} + +fn write_split_volumes(entry: SplitVolumeRecord<'_>) -> Result>> { + crate::write_progress::check_cancelled(Some(entry.progress))?; + if entry.max_packed_per_volume == 0 { + return Err(Error::InvalidArgument( + "RAR 1.5 volume payload size must be non-zero", + )); + } + if entry.packed.is_empty() { + return Err(Error::InvalidArgument( + "RAR 1.5 volume writer needs a non-empty packed payload", + ) + .at_entry(entry.name.to_vec(), "preparing volume member")); + } + + let mut packed = entry.packed.to_vec(); + let split_salt = if let Some(password) = entry.password { + encrypt_split_packed_data(&mut packed, entry.target, password).map_err(|error| { + crate::write_stream::member_error(error, entry.name, "encrypting volume member") + })? + } else { + None + }; + let base_flags = entry.base_flags | if split_salt.is_some() { FHD_SALT } else { 0 }; + + // Packing well enough to need only one volume is not an error. The loop + // below already writes that set correctly: with one chunk neither split + // flag is set, so the member is an ordinary unsplit file inside a volume + // set of one, which is what WinRAR writes and what RAR 5 here already did. + // Refusing it meant a fixed `--volume-size` in a script could start failing + // because compression improved, which is how it was found. + let chunks: Vec<&[u8]> = packed.chunks(entry.max_packed_per_volume).collect(); + + let mut volumes = Vec::with_capacity(chunks.len()); + let unpacked_crc = crc32(entry.unpacked); + for (index, chunk) in chunks.iter().enumerate() { + crate::write_progress::check_cancelled(Some(entry.progress))?; + let split_before = index > 0; + let split_after = index + 1 < chunks.len(); + let mut file_flags = base_flags; + if split_before { + file_flags |= FHD_SPLIT_BEFORE; + } + if split_after { + file_flags |= FHD_SPLIT_AFTER; + } + + let mut main_flags = MHD_VOLUME | entry.main_flags; + if index == 0 { + main_flags |= MHD_FIRSTVOLUME; + } + + let mut out = Vec::new(); + out.extend_from_slice(RAR15_SIGNATURE); + write_main_header(&mut out, main_flags); + write_file_header_and_data( + &mut out, + FileRecord { + head_type: FILE_HEAD, + name: entry.name, + unpacked_size: entry.unpacked.len(), + file_crc: if split_after { + crc32(chunk) + } else { + unpacked_crc + }, + packed_size: chunk.len(), + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + target: entry.target, + method: entry.method, + dictionary_flags: entry.dictionary_flags, + flags: file_flags, + salt: split_salt, + extra: &[], + }, + chunk, + ) + .map_err(|error| { + crate::write_stream::member_error(error, entry.name, "writing volume member") + })?; + report_volume(entry.progress, index, chunks.len(), out.len())?; + volumes.push(out); + } + + Ok(volumes) +} + +fn write_header_encrypted_split_volumes(entry: SplitVolumeRecord<'_>) -> Result>> { + crate::write_progress::check_cancelled(Some(entry.progress))?; + validate_header_encrypted_archive_options(entry.target, entry.password.is_some())?; + let password = entry.password.ok_or(Error::NeedPassword)?; + if entry.max_packed_per_volume == 0 { + return Err(Error::InvalidArgument( + "RAR 1.5 volume payload size must be non-zero", + )); + } + if entry.packed.is_empty() { + return Err(Error::InvalidArgument( + "RAR 1.5 volume writer needs a non-empty packed payload", + ) + .at_entry(entry.name.to_vec(), "preparing volume member")); + } + + let mut packed = entry.packed.to_vec(); + let split_salt = + encrypt_split_packed_data(&mut packed, entry.target, password).map_err(|error| { + crate::write_stream::member_error(error, entry.name, "encrypting volume member") + })?; + let base_flags = entry.base_flags | FHD_SALT; + // Packing well enough to need only one volume is not an error. The loop + // below already writes that set correctly: with one chunk neither split + // flag is set, so the member is an ordinary unsplit file inside a volume + // set of one, which is what WinRAR writes and what RAR 5 here already did. + // Refusing it meant a fixed `--volume-size` in a script could start failing + // because compression improved, which is how it was found. + let chunks: Vec<&[u8]> = packed.chunks(entry.max_packed_per_volume).collect(); + + let mut volumes = Vec::with_capacity(chunks.len()); + let unpacked_crc = crc32(entry.unpacked); + for (index, chunk) in chunks.iter().enumerate() { + crate::write_progress::check_cancelled(Some(entry.progress))?; + let split_before = index > 0; + let split_after = index + 1 < chunks.len(); + let mut file_flags = base_flags; + if split_before { + file_flags |= FHD_SPLIT_BEFORE; + } + if split_after { + file_flags |= FHD_SPLIT_AFTER; + } + + let mut main_flags = MHD_VOLUME | MHD_PASSWORD | entry.main_flags; + if index == 0 { + main_flags |= MHD_FIRSTVOLUME; + } + + let mut out = Vec::new(); + out.extend_from_slice(RAR15_SIGNATURE); + write_main_header(&mut out, main_flags); + let mut header = Vec::new(); + write_file_header( + &mut header, + &FileRecord { + head_type: FILE_HEAD, + name: entry.name, + unpacked_size: entry.unpacked.len(), + file_crc: if split_after { + crc32(chunk) + } else { + unpacked_crc + }, + packed_size: chunk.len(), + file_time: entry.file_time, + file_attr: entry.file_attr, + host_os: entry.host_os, + target: entry.target, + method: entry.method, + dictionary_flags: entry.dictionary_flags, + flags: file_flags, + salt: split_salt, + extra: &[], + }, + ) + .map_err(|error| { + crate::write_stream::member_error(error, entry.name, "writing volume member") + })?; + write_encrypted_header(&mut out, &header, password).map_err(|error| { + crate::write_stream::member_error(error, entry.name, "writing volume member") + })?; + out.extend_from_slice(chunk); + report_volume(entry.progress, index, chunks.len(), out.len())?; + volumes.push(out); + } + + Ok(volumes) +} + +fn write_header_crc(out: &mut [u8], start: usize) { + let crc = (crc32(&out[start + 2..]) & 0xffff) as u16; + out[start..start + 2].copy_from_slice(&crc.to_le_bytes()); +} + +fn write_file_header_crc(out: &mut [u8], start: usize, name_len: usize, flags: u16) { + let end = if flags & FHD_COMMENT != 0 { + // Readers stop the CRC where the fields they parse stop, which leaves + // out the comment block. Miss the salt out of the range and an + // encrypted member with a comment gets a CRC nothing agrees with. + let salt_len = if flags & FHD_SALT != 0 { 8 } else { 0 }; + start + 32 + name_len + salt_len + } else { + out.len() + }; + let crc = (crc32(&out[start + 2..end]) & 0xffff) as u16; + out[start..start + 2].copy_from_slice(&crc.to_le_bytes()); +} + +fn write_comment_header_crc(out: &mut [u8], start: usize) { + let end = start + 13; + let crc = (crc32(&out[start + 2..end]) & 0xffff) as u16; + out[start..start + 2].copy_from_slice(&crc.to_le_bytes()); +} + +#[cfg(test)] +mod tests { + #[test] + fn streaming_directory_sources_cannot_discard_payloads() { + for (host_os, attributes) in [(3, 0o040750), (2, 0x10)] { + let entry = super::StreamingEntry::new( + b"directory".to_vec(), + crate::EntrySource::from_bytes(b"must not be discarded".to_vec()), + ) + .with_host_os(host_os) + .with_file_attr(attributes); + let error = super::write_streaming_archive_to( + &[entry], + super::WriterOptions::new( + crate::ArchiveVersion::Rar30, + crate::FeatureSet::store_only(), + ), + crate::MemberCoding::Stored, + None, + &crate::WriterResources::default(), + None, + &mut Vec::new(), + ) + .unwrap_err(); + assert!(error.to_string().contains("directory source must be empty")); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_payload_padding_preserves_cipher_state_across_chunks() { + for target in [crate::ArchiveVersion::Rar20, crate::ArchiveVersion::Rar30] { + for length in [0usize, 1, 15, 16, 17, 65535, 65536, 65537] { + let plaintext: Vec = (0..length) + .map(|index| (index as u8).wrapping_mul(37)) + .collect(); + for split in [false, true] { + let mut data = plaintext.clone(); + let salt = if split { + super::encrypt_split_packed_data(&mut data, target, b"pw") + } else { + super::encrypt_packed_data_for_writer(&mut data, target, Some(b"pw")) + } + .unwrap(); + let padded = length.div_ceil(16) * 16; + assert_eq!(data.len(), padded); + match target { + crate::ArchiveVersion::Rar20 => { + assert_eq!(salt, None); + super::Rar20Cipher::new(b"pw") + .decrypt_in_place(&mut data) + .unwrap(); + } + crate::ArchiveVersion::Rar30 => { + assert!(salt.is_some()); + super::Rar30Cipher::new(b"pw", salt) + .unwrap() + .decrypt_in_place(&mut data) + .unwrap(); + } + _ => unreachable!(), + } + assert_eq!(&data[..length], plaintext); + assert!(data[length..].iter().all(|&byte| byte == 0)); + } + } + } + } + + #[test] + fn member_cancellation_during_workspace_admission_is_preserved() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CancelOnSecondPoll(AtomicUsize); + impl crate::WriteProgress for CancelOnSecondPoll { + fn report(&self, _: crate::WriteProgressEvent<'_>) {} + + fn is_cancelled(&self) -> bool { + self.0.fetch_add(1, Ordering::Relaxed) != 0 + } + } + + let entry = super::FileEntry { + name: b"file", + data: b"payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let callback = CancelOnSecondPoll(AtomicUsize::new(0)); + let work = super::WorkTracker::new( + Some(super::ProgressReporter(&callback)), + crate::WriteOperation::Compression, + 7, + ); + let resources = crate::WriterResources::new(1); + assert!(matches!( + super::encode_member( + &super::Member::from_file(&entry), + super::WriterOptions::new( + crate::ArchiveVersion::Rar29, + crate::FeatureSet::store_only(), + ), + &crate::MemberCoding::Stored, + &mut None, + &resources, + &work, + ), + Err(crate::Error::Cancelled) + )); + assert_eq!(callback.0.load(Ordering::Relaxed), 2); + assert_eq!(resources.workspace_in_use(), 0); + } + + #[test] + fn retained_archive_comment_metadata_requires_rar3_or_4_comment() { + let mut options = super::WriterOptions::new( + crate::ArchiveVersion::Rar30, + crate::FeatureSet::store_only(), + ); + options.archive_comment_metadata = Some((123, 0)); + let shape = crate::write_plan::PlanShape::new(); + assert_eq!( + super::validate_plan(options, shape, false, false).unwrap_err(), + crate::Error::InvalidArgument( + "retained archive comment metadata requires a RAR3/4 archive comment" + ) + ); + options.target = crate::ArchiveVersion::Rar20; + assert_eq!( + super::validate_plan(options, shape, true, false).unwrap_err(), + crate::Error::InvalidArgument( + "retained archive comment metadata requires a RAR3/4 archive comment" + ) + ); + options.target = crate::ArchiveVersion::Rar30; + assert!(super::validate_plan(options, shape, true, false).is_ok()); + } + + #[test] + fn writer_limits_and_invalid_options_are_not_archive_corruption() { + for error in [ + super::validate_member(b"", 0).unwrap_err(), + super::validate_member(&vec![b'a'; 65536], 0).unwrap_err(), + super::checked_align16(usize::MAX, "padding overflow").unwrap_err(), + super::write_comment_header(&mut Vec::new(), Some(&vec![0; 65536])).unwrap_err(), + super::header_encryption_password([Some(&b"one"[..]), Some(&b"two"[..])].into_iter()) + .unwrap_err(), + super::validate_rar29_filter_policy( + &super::FilterPolicy::Auto, + super::Rar29Method::Ppmd, + ) + .unwrap_err(), + super::rar29_encode_options_for_level(Some(6)).unwrap_err(), + super::rar20_encode_options_for_level(Some(6)).unwrap_err(), + super::rar15_encode_options_for_level(Some(6)).unwrap_err(), + ] { + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument, "{error}"); + } + assert_eq!(super::checked_align16(17, "padding overflow").unwrap(), 32); + assert_eq!( + super::header_encryption_password([None].into_iter()).unwrap_err(), + crate::Error::NeedPassword + ); + #[cfg(target_pointer_width = "64")] + assert_eq!( + super::validate_member(b"large", u32::MAX as usize + 1) + .unwrap_err() + .kind(), + crate::ErrorKind::InvalidArgument + ); + } + + use super::{ + encode_rar29_auto_filtered_member, encode_rar29_filtered_member, + encode_rar29_filtered_members, is_audio_filter_candidate, rar20_encode_options_for_level, + rar20_encode_options_for_options, rar29_encode_options_for_options, FilterKind, FilterSpec, + }; + use crate::codec::rar29::{unpack29_decode, EncodeOptions}; + + /// A member comfortably past the 1 MiB ceiling the PPMd trial used to have. + /// Text this size kept the LZ bytes without PPMd ever being encoded to + /// compare against, all the way up to 16 MiB. + const OVER_THE_OLD_TRIAL_LIMIT: usize = 1024 * 1024 + 4096; + use crate::filter_search::{ + auto_delta_filter_range, disjoint_filter_ranges, AUTO_DELTA_EDGE_SKIP, + }; + use crate::x86_filter_scan::auto_x86_filter_ranges; + use crate::{ArchiveVersion, FeatureSet}; + + #[test] + fn auto_x86_filter_ranges_select_dense_opcode_clusters() { + let mut data = vec![0x41; 20_000]; + for pos in [1024, 1050, 1090, 1130] { + data[pos] = 0xe8; + } + for pos in [12_000, 12_040, 12_080] { + data[pos] = 0xe9; + } + + let e8_ranges = auto_x86_filter_ranges(&data, false); + assert_eq!(e8_ranges.len(), 1); + assert!(e8_ranges[0].contains(&1024)); + assert!(e8_ranges[0].contains(&(1130 + 4))); + assert!(!e8_ranges[0].contains(&12_000)); + + let e8e9_ranges = auto_x86_filter_ranges(&data, true); + assert_eq!(e8e9_ranges.len(), 3); + assert!(e8e9_ranges[0].contains(&1024)); + assert!(e8e9_ranges[0].contains(&12_000)); + assert!(e8e9_ranges.iter().any(|range| range.contains(&1024))); + assert!(e8e9_ranges.iter().any(|range| range.contains(&12_000))); + } + + #[test] + fn large_text_ppmd_candidate_accepts_html_like_payloads() { + let mut data = vec![b'a'; OVER_THE_OLD_TRIAL_LIMIT]; + for index in (0..data.len()).step_by(79) { + data[index] = b'\n'; + } + data[..32].copy_from_slice(b"RAR PPMd text sample"); + + assert!(super::is_text_ppmd_candidate(&data)); + } + + /// Reading past padding leaves the question of a member that is nothing + /// else. There is no evidence either way in it, and PPMd is not the answer + /// to a member the store rule is about to take anyway. + #[test] + fn a_member_of_nothing_but_padding_is_not_text() { + let data = vec![0u8; OVER_THE_OLD_TRIAL_LIMIT]; + + assert!(!super::is_text_ppmd_candidate(&data)); + } + + /// Scattered NULs are not padding and still count against the member: a + /// run has to be long enough to be structure before it is discounted. + #[test] + fn scattered_nuls_still_read_as_binary() { + let mut data = vec![b'A'; OVER_THE_OLD_TRIAL_LIMIT]; + for index in (0..data.len()).step_by(3) { + data[index] = 0; + } + + assert!(!super::is_text_ppmd_candidate(&data)); + } + + /// The bench corpus is man pages in several languages, and counting only + /// ASCII scored the sample 77% against an 85% bar and sent it to LZ. + #[test] + fn the_text_screen_reads_multibyte_utf8_as_text() { + let line = "Überprüfen Sie die Größe der Datei — ändern Sie sie nicht. \ + 日本語ã�®ãƒžãƒ‹ãƒ¥ã‚¢ãƒ«ãƒšãƒ¼ã‚¸ã‚‚テキストã�§ã�™ã€‚\n"; + let data = line.repeat(4096).into_bytes(); + + let high = data.iter().filter(|&&byte| byte >= 0x80).count(); + assert!( + high * 100 / data.len() > 15, + "sample must be multibyte enough to have failed the ASCII-only test" + ); + assert!(super::is_text_ppmd_candidate(&data)); + } + + /// Accepting multibyte sequences must not accept binary that happens to + /// contain a few: a lead byte followed by a continuation byte turns up at + /// random often enough to notice, nowhere near often enough to pass. + #[test] + fn the_text_screen_still_rejects_high_entropy_binary() { + let mut state = 0x1234_5678u32; + let data: Vec<_> = (0..64_000) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + + assert!(!super::is_text_ppmd_candidate(&data)); + } + + /// A tar pads every member to a 512 byte block and closes with a block of + /// nothing, which is 27% NUL in front of a 1% bar. The source tree in the + /// bench corpus was read as binary and packed 22% behind WinRAR 3.00 for it. + #[test] + fn the_text_screen_reads_past_tar_padding() { + let mut data = Vec::new(); + for index in 0..512 { + let mut header = vec![0u8; 512]; + let name = format!("src/module{index}.rs"); + header[..name.len()].copy_from_slice(name.as_bytes()); + data.extend_from_slice(&header); + let body = format!( + "pub fn thing{index}(value: usize) -> usize {{\n value.wrapping_mul({index})\n}}\n" + ); + data.extend_from_slice(body.as_bytes()); + // Every member is padded out to the next block boundary. + data.resize(data.len().next_multiple_of(512), 0); + } + data.extend_from_slice(&[0u8; 1024]); + + let nul = data.iter().filter(|&&byte| byte == 0).count(); + assert!( + nul * 100 / data.len() > 15, + "the padding has to be heavy enough to have failed the old rule" + ); + assert!(super::is_text_ppmd_candidate(&data)); + } + + /// The other side of that: an object file is full of NUL runs too, and + /// reading past them must not turn its code into text. A member wrongly + /// called text skips the x86 filter search, so this one costs bytes. + #[test] + fn reading_past_padding_does_not_make_an_object_file_text() { + let mut state = 0x8badf00du32; + let mut data = Vec::new(); + for section in 0..64 { + // Alignment padding, the way a linker leaves it. + data.resize(data.len() + 64, 0); + data.extend_from_slice(format!("_ZN4rars{section}E\0").as_bytes()); + for _ in 0..1024 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + data.push(state as u8); + } + } + + assert!(!super::is_text_ppmd_candidate(&data)); + } + + /// Naming the level you were already getting must not change the archive. + /// The default resolves to method 0x33, the same as `--level 3`, but the + /// trial decision read the option rather than the level and the two came + /// out 24% apart on the same text. + #[test] + fn the_default_level_and_level_three_agree_about_the_ppmd_trial() { + let options = crate::rar15_40::WriterOptions { + target: crate::ArchiveVersion::Rar30, + ..Default::default() + }; + let default_level = super::compression_method_for_level(options); + let named_level = super::compression_method_for_level(crate::rar15_40::WriterOptions { + compression_level: Some(3), + ..options + }); + + assert_eq!(default_level, named_level); + assert_eq!( + super::ppmd_trial_pays(default_level - 0x30), + super::ppmd_trial_pays(named_level - 0x30), + ); + assert!(super::ppmd_trial_pays(3)); + assert!(!super::ppmd_trial_pays(2)); + } + + #[test] + fn auto_ppmd_candidate_rejects_binary_audio_shaped_payloads() { + let mut data = Vec::new(); + for sample in 0..8192i16 { + let left = sample.wrapping_mul(5).wrapping_add(200); + let right = sample.wrapping_mul(7).wrapping_sub(200); + data.extend_from_slice(&left.to_le_bytes()); + data.extend_from_slice(&right.to_le_bytes()); + } + + assert!(!super::is_text_ppmd_candidate(&data)); + } + + #[test] + fn auto_ppmd_candidate_accepts_text_payloads() { + let data = b"fn main() {\n println!(\"rar ppmd text candidate\");\n}\n".repeat(256); + + assert!(super::is_text_ppmd_candidate(&data)); + } + + #[test] + fn audio_filter_candidate_accepts_interleaved_pcm_like_payloads() { + let mut data = Vec::new(); + for sample in 0..4096i16 { + let left = sample.wrapping_mul(3).wrapping_add(200); + let right = sample.wrapping_mul(3).wrapping_sub(200); + data.extend_from_slice(&left.to_le_bytes()); + data.extend_from_slice(&right.to_le_bytes()); + } + + assert!(is_audio_filter_candidate(&data, 4)); + assert!(!is_audio_filter_candidate(&data, 3)); + assert!(!is_audio_filter_candidate(&data, 0)); + assert!(!is_audio_filter_candidate(&data, 5)); + } + + #[test] + fn audio_filter_candidate_checks_the_smallest_admitted_window() { + for channels in 1..=4 { + assert!(!is_audio_filter_candidate( + &vec![0; channels * 64 - 1], + channels + )); + assert!(is_audio_filter_candidate(&vec![0; channels * 64], channels)); + } + } + + #[test] + fn rar15_unix_directory_and_file_modes_become_dos_attributes() { + use crate::ArchiveVersion; + + assert_eq!( + super::rar15_compatible_metadata(ArchiveVersion::Rar15, 3, 0o040755), + (0, 0x10) + ); + assert_eq!( + super::rar15_compatible_metadata(ArchiveVersion::Rar15, 3, 0o100644), + (0, 0x20) + ); + assert_eq!( + super::rar15_compatible_metadata(ArchiveVersion::Rar20, 3, 0o040755), + (3, 0o040755) + ); + } + + #[test] + fn codec_progress_restart_counts_new_work_and_preserves_codec_errors() { + use crate::write_progress::{ProgressReporter, WorkTracker}; + use crate::{WriteOperation, WriteProgressEvent}; + use std::sync::Mutex; + + let advanced = Mutex::new(Vec::new()); + let reporter = |event: WriteProgressEvent<'_>| { + if let WriteProgressEvent::Advanced { + completed_bytes, .. + } = event + { + advanced.lock().unwrap().push(completed_bytes); + } + }; + let work = WorkTracker::new( + Some(ProgressReporter(&reporter)), + WriteOperation::Compression, + 30, + ); + let encoded = super::with_codec_progress(Some(&work), |progress| { + assert!(progress(10)); + assert!(progress(3)); + assert!(progress(5)); + Ok(b"packed".to_vec()) + }) + .unwrap(); + assert_eq!(encoded, b"packed"); + assert_eq!(*advanced.lock().unwrap(), [10, 13, 15]); + + let error = super::with_codec_progress(None, |_| { + Err(crate::codec::Error::InvalidData("injected codec failure")) + }) + .unwrap_err(); + assert!(matches!(error, crate::Error::Codec(_))); + } + + #[test] + fn audio_filter_candidate_rejects_high_entropy_binary_payloads() { + let mut state = 0xfeed_faceu32; + let data: Vec<_> = (0..16_384) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + + for channels in 1..=4 { + assert!(!is_audio_filter_candidate(&data, channels)); + } + } + + /// Prose with no long repeats: PPMd's model has plenty to work with and LZ + /// has little to match, which is the shape the engine choice exists for. + fn prose_like_text(len: usize) -> Vec { + const WORDS: [&str; 24] = [ + "archive", + "compression", + "dictionary", + "encoder", + "filter", + "header", + "member", + "method", + "offset", + "payload", + "reader", + "solid", + "stream", + "volume", + "window", + "writer", + "the", + "of", + "a", + "and", + "to", + "in", + "that", + "with", + ]; + let mut state = 0x9e37_79b9u32; + let mut out = Vec::with_capacity(len + 16); + while out.len() < len { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + out.extend_from_slice(WORDS[state as usize % WORDS.len()].as_bytes()); + out.push(if state.is_multiple_of(13) { + b'\n' + } else { + b' ' + }); + } + out.truncate(len); + out + } + + /// Text past the trial limit used to keep the LZ bytes without ever + /// encoding PPMd to compare, which cost 24% on 2 MiB of man pages and 12% + /// on 4 MiB. The method byte cannot tell the two engines apart (PPMd and + /// `-m5` LZ both write 0x35), so ask for the member both ways instead. + #[test] + fn text_over_the_trial_limit_is_measured_against_ppmd() { + let data = prose_like_text(OVER_THE_OLD_TRIAL_LIMIT); + assert!(super::is_text_ppmd_candidate(&data)); + + let offered = + encode_rar29_auto_filtered_member(&data, EncodeOptions::default(), 0x35, true).unwrap(); + let lz_only = + encode_rar29_auto_filtered_member(&data, EncodeOptions::default(), 0x35, false) + .unwrap(); + + assert!( + offered.data.len() < lz_only.data.len(), + "PPMd should have won this member: offered {} bytes, LZ alone {}", + offered.data.len(), + lz_only.data.len() + ); + } + + /// The other half of the same rule: winning is measured, not assumed. Text + /// that is one phrase repeating is all match and belongs to LZ, and the + /// member has to come back no larger than LZ alone would have made it. + #[test] + fn text_ppmd_cannot_lose_to_the_engine_it_replaced() { + let data = b"the quick brown fox jumps over the lazy dog. " + .repeat(OVER_THE_OLD_TRIAL_LIMIT / 45 + 64); + assert!(super::is_text_ppmd_candidate(&data)); + + let offered = + encode_rar29_auto_filtered_member(&data, EncodeOptions::default(), 0x35, true).unwrap(); + let lz_only = + encode_rar29_auto_filtered_member(&data, EncodeOptions::default(), 0x35, false) + .unwrap(); + + assert!( + offered.data.len() <= lz_only.data.len(), + "offering PPMd made the member bigger: {} bytes against {}", + offered.data.len(), + lz_only.data.len() + ); + + let plain_auto = super::encode_rar29_policy_filtered_payload( + &data, + &super::FilterPolicy::None, + super::Rar29Method::Auto, + EncodeOptions::default(), + 0x35, + true, + None, + ) + .unwrap(); + let forced_ppmd = super::encode_rar29_policy_filtered_payload( + &data, + &super::FilterPolicy::None, + super::Rar29Method::Ppmd, + EncodeOptions::default(), + 0x35, + false, + None, + ) + .unwrap(); + assert_eq!(plain_auto.data, forced_ppmd.data); + } + + #[test] + fn repeated_high_entropy_text_keeps_lz_when_ppmd_loses() { + let mut state = 0x9e37_79b9u32; + let mut block = Vec::with_capacity(64 * 1024); + for _ in 0..64 * 1024 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + block.push(b' ' + (state % 95) as u8); + } + let mut data = block.clone(); + data.extend_from_slice(&block); + assert!(super::is_text_ppmd_candidate(&data)); + let options = EncodeOptions::default(); + let offered = encode_rar29_auto_filtered_member(&data, options, 0x35, true).unwrap(); + let lz_only = encode_rar29_auto_filtered_member(&data, options, 0x35, false).unwrap(); + assert_eq!(offered.data, lz_only.data); + + let plain_offered = super::encode_rar29_policy_filtered_payload( + &data, + &super::FilterPolicy::None, + super::Rar29Method::Auto, + options, + 0x35, + true, + None, + ) + .unwrap(); + let plain_lz = super::encode_rar29_policy_filtered_payload( + &data, + &super::FilterPolicy::None, + super::Rar29Method::Auto, + options, + 0x35, + false, + None, + ) + .unwrap(); + assert_eq!(plain_offered.data, plain_lz.data); + } + + #[test] + fn auto_x86_filter_ranges_include_code_section_spans() { + let mut data = vec![0x41; 32_000]; + for pos in [4096, 4128, 4160] { + data[pos] = 0xe8; + } + for pos in [14_000, 14_032, 14_064] { + data[pos] = 0xe8; + } + + let ranges = auto_x86_filter_ranges(&data, false); + + assert!(ranges[0].contains(&4096)); + assert!(ranges[0].contains(&14_064)); + assert!(ranges.iter().any(|range| range.contains(&4096))); + assert!(ranges.iter().any(|range| range.contains(&14_064))); + } + + #[test] + fn auto_x86_policy_can_encode_multiple_disjoint_ranges() { + let mut data = vec![0x41u8; 80_000]; + for cluster_start in [8_000, 60_000] { + for index in 0..8 { + let pos = cluster_start + index * 64; + data[pos] = 0xe8; + data[pos + 1..pos + 5].copy_from_slice(&(0x2000u32 + index as u32).to_le_bytes()); + } + } + let filters: Vec<_> = disjoint_filter_ranges(auto_x86_filter_ranges(&data, false)) + .into_iter() + .map(|range| FilterSpec::range(FilterKind::E8, range)) + .collect(); + + let packed = encode_rar29_filtered_members(&data, &filters, EncodeOptions::default()) + .expect("multi-filter RAR29 member should encode"); + let decoded = unpack29_decode(&packed, data.len()).unwrap(); + + assert_eq!(filters.len(), 2); + assert!( + decoded == data, + "RAR 2.9 auto multi-filter E8 round-trip failed" + ); + } + + #[test] + fn auto_x86_policy_considers_tight_ranges_inside_sparse_spans() { + let mut data = Vec::new(); + data.extend((0..1024).map(|index| (index * 37 + 11) as u8)); + let first_cluster_start = data.len(); + for index in 0..16usize { + data.extend_from_slice(&[0x55, 0x8b, 0xec, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = data.len(); + data.push(0xe8); + let target = first_cluster_start + 0x500; + let relative = (target as i64 - (call_pos + 5) as i64) as i32; + data.extend_from_slice(&relative.to_le_bytes()); + data.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5d, 0xc3]); + } + data.extend((0..3200).map(|index| (index * 251 + 17) as u8)); + let second_cluster_start = data.len(); + for index in 0..16usize { + data.extend_from_slice(&[0x56, 0x8b, 0xf1, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = data.len(); + data.push(0xe8); + let target = second_cluster_start + 0x500; + let relative = (target as i64 - (call_pos + 5) as i64) as i32; + data.extend_from_slice(&relative.to_le_bytes()); + data.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5e, 0xc3]); + } + data.extend((0..1024).map(|index| (index * 53 + 7) as u8)); + + let ranges = disjoint_filter_ranges(auto_x86_filter_ranges(&data, false)); + let filters: Vec<_> = ranges + .iter() + .cloned() + .map(|range| FilterSpec::range(FilterKind::E8, range)) + .collect(); + let broad_range = first_cluster_start..second_cluster_start + 16 * 16; + let broad = encode_rar29_filtered_member( + &data, + FilterSpec::range(FilterKind::E8, broad_range), + EncodeOptions::default(), + ) + .unwrap(); + let tight = encode_rar29_filtered_members(&data, &filters, EncodeOptions::default()) + .expect("tight sparse x86 filters should encode"); + let auto = encode_rar29_auto_filtered_member(&data, EncodeOptions::default(), 0x35, false) + .unwrap(); + + assert!( + tight.len() < broad.len(), + "tight x86 ranges should avoid filtering sparse data gaps" + ); + assert!(auto.data.len() <= tight.len()); + assert_eq!(unpack29_decode(&auto.data, data.len()).unwrap(), data); + } + + #[test] + fn auto_delta_filter_range_skips_container_edges_and_aligns_channels() { + let data = vec![0u8; 512]; + + let range = auto_delta_filter_range(&data, 3).unwrap(); + + assert!(range.start >= AUTO_DELTA_EDGE_SKIP); + assert!(range.end <= data.len() - AUTO_DELTA_EDGE_SKIP); + assert_eq!(range.start % 3, 0); + assert_eq!((range.end - range.start) % 3, 0); + assert!(auto_delta_filter_range(&data[..80], 3).is_none()); + } + + #[test] + fn auto_filter_policy_considers_ranged_delta_candidates() { + let mut data = vec![0x55u8; AUTO_DELTA_EDGE_SKIP]; + for sample in 0..256u16 { + let left = sample as u8; + let right = left.wrapping_add(1); + data.extend_from_slice(&[left, right]); + } + data.extend(std::iter::repeat_n(0xaa, AUTO_DELTA_EDGE_SKIP)); + let options = EncodeOptions::default(); + + let plain = + crate::codec::rar29::unpack29_encode_literals_with_options(&data, options).unwrap(); + let ranged = encode_rar29_filtered_member( + &data, + FilterSpec::range( + FilterKind::Delta { channels: 2 }, + auto_delta_filter_range(&data, 2).unwrap(), + ), + options, + ) + .unwrap(); + let auto = encode_rar29_auto_filtered_member(&data, options, 0x35, true).unwrap(); + + assert!(ranged.len() < plain.len()); + assert!(auto.data.len() <= ranged.len()); + } + + #[test] + fn rar29_options_cap_match_distance_to_target_dictionary() { + assert_eq!( + rar29_encode_options_for_options( + super::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_compression_level(5) + ) + .unwrap() + .max_match_distance, + 1024 * 1024 + ); + assert_eq!( + rar29_encode_options_for_options( + super::WriterOptions::new(ArchiveVersion::Rar40, FeatureSet::store_only()) + .with_compression_level(5) + ) + .unwrap() + .max_match_distance, + 128 * 1024 + ); + assert_eq!( + rar29_encode_options_for_options( + super::WriterOptions::new(ArchiveVersion::Rar40, FeatureSet::store_only()) + .with_compression_level(5) + .with_dictionary_size(4 * 1024 * 1024) + ) + .unwrap() + .max_match_distance, + 4 * 1024 * 1024 + ); + } + + #[test] + fn rar20_options_cap_match_distance_to_header_dictionary() { + assert_eq!( + rar20_encode_options_for_options(super::WriterOptions::new( + ArchiveVersion::Rar20, + FeatureSet::store_only(), + )) + .unwrap() + .max_match_distance, + 64 * 1024 + ); + assert_eq!( + rar20_encode_options_for_options( + super::WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()) + .with_dictionary_size(1024 * 1024), + ) + .unwrap() + .max_match_distance, + 1024 * 1024 + ); + } + #[test] + fn only_the_top_two_levels_pay_for_the_rar20_optimal_parse() { + for level in 0..=3u8 { + let options = rar20_encode_options_for_level(Some(level)).unwrap(); + assert!(!options.optimal_parse, "level {level} priced every path"); + } + for level in 4..=5u8 { + let options = rar20_encode_options_for_level(Some(level)).unwrap(); + assert!(options.optimal_parse, "level {level} settled for greedy"); + } + assert!( + !rar20_encode_options_for_level(None).unwrap().optimal_parse, + "an absent level is the default level, which is not the top of the ladder", + ); + } + + #[test] + fn every_rar20_level_looks_two_positions_ahead() { + for level in 1..=5u8 { + let options = rar20_encode_options_for_level(Some(level)).unwrap(); + assert!(options.lazy_matching, "level {level} stopped being lazy"); + assert_eq!(options.lazy_lookahead, 2, "level {level}"); + } + } +} diff --git a/arca-rar/vendor/rars/src/rar50.rs b/arca-rar/vendor/rars/src/rar50.rs new file mode 100644 index 00000000..2b56f817 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50.rs @@ -0,0 +1,4677 @@ +use crate::crc32::crc32; +#[cfg(feature = "encryption")] +use crate::crypto::rar50::Rar50Cipher; +use crate::crypto::rar50::Rar50Keys; +use crate::detect::{find_archive_start, ArchiveSignature, RAR50_SIGNATURE, SFX_SCAN_LIMIT}; +use crate::error::{Error, Result}; +#[cfg(feature = "encryption")] +use crate::io_util::align16 as checked_align16; +use crate::io_util::read_exact_at; +pub(crate) use crate::source::ArchiveSource; +use crate::version::ArchiveFamily; +use std::fs::File; +use std::io::{Read, Write}; +use std::ops::Range; +use std::path::Path; +use std::sync::Arc; + +pub use crate::filter::{FilterKind, FilterPolicy, FilterSpec}; + +mod blake2sp; +mod extract; +#[cfg(any(feature = "write", feature = "recovery"))] +mod framing; +#[cfg(feature = "write")] +pub(crate) mod write; + +pub(crate) use extract::test_volumes_to; +pub use extract::{extract_volumes_to, extract_volumes_to_with_redirections}; +#[cfg(feature = "write")] +pub use write::{ + write_streaming_archive_to, write_streaming_archive_with_progress, write_streaming_volumes_to, + write_streaming_volumes_with_progress, ArchiveEntry, ArchiveExtras, ArchiveMetadataEntry, + CollectedVolumes, Rar50Writer, ServiceEntry, VolumeSink, WriterOptions, +}; + +const HEAD_MAIN: u64 = 1; +const HEAD_FILE: u64 = 2; +const HEAD_SERVICE: u64 = 3; +const HEAD_CRYPT: u64 = 4; +const HEAD_END: u64 = 5; +const REV5_SIGNATURE: &[u8] = b"Rar!\x1aRev"; + +const HFL_EXTRA: u64 = 0x0001; +const HFL_DATA: u64 = 0x0002; +const HFL_SPLIT_BEFORE: u64 = 0x0008; +const HFL_SPLIT_AFTER: u64 = 0x0010; + +/// Set on the end-of-archive block of every volume that is not the last, so a +/// reader knows to look for the next one rather than stopping where the file +/// does. +const EFL_NEXT_VOLUME: u64 = 0x0001; + +const MHFL_VOLUME: u64 = 0x0001; +const MHFL_VOLUME_NUMBER: u64 = 0x0002; +const MHFL_SOLID: u64 = 0x0004; +const MHFL_RECOVERY: u64 = 0x0008; +const MHFL_LOCKED: u64 = 0x0010; + +const FHFL_DIRECTORY: u64 = 0x0001; +const FHFL_MTIME: u64 = 0x0002; +const FHFL_CRC32: u64 = 0x0004; +const FHFL_UNP_SIZE_UNKNOWN: u64 = 0x0008; + +const MHEXTRA_LOCATOR: u64 = 0x01; +const MHEXTRA_LOCATOR_QUICK_OPEN: u64 = 0x0001; +const MHEXTRA_LOCATOR_RECOVERY: u64 = 0x0002; + +const FHEXTRA_CRYPT: u64 = 0x01; +const FHEXTRA_HASH: u64 = 0x02; +const FHEXTRA_HTIME: u64 = 0x03; +const FHEXTRA_REDIR: u64 = 0x05; +const FHEXTRA_SUBDATA: u64 = 0x07; +const MHEXTRA_ARCHIVE_METADATA: u64 = 0x02; +const MHEXTRA_ARCHIVE_METADATA_NAME: u64 = 0x0001; +const MHEXTRA_ARCHIVE_METADATA_TIME: u64 = 0x0002; + +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Archive { + pub sfx_offset: usize, + pub main: MainHeader, + pub blocks: Vec, + source: ArchiveSource, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct MainHeader { + pub block: BlockHeader, + pub archive_flags: u64, + pub volume_number: Option, + pub extras: Vec, + /// Whether the archive uses encrypted headers. + pub encrypted_headers: bool, + pub(crate) rewrite_metadata_complete: bool, +} + +impl MainHeader { + pub fn is_volume(&self) -> bool { + self.archive_flags & MHFL_VOLUME != 0 + } + + pub fn is_solid(&self) -> bool { + self.archive_flags & MHFL_SOLID != 0 + } + + pub fn has_recovery_record(&self) -> bool { + self.archive_flags & MHFL_RECOVERY != 0 + } + + pub fn is_locked(&self) -> bool { + self.archive_flags & MHFL_LOCKED != 0 + } + + pub fn locator(&self) -> Option<&LocatorRecord> { + self.extras.iter().find_map(|record| match record { + MainExtraRecord::Locator(locator) => Some(locator), + _ => None, + }) + } + + pub fn archive_metadata(&self) -> Option<&ArchiveMetadataRecord> { + self.extras.iter().find_map(|record| match record { + MainExtraRecord::ArchiveMetadata(metadata) => Some(metadata), + _ => None, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum MainExtraRecord { + Locator(LocatorRecord), + ArchiveMetadata(ArchiveMetadataRecord), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct LocatorRecord { + pub flags: u64, + pub quick_open_offset: Option, + pub recovery_record_offset: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct ArchiveMetadataRecord { + pub flags: u64, + /// Encoded name buffer, including any reserved zero padding. A leading + /// zero means no original name was stored, even with a nonzero length. + pub name: Option>, + pub creation_time: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Block { + File(FileHeader), + Service(FileHeader), + End(EndHeader), + Unknown(BlockHeader), +} + +/// The block that closes an archive. Its one field says whether the set carries +/// on into another volume, which a reader has to honour: a volume set is not +/// over because the file is. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct EndHeader { + pub block: BlockHeader, + pub flags: u64, +} + +impl EndHeader { + pub fn has_next_volume(&self) -> bool { + self.flags & EFL_NEXT_VOLUME != 0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct BlockHeader { + pub header_crc: u32, + pub header_size: u64, + pub header_type: u64, + pub flags: u64, + pub extra_area_size: Option, + pub data_size: Option, + pub offset: usize, + // Type-specific header bytes are archive-relative. Payload bytes are + // source-absolute so SFX-prefixed archives can be read directly. + pub header_range: Range, + pub data_range: Range, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct FileHeader { + pub block: BlockHeader, + pub file_flags: u64, + pub(crate) rewrite_metadata_complete: bool, + pub unpacked_size: u64, + pub attributes: u64, + pub mtime: Option, + /// Modification time from the `FHEXTRA_HTIME` record, in Unix seconds. + /// + /// Separate from `mtime`, which is the optional base-header field. Modern + /// WinRAR writes the time here and leaves that field out: 39 of the 40 + /// RAR 5 fixtures in this tree carry `FHEXTRA_HTIME` and one carries the + /// header field, so a reader that only looks at `mtime` restores nothing + /// on almost every real archive. + pub htime_mtime: Option, + /// Fractional detail belonging to the extended modification time. + pub htime_mtime_refinement: Option, + /// Complete supported time record, retaining FILETIME ticks without narrowing. + pub file_times: Option, + pub data_crc32: Option, + pub compression_info: u64, + pub host_os: u64, + pub name: Vec, + pub hash: Option, + pub redirection: Option, + pub service_data: Option>, + pub encrypted: bool, + pub encryption: Option, + crypto: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct FileRedirection { + pub redirection_type: u64, + pub flags: u64, + pub target_name: Vec, +} + +impl FileRedirection { + /// Validates the redirection kinds whose metadata the writer can retain. + pub fn is_supported(&self) -> bool { + (1..=5).contains(&self.redirection_type) + && self.flags & !1 == 0 + && (self.redirection_type < 4 || self.flags == 0) + && !self.target_name.is_empty() + && !self.target_name.contains(&0) + && std::str::from_utf8(&self.target_name).is_ok() + } + + #[cfg(feature = "write")] + pub(crate) fn supports_header(&self, host: u64, attr: u64, directory: bool) -> bool { + if !self.is_supported() { + return false; + } + if self.redirection_type == 1 { + host == 1 && attr & !0o7777 == 0o120000 && !directory + } else if self.redirection_type <= 3 { + host == 0 + && attr & 0x400 != 0 + && (attr & 0x10 != 0) == directory + && (self.redirection_type != 3 || self.flags == 1) + && directory == (self.flags & 1 != 0) + } else { + // is_supported admitted only kinds 1..=5 above. + !directory + && match host { + 0 => attr & (0x400 | 0x10) == 0, + 1 => attr & !0o7777 == 0o100000, + _ => false, + } + } + } + + /// Whether this target can be emitted by the Unix symbolic link writer. + /// Targets use RAR5 wire bytes and are never resolved against the filesystem. + pub fn is_supported_unix_symlink(&self) -> bool { + self.redirection_type == 1 + && self.flags & !1 == 0 + && !self.target_name.is_empty() + && !self.target_name.contains(&0) + && std::str::from_utf8(&self.target_name).is_ok() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct FileHash { + pub hash_type: u64, + pub data: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct RecoveryRecord { + pub percent: u64, + pub payload_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct FileEncryption { + pub version: u64, + pub flags: u64, + pub kdf_count: u8, + pub salt: [u8; 16], + pub iv: [u8; 16], + pub check_value: Option<[u8; 12]>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct FileCryptoState { + keys: Rar50Keys, + iv: [u8; 16], +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Rev5Volume { + pub version: u8, + pub data_count: u16, + pub recovery_count: u16, + pub recovery_number: u16, + pub payload_crc32: u32, + pub payload_size: u64, + pub payload: Vec, + pub data_volumes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Rev5VolumeMeta { + pub version: u8, + pub data_count: u16, + pub recovery_count: u16, + pub recovery_number: u16, + pub payload_crc32: u32, + pub payload_size: u64, + pub data_volumes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Rev5DataVolume { + pub file_size: u64, + pub crc32: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct CompressionInfo { + pub algorithm_version: u8, + pub solid: bool, + pub method: u8, + pub dictionary_power: u8, + pub dictionary_fraction: u8, + pub rar5_compat: bool, + pub dictionary_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct ExtractedEntryMeta { + pub name: Vec, + /// Unix modification time in seconds; absence is distinct from epoch. + pub file_time: Option, + pub mtime_refinement: Option, + pub attr: u64, + pub host_os: u64, + pub is_directory: bool, +} + +impl FileHeader { + /// Declared logical output size, unless the format marks it unknown. + /// Early split fragments can have unknown sizes while the final one is known. + /// This does not imply that the decoder supports unknown-size streams. + pub fn known_unpacked_size(&self) -> Option { + (self.file_flags & FHFL_UNP_SIZE_UNKNOWN == 0).then_some(self.unpacked_size) + } + + pub fn name_bytes(&self) -> &[u8] { + &self.name + } + + /// Returns the file name with invalid UTF-8 replaced for display only. + /// + /// Use [`Self::name_bytes`] when exact archive bytes matter. + pub fn name_lossy(&self) -> String { + String::from_utf8_lossy(&self.name).into_owned() + } + + pub fn is_split_before(&self) -> bool { + self.block.flags & HFL_SPLIT_BEFORE != 0 + } + + pub fn is_split_after(&self) -> bool { + self.block.flags & HFL_SPLIT_AFTER != 0 + } + + pub fn is_directory(&self) -> bool { + self.file_flags & FHFL_DIRECTORY != 0 + } + + pub fn is_stored(&self) -> bool { + compression_method(self.compression_info) == 0 + } + + pub fn is_redirection(&self) -> bool { + self.redirection.is_some() + } + + pub fn decoded_compression_info(&self) -> Result { + decode_compression_info(self.compression_info) + } + + pub fn packed_size(&self) -> u64 { + self.block.data_size.unwrap_or(0) + } + + pub fn packed_data(&self, archive: &Archive) -> Result> { + archive.read_range(self.block.data_range.clone()) + } + + pub fn verify_crc32(&self, data: &[u8]) -> Result<()> { + let Some(expected) = self.data_crc32 else { + return Ok(()); + }; + if self.uses_hash_mac() { + return Err(Error::InvalidHeader( + "RAR 5 encrypted CRC32 verification needs encryption keys", + )); + } + let actual = crc32(data); + if actual == expected { + Ok(()) + } else { + Err(Error::Crc32Mismatch { expected, actual }) + } + } + + pub fn verify_hash(&self, data: &[u8]) -> Result<()> { + let Some(hash) = &self.hash else { + return Ok(()); + }; + if self.uses_hash_mac() { + return Err(Error::InvalidHeader( + "RAR 5 encrypted hash verification needs encryption keys", + )); + } + match hash.hash_type { + 0 if hash.data.len() == 32 => { + let actual = blake2sp::hash(data); + if hash.data == actual { + Ok(()) + } else { + Err(Error::HashMismatch { hash_type: 0 }) + } + } + 0 => Err(Error::InvalidHeader( + "RAR 5 BLAKE2sp hash record has invalid length", + )), + _ => Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 unknown file hash type", + }), + } + } + + /// Checks the authoritative payload checksum, as extraction does. + pub fn verify_integrity(&self, data: &[u8]) -> Result<()> { + self.verify_integrity_with_keys(data, None) + } + + fn uses_hash_mac(&self) -> bool { + self.encryption + .as_ref() + .is_some_and(|encryption| encryption.flags & 0x0002 != 0) + } + + pub fn recovery_record(&self) -> Result> { + if self.name != b"RR" { + return Ok(None); + } + let Some(data) = &self.service_data else { + return Err(Error::InvalidHeader( + "RAR 5 recovery service is missing service data", + )); + }; + let (percent, len) = read_vint_at(data, 0, data.len())?; + if len != data.len() { + return Err(Error::InvalidHeader( + "RAR 5 recovery service data has trailing bytes", + )); + } + Ok(Some(RecoveryRecord { + percent, + payload_size: self.packed_size(), + })) + } +} + +impl Archive { + pub fn parse(input: &[u8]) -> Result { + Self::parse_with_options(input, crate::ArchiveReadOptions::default()) + } + + pub fn parse_owned(input: Vec) -> Result { + Self::parse_owned_with_options(input, crate::ArchiveReadOptions::default()) + } + + pub fn parse_with_options( + input: &[u8], + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let data: Arc<[u8]> = Arc::from(input.to_vec().into_boxed_slice()); + Self::parse_shared(data, options) + } + + pub fn parse_owned_with_options( + input: Vec, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + Self::parse_shared(Arc::from(input.into_boxed_slice()), options) + } + + pub fn parse_with_password(input: &[u8], password: Option<&[u8]>) -> Result { + Self::parse_with_options( + input, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_owned_with_password(input: Vec, password: Option<&[u8]>) -> Result { + Self::parse_owned_with_options( + input, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_path(path: impl AsRef) -> Result { + Self::parse_path_with_options(path, crate::ArchiveReadOptions::default()) + } + + pub fn parse_path_with_password( + path: impl AsRef, + password: Option<&[u8]>, + ) -> Result { + Self::parse_path_with_options( + path, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_path_with_options( + path: impl AsRef, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let path = Arc::new(path.as_ref().to_path_buf()); + let mut file = File::open(path.as_ref())?; + let len = file.metadata()?.len(); + let scan_len = len.min(SFX_SCAN_LIMIT as u64) as usize; + let mut scan = vec![0; scan_len]; + file.read_exact(&mut scan)?; + options.check_cancelled()?; + let sig = find_archive_start(&scan, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + if sig.family != ArchiveFamily::Rar50Plus { + return Err(Error::UnsupportedSignature); + } + // The detector found sig.offset inside the scanned prefix of this file. + let archive_len = usize::try_from(len) + .map_err(|_| Error::InvalidHeader("RAR 5 archive size overflows usize"))? + - sig.offset; + Self::parse_file_backed( + &mut file, + archive_len, + sig.offset, + ArchiveSource::File(path), + options, + ) + } + + pub fn parse_path_with_signature_and_password( + path: impl AsRef, + signature: ArchiveSignature, + password: Option<&[u8]>, + ) -> Result { + Self::parse_path_with_signature( + path, + signature, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + pub fn parse_path_with_signature( + path: impl AsRef, + signature: ArchiveSignature, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + if signature.family != ArchiveFamily::Rar50Plus { + return Err(Error::UnsupportedSignature); + } + let path = Arc::new(path.as_ref().to_path_buf()); + let mut file = File::open(path.as_ref())?; + let len = file.metadata()?.len(); + let archive_len = usize::try_from(len) + .map_err(|_| Error::InvalidHeader("RAR 5 archive size overflows usize"))? + .checked_sub(signature.offset) + .ok_or(Error::TooShort)?; + Self::parse_file_backed( + &mut file, + archive_len, + signature.offset, + ArchiveSource::File(path), + options, + ) + } + + fn parse_shared(input: Arc<[u8]>, options: crate::ArchiveReadOptions<'_>) -> Result { + options.check_cancelled()?; + let sig = find_archive_start(&input, SFX_SCAN_LIMIT).ok_or(Error::UnsupportedSignature)?; + if sig.family != ArchiveFamily::Rar50Plus { + return Err(Error::UnsupportedSignature); + } + // Signature detection admitted this offset within the immutable input. + let archive = &input[sig.offset..]; + let mut parsed = Self::parse_seekable( + archive, + sig.offset, + ArchiveSource::Memory(Arc::clone(&input)), + options, + )?; + parsed.sfx_offset = sig.offset; + Ok(parsed) + } + + fn parse_seekable( + input: &[u8], + sfx_offset: usize, + source: ArchiveSource, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + // parse_shared supplies the exact signature found in this buffer. + let archive_len = input.len(); + let (main, blocks) = parse_archive_blocks( + archive_len, + options, + |offset, budget| { + parse_block_header_bytes(input, offset, archive_len, sfx_offset, budget) + }, + |offset, keys, budget| { + parse_encrypted_block_header_bytes( + input, + offset, + archive_len, + sfx_offset, + keys, + budget, + ) + }, + )?; + + options.check_cancelled()?; + Ok(Self { + sfx_offset, + main, + blocks, + source, + }) + } + + pub(crate) fn parse_file_backed( + file: &mut (impl Read + std::io::Seek), + archive_len: usize, + sfx_offset: usize, + source: ArchiveSource, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + options.check_cancelled()?; + let signature = read_exact_at(file, sfx_offset, RAR50_SIGNATURE.len())?; + if signature != RAR50_SIGNATURE { + return Err(Error::UnsupportedSignature); + } + + let control = crate::read_control::ReadControl::new(options.cancellation); + let file_cell = std::cell::RefCell::new(control.reader(file)); + let (main, blocks) = parse_archive_blocks( + archive_len, + options, + |offset, budget| { + read_block_header_at( + &mut *file_cell.borrow_mut(), + offset, + archive_len, + sfx_offset, + budget, + ) + }, + |offset, keys, budget| { + read_encrypted_block_header_at( + &mut *file_cell.borrow_mut(), + offset, + archive_len, + sfx_offset, + keys, + budget, + ) + }, + )?; + + options.check_cancelled()?; + Ok(Self { + sfx_offset, + main, + blocks, + source, + }) + } + + fn read_range(&self, range: Range) -> Result> { + self.source.read_range(range) + } + + fn source_len(&self) -> Result { + self.source.len() + } + + fn range_reader(&self, range: Range) -> Result> { + self.source.range_reader(range) + } + + fn copy_range_to(&self, range: Range, writer: &mut dyn Write) -> Result<()> { + let source_len = self.source_len()?; + if range.start > range.end || range.end > source_len { + return Err(Error::InvalidHeader("RAR 5 repair range is out of bounds")); + } + let mut reader = self.range_reader(range)?; + std::io::copy(&mut reader, writer)?; + Ok(()) + } + + pub fn files(&self) -> impl Iterator { + self.blocks.iter().filter_map(|block| match block { + Block::File(file) => Some(file), + _ => None, + }) + } + + pub fn services(&self) -> impl Iterator { + self.blocks.iter().filter_map(|block| match block { + Block::Service(service) => Some(service), + _ => None, + }) + } + + /// Decodes the archive-level `CMT` service payload, if any. + /// + /// RAR 5 stores comments as `Service` blocks named `CMT`. Archive-level + /// comments appear before any `File` block; service blocks attached to a + /// specific file follow that file. This returns only the former. + pub fn archive_comment(&self) -> Result>> { + self.archive_comment_with_password(None) + } + + /// Same as [`Self::archive_comment`] but supplies a password for + /// individually-encrypted comment services. + pub fn archive_comment_with_password( + &self, + password: Option<&[u8]>, + ) -> Result>> { + self.archive_comment_with_options(crate::ArchiveReadOptions::with_optional_password( + password, + )) + } + + /// Decodes the archive comment under the same policy as [`crate::Archive::comment_with_options`]. + pub fn archive_comment_with_options( + &self, + options: crate::ArchiveReadOptions<'_>, + ) -> Result>> { + options.check_cancelled()?; + for block in &self.blocks { + match block { + Block::File(_) => return Ok(None), + Block::Service(service) if service.name == b"CMT" => { + return service + .decoded_comment_with_options(self, options) + .map(Some); + } + _ => {} + } + } + Ok(None) + } + + pub fn repair_recovery(&self) -> Result> { + Ok(self.repair_recovery_with_report(None)?.data) + } + + pub fn repair_recovery_with_report( + &self, + password: Option<&[u8]>, + ) -> Result { + self.repair_recovery_with_options(crate::ArchiveReadOptions::with_optional_password( + password, + )) + } + + /// Repairs embedded recovery data using password and cancellation only. + pub fn repair_recovery_with_options( + &self, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + let mut data = Vec::new(); + let report = self.repair_recovery_to_with_options(&mut data, options)?; + Ok(crate::RecoveryRepairResult { data, report }) + } + + pub fn repair_recovery_to(&self, writer: &mut dyn Write) -> Result<()> { + self.repair_recovery_to_with_report(writer, None) + .map(|_| ()) + } + + pub fn repair_recovery_to_with_report( + &self, + writer: &mut dyn Write, + password: Option<&[u8]>, + ) -> Result { + self.repair_recovery_to_with_options( + writer, + crate::ArchiveReadOptions::with_optional_password(password), + ) + } + + /// Streams repaired bytes with cooperative cancellation; partial output can remain. + /// Only password and cancellation apply from the read options. + pub fn repair_recovery_to_with_options( + &self, + writer: &mut dyn Write, + options: crate::ArchiveReadOptions<'_>, + ) -> Result { + #[cfg(not(feature = "recovery"))] + { + let _ = (self, writer, options); + Err(Error::FeatureDisabled { + feature: "recovery", + }) + } + #[cfg(feature = "recovery")] + { + options.check_cancelled()?; + let password = options.password; + let control = crate::read_control::ReadControl::new(options.cancellation); + let recovery = self.recovery_service()?; + let recovery_data = recovery.decoded_recovery_data(self, password, &control)?; + let (available, expected) = + crate::recovery::rar5::inline_recovery_chunk_counts_with_control( + &recovery_data, + &control, + )?; + if available == expected || self.sfx_offset != 0 { + return self.repair_recovery_to_legacy( + writer, + &recovery_data, + available, + expected, + &control, + ); + } + let bytes = self.read_range(0..self.source_len()?)?; + let options = crate::recovery::rar5::InlineRepairOptions { + password, + control: control.clone(), + record_range: Some(recovery.block.data_range.clone()), + }; + let (data, report) = crate::recovery::rar5::repair_inline_recovery_archive_with_report( + &bytes, &options, + )?; + control.finish(control.write_all(writer, &data).map_err(Error::from))?; + control.check()?; + Ok(report) + } + } + + #[cfg(feature = "recovery")] + fn recovery_service(&self) -> Result<&FileHeader> { + self.services() + .find(|service| matches!(service.recovery_record(), Ok(Some(_)))) + .ok_or(Error::InvalidHeader( + "RAR 5 archive does not contain an inline recovery record", + )) + } + + #[cfg(feature = "recovery")] + fn repair_recovery_to_legacy( + &self, + writer: &mut dyn Write, + recovery_data: &[u8], + available: u64, + expected: u64, + control: &crate::read_control::ReadControl, + ) -> Result { + control.check()?; + let recovery = self.recovery_service()?; + let prefix_start = self.sfx_offset; + let prefix_end = recovery.block.offset; + let prefix_len = prefix_end + .checked_sub(prefix_start) + .ok_or(Error::InvalidHeader( + "RAR 5 recovery prefix range overflows archive bounds", + ))?; + let source_len = self.source_len()?; + if prefix_end > source_len { + return Err(Error::InvalidHeader( + "RAR 5 recovery prefix is out of bounds", + )); + } + let repaired_shards = + crate::recovery::rar5::repair_inline_recovery_prefix_shards_with_control( + prefix_len, + recovery_data, + |range| { + // The repair helper bounds every range by prefix_len, and + // prefix_start + prefix_len equals admitted prefix_end. + let start = prefix_start + range.start; + let end = prefix_start + range.end; + self.read_range(start..end) + .map_err(|_| crate::recovery::rar5::Error::BadRecoveryChunk) + }, + control, + )?; + + self.copy_repair_range(0..prefix_start, writer, control)?; + let mut cursor = 0usize; + let data_repaired = !repaired_shards.is_empty(); + for (range, data) in repaired_shards { + control.check()?; + self.copy_repair_range( + prefix_start + cursor..prefix_start + range.start, + writer, + control, + )?; + control.finish(control.write_all(writer, &data).map_err(Error::from))?; + cursor = range.end; + } + self.copy_repair_range(prefix_start + cursor..prefix_end, writer, control)?; + self.copy_repair_range(prefix_end..source_len, writer, control)?; + control.check()?; + Ok(crate::RecoveryRepairReport { + changed: data_repaired, + data_repaired, + recovery_record_rebuilt: false, + end_record_rebuilt: false, + available_recovery_shards: Some(available), + expected_recovery_shards: Some(expected), + }) + } + #[cfg(feature = "recovery")] + fn copy_repair_range( + &self, + range: Range, + writer: &mut dyn Write, + control: &crate::read_control::ReadControl, + ) -> Result<()> { + for start in (range.start..range.end).step_by(64 * 1024) { + control.check()?; + let data = self.read_range(start..(start.saturating_add(64 * 1024)).min(range.end))?; + control.finish(control.write_all(writer, &data).map_err(Error::from))?; + } + control.check() + } +} + +impl Rev5Volume { + pub fn parse(input: &[u8]) -> Result { + let (meta, payload_range) = Rev5VolumeMeta::parse_with_payload_range(input)?; + let payload = &input[payload_range]; + let actual_payload_crc = crc32(payload); + if actual_payload_crc != meta.payload_crc32 { + return Err(Error::Crc32Mismatch { + expected: meta.payload_crc32, + actual: actual_payload_crc, + }); + } + + Ok(Self { + version: meta.version, + data_count: meta.data_count, + recovery_count: meta.recovery_count, + recovery_number: meta.recovery_number, + payload_crc32: meta.payload_crc32, + payload_size: meta.payload_size, + payload: payload.to_vec(), + data_volumes: meta.data_volumes, + }) + } +} + +impl Rev5VolumeMeta { + pub fn parse(input: &[u8]) -> Result { + Self::parse_with_payload_range(input).map(|(meta, _)| meta) + } + + fn parse_with_payload_range(input: &[u8]) -> Result<(Self, Range)> { + if !input.starts_with(REV5_SIGNATURE) { + return Err(Error::UnsupportedSignature); + } + if input.len() < 16 { + return Err(Error::TooShort); + } + // The complete 16-byte REV prefix was admitted above. + let header_crc = u32::from_le_bytes([input[8], input[9], input[10], input[11]]); + let header_size = u32::from_le_bytes([input[12], input[13], input[14], input[15]]) as usize; + if header_size <= 5 || header_size > 0x100000 { + return Err(Error::InvalidHeader("RAR 5 REV header size is invalid")); + } + // The header-size limit above keeps this sum within usize even on 32-bit hosts. + let header_end = 16 + header_size; + if header_end > input.len() { + return Err(Error::TooShort); + } + let actual_header_crc = crc32(&input[12..header_end]); + if actual_header_crc != header_crc { + return Err(Error::Crc32Mismatch { + expected: header_crc, + actual: actual_header_crc, + }); + } + + let body = &input[16..header_end]; + if body.len() < 11 { + return Err(Error::TooShort); + } + // The complete fixed 11-byte REV prefix was admitted above. + let version = body[0]; + if version != 1 { + return Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 REV version", + }); + } + let data_count = u16::from_le_bytes([body[1], body[2]]); + let recovery_count = u16::from_le_bytes([body[3], body[4]]); + let recovery_number = u16::from_le_bytes([body[5], body[6]]); + let payload_crc32 = u32::from_le_bytes([body[7], body[8], body[9], body[10]]); + let first_recovery_number = u32::from(data_count); + let recovery_end = first_recovery_number + u32::from(recovery_count); + let recovery_number = u32::from(recovery_number); + if recovery_count == 0 + || recovery_number < first_recovery_number + || recovery_number >= recovery_end + { + return Err(Error::InvalidHeader("RAR 5 REV volume number is invalid")); + } + + // data_count is u16, so the complete table fits in usize on 32-bit hosts. + let expected_table_end = 11 + data_count as usize * 12; + if body.len() < expected_table_end { + return Err(Error::InvalidHeader( + "RAR 5 REV metadata table size is invalid", + )); + } + let mut data_volumes = Vec::with_capacity(data_count as usize); + for index in 0..usize::from(data_count) { + // The whole 12-byte metadata table was admitted above. + let entry = &body[11 + index * 12..11 + (index + 1) * 12]; + let file_size = u64::from_le_bytes([ + entry[0], entry[1], entry[2], entry[3], entry[4], entry[5], entry[6], entry[7], + ]); + let crc = u32::from_le_bytes([entry[8], entry[9], entry[10], entry[11]]); + data_volumes.push(Rev5DataVolume { + file_size, + crc32: crc, + }); + } + + Ok(( + Self { + version, + data_count, + recovery_count, + recovery_number: recovery_number as u16, + payload_crc32, + payload_size: (input.len() - header_end) as u64, + data_volumes, + }, + header_end..input.len(), + )) + } +} + +impl From<&Rev5Volume> for Rev5VolumeMeta { + fn from(volume: &Rev5Volume) -> Self { + Self { + version: volume.version, + data_count: volume.data_count, + recovery_count: volume.recovery_count, + recovery_number: volume.recovery_number, + payload_crc32: volume.payload_crc32, + payload_size: volume.payload_size, + data_volumes: volume.data_volumes.clone(), + } + } +} + +impl From for Rev5VolumeMeta { + fn from(volume: Rev5Volume) -> Self { + Self { + version: volume.version, + data_count: volume.data_count, + recovery_count: volume.recovery_count, + recovery_number: volume.recovery_number, + payload_crc32: volume.payload_crc32, + payload_size: volume.payload_size, + data_volumes: volume.data_volumes, + } + } +} + +pub fn repair_rev5_volumes_to( + data_volumes: &[Option<&[u8]>], + recovery_volumes: &[Rev5Volume], + mut write: F, +) -> Result<()> +where + F: FnMut(usize, &[u8]) -> Result<()>, +{ + #[cfg(not(feature = "recovery"))] + { + let _ = (data_volumes, recovery_volumes, &mut write); + Err(Error::FeatureDisabled { + feature: "recovery", + }) + } + #[cfg(feature = "recovery")] + { + let first = recovery_volumes.first().ok_or(Error::InvalidHeader( + "RAR 5 REV recovery volume set is empty", + ))?; + let data_count = usize::from(first.data_count); + if data_volumes.len() != data_count { + return Err(Error::InvalidHeader( + "RAR 5 REV data volume count does not match metadata", + )); + } + if recovery_volumes.iter().any(|rev| { + rev.version != first.version + || rev.data_count != first.data_count + || rev.recovery_count != first.recovery_count + || rev.data_volumes != first.data_volumes + || rev.payload.len() != first.payload.len() + }) { + return Err(Error::InvalidHeader( + "RAR 5 REV recovery volume metadata differs across files", + )); + } + + let mut shards = Vec::with_capacity(data_count); + for (index, data) in data_volumes.iter().enumerate() { + let Some(data) = data else { + shards.push(None); + continue; + }; + let meta = &first.data_volumes[index]; + if data.len() as u64 != meta.file_size || crc32(data) != meta.crc32 { + shards.push(None); + } else { + shards.push(Some(*data)); + } + } + + let recovery_rows: Vec<_> = recovery_volumes + .iter() + .map(|rev| { + let row = usize::from(rev.recovery_number) + .checked_sub(data_count) + .ok_or(Error::InvalidHeader("RAR 5 REV recovery number is invalid"))?; + Ok((row, rev.payload.as_slice())) + }) + .collect::>()?; + let mut seen_recovery_rows = std::collections::HashSet::with_capacity(recovery_rows.len()); + if recovery_rows + .iter() + .any(|(row, _)| !seen_recovery_rows.insert(*row)) + { + return Err(Error::InvalidHeader( + "RAR 5 REV recovery volume set contains duplicate recovery rows", + )); + } + let repaired = crate::recovery::rar5::reconstruct_data_shards(&shards, &recovery_rows)?; + + for (index, (mut shard, meta)) in repaired.into_iter().zip(&first.data_volumes).enumerate() + { + let file_size = usize::try_from(meta.file_size) + .map_err(|_| Error::InvalidHeader("RAR 5 REV data volume size overflows usize"))?; + if shard.len() < file_size { + return Err(Error::InvalidHeader( + "RAR 5 REV repaired shard is shorter than data volume size", + )); + } + shard.truncate(file_size); + let actual = crc32(&shard); + if actual != meta.crc32 { + return Err(Error::Crc32Mismatch { + expected: meta.crc32, + actual, + }); + } + write(index, &shard)?; + } + Ok(()) + } +} + +pub fn repair_inline_recovery_bytes(input: &[u8]) -> Result> { + Ok(repair_inline_recovery_bytes_with_report(input)?.data) +} + +pub fn repair_inline_recovery_bytes_with_report( + input: &[u8], +) -> Result { + repair_inline_recovery_bytes_with_options(input, crate::ArchiveReadOptions::new()) +} + +pub fn repair_inline_recovery_bytes_with_options( + input: &[u8], + options: crate::ArchiveReadOptions<'_>, +) -> Result { + #[cfg(not(feature = "recovery"))] + { + let _ = (input, options); + Err(Error::FeatureDisabled { + feature: "recovery", + }) + } + #[cfg(feature = "recovery")] + { + options.check_cancelled()?; + if !input.starts_with(RAR50_SIGNATURE) { + return Err(Error::UnsupportedSignature); + } + let repair_options = crate::recovery::rar5::InlineRepairOptions { + password: options.password, + control: crate::read_control::ReadControl::new(options.cancellation), + ..Default::default() + }; + let (repaired, report) = crate::recovery::rar5::repair_inline_recovery_archive_with_report( + input, + &repair_options, + ) + .map_err(Error::from)?; + let parse_target = if repaired == input { input } else { &repaired }; + let _ = Archive::parse_with_options(parse_target, options)?; + Ok(crate::RecoveryRepairResult { + data: repaired, + report, + }) + } +} + +/// Frames a replacement end-of-archive header for an archive that lost its +/// own. +/// +/// A volume whose last entry runs into the next part has to keep saying so, +/// and only the parsed blocks show that, so `input` is read back here. It is +/// the repaired archive rather than the damaged one, so the blocks it walks +/// are the ones the caller is about to write. A volume that splits cleanly on +/// an entry boundary is indistinguishable from a final one and loses the flag; +/// unrar and WinRAR both walk such a set from the main header anyway. +#[cfg(feature = "recovery")] +pub(crate) fn recovery_end_header( + input: &[u8], + options: crate::ArchiveReadOptions<'_>, +) -> Result> { + options.check_cancelled()?; + let password = options.password; + let end_flags = match Archive::parse_with_options(input, options) { + Ok(archive) + if archive + .files() + .last() + .is_some_and(|file| file.is_split_after()) => + { + EFL_NEXT_VOLUME + } + Err(error) => recovery_end_parse_error(error)?, + _ => 0, + }; + let first = parse_block_header_bytes( + input, + RAR50_SIGNATURE.len(), + input.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(options), + )?; + let keys = if first.block.header_type == HEAD_CRYPT { + Some(parse_archive_encryption_header(&first, password)?.0) + } else { + None + }; + let mut specific = framing::HeaderScratch::<10>::new(); + specific.vint(end_flags); + let image = framing::HeaderImage::new(HEAD_END, 0, None, &specific, &[])?; + let mut end = vec![0; image.image_len(keys.is_some(), 0)?]; + image.render_into(keys.as_ref(), &[], &mut end)?; + Ok(end) +} + +#[cfg(feature = "recovery")] +fn recovery_end_parse_error(error: Error) -> Result { + if error.kind() == crate::ErrorKind::Cancelled { + Err(error) + } else { + Ok(0) + } +} + +fn parse_main_header_bytes(parsed: &ParsedBlockHeader) -> Result { + let mut reader = HeaderReader::new(&parsed.header, parsed.type_specific_range.clone()); + let archive_flags = reader.read_vint()?; + let volume_number = if archive_flags & MHFL_VOLUME_NUMBER != 0 { + Some(reader.read_vint()?) + } else { + None + }; + let (extras, complete) = + parse_main_extra_area(&parsed.header, parsed.extra_range.clone(), &parsed.control)?; + Ok(MainHeader { + block: parsed.block.clone(), + archive_flags, + volume_number, + extras, + encrypted_headers: false, + rewrite_metadata_complete: complete && reader.pos == reader.range.end, + }) +} + +fn parse_main_extra_area( + input: &[u8], + range: Range, + control: &crate::read_control::ReadControl, +) -> Result<(Vec, bool)> { + let mut records = Vec::new(); + let mut complete = true; + let mut seen = std::collections::HashSet::new(); + let parsed_complete = + parse_extra_records(input, range, false, control, |record_type, data| { + complete &= seen.insert(record_type); + match record_type { + MHEXTRA_LOCATOR => { + let mut reader = SliceReader::new(input, data.start, data.end); + let flags = reader.read_vint()?; + let quick_open_offset = if flags & MHEXTRA_LOCATOR_QUICK_OPEN != 0 { + Some(reader.read_vint()?) + } else { + None + }; + let recovery_record_offset = if flags & MHEXTRA_LOCATOR_RECOVERY != 0 { + Some(reader.read_vint()?) + } else { + None + }; + complete &= flags & !3 == 0 && reader.pos == reader.end; + // LOCATOR records are intentionally forward-compatible: known + // offsets are parsed and any trailing bytes remain reserved for + // future flags. + records.push(MainExtraRecord::Locator(LocatorRecord { + flags, + quick_open_offset, + recovery_record_offset, + })); + Ok(()) + } + MHEXTRA_ARCHIVE_METADATA => { + let mut reader = SliceReader::new(input, data.start, data.end); + let flags = reader.read_vint()?; + let name = if flags & MHEXTRA_ARCHIVE_METADATA_NAME != 0 { + let name_len = usize_from_u64( + reader.read_vint()?, + "RAR 5 archive metadata name length overflows usize", + )?; + Some(reader.read_bytes(name_len)?.to_vec()) + } else { + None + }; + let creation_time = if flags & MHEXTRA_ARCHIVE_METADATA_TIME != 0 { + Some(if flags & 4 != 0 && flags & 8 == 0 { + u64::from(reader.read_u32()?) + } else { + reader.read_u64()? + }) + } else { + None + }; + if reader.pos != reader.end { + return Err(Error::InvalidHeader( + "RAR 5 archive metadata record has trailing bytes", + )); + } + complete &= flags & !15 == 0 && (flags & 8 == 0 || flags & 4 != 0); + records.push(MainExtraRecord::ArchiveMetadata(ArchiveMetadataRecord { + flags, + name, + creation_time, + })); + Ok(()) + } + _ => { + complete = false; + Ok(()) + } + } + })?; + Ok((records, complete && parsed_complete)) +} + +fn parse_file_header_bytes(parsed: &ParsedBlockHeader) -> Result { + let mut reader = HeaderReader::new(&parsed.header, parsed.type_specific_range.clone()); + let file_flags = reader.read_vint()?; + let unpacked_size = reader.read_vint()?; + let attributes = reader.read_vint()?; + let mtime = if file_flags & FHFL_MTIME != 0 { + Some(reader.read_u32()?) + } else { + None + }; + let data_crc32 = if file_flags & FHFL_CRC32 != 0 { + Some(reader.read_u32()?) + } else { + None + }; + let compression_info = reader.read_vint()?; + let host_os = reader.read_vint()?; + let name_len = usize_from_u64( + reader.read_vint()?, + "RAR 5 file name length overflows usize", + )?; + let name = reader.read_bytes(name_len)?.to_vec(); + let mut file = FileHeader { + block: parsed.block.clone(), + file_flags, + rewrite_metadata_complete: reader.pos == reader.range.end, + unpacked_size, + attributes, + mtime, + htime_mtime: None, + htime_mtime_refinement: None, + file_times: None, + data_crc32, + compression_info, + host_os, + name, + hash: None, + redirection: None, + service_data: None, + encrypted: false, + encryption: None, + crypto: None, + }; + parse_file_extra_area( + &parsed.header, + parsed.extra_range.clone(), + parsed.block.header_type == HEAD_SERVICE, + &mut file, + &parsed.control, + )?; + Ok(file) +} + +fn parse_file_extra_area( + input: &[u8], + range: Range, + is_service: bool, + file: &mut FileHeader, + control: &crate::read_control::ReadControl, +) -> Result<()> { + if file.block.extra_area_size.is_none() { + return Ok(()); + } + let mut seen = 0u64; + let complete = parse_extra_records(input, range, is_service, control, |record_type, data| { + let bit = 1u64 + .checked_shl(record_type as u32) + .filter(|_| record_type < 64) + .unwrap_or(0); + if bit == 0 || seen & bit != 0 { + file.rewrite_metadata_complete = false; + } + seen |= bit; + match record_type { + FHEXTRA_CRYPT => { + let encryption = parse_file_encryption_record(input, data)?; + file.rewrite_metadata_complete &= + encryption.version == 0 && encryption.flags & !3 == 0; + file.encrypted = true; + file.encryption = Some(encryption); + } + FHEXTRA_HASH => { + let (hash_type, hash_type_len) = read_vint_at(input, data.start, data.end)?; + file.rewrite_metadata_complete &= + hash_type == 0 && data.len() == hash_type_len + 32; + file.hash = Some(FileHash { + hash_type, + data: input[data.start + hash_type_len..data.end].to_vec(), + }); + } + FHEXTRA_REDIR => { + let link = parse_file_redirection_record(input, data)?; + file.rewrite_metadata_complete &= !is_service && link.is_supported(); + file.redirection = Some(link); + } + FHEXTRA_HTIME => { + let flags = read_vint_at(input, data.start, data.end).ok(); + let times = flags.and_then(|(flags, len)| { + crate::FileTimes::parse(flags, &input[data.start + len..data.end]) + }); + let parsed = parse_htime_mtime(input, data); + file.rewrite_metadata_complete &= times.is_some() + && !(file.mtime.is_some() + && times.is_some_and(|times| times.modified.is_some())); + file.file_times = times; + file.htime_mtime = parsed.map(|(seconds, _)| seconds); + file.htime_mtime_refinement = parsed.and_then(|(_, detail)| detail); + } + FHEXTRA_SUBDATA => { + file.rewrite_metadata_complete &= + is_service && (data.is_empty() || file.name == b"RR"); + file.service_data = Some(input[data].to_vec()); + } + _ => { + file.rewrite_metadata_complete = false; + } + } + Ok(()) + })?; + file.rewrite_metadata_complete &= complete; + Ok(()) +} + +/// Reads the modification time out of an `FHEXTRA_HTIME` record. +/// +/// Layout is flags, then the present times in the order mtime, ctime, atime, +/// then, when flag `0x0010` is set, one sub-second remainder per time in that +/// same order. Under flag `0x0001` a time is `uint32` Unix seconds; without it +/// a time is a `uint64` Windows FILETIME. Unix modification-time fractions follow +/// all the present whole-second values; FILETIME embeds its fraction in the ticks. +/// +/// A malformed whole-second value yields `None`; malformed fractional detail is +/// ignored while retaining valid seconds. The reference +/// readers do not fail an archive over a time they cannot read, and neither +/// should we lose the file over it. +fn parse_htime_mtime( + input: &[u8], + range: Range, +) -> Option<(u32, Option)> { + // Slice to the record first: malformed time fields must not borrow bytes + // from a following extra record or the file payload. + let data = &input[range]; + let (flags, at) = read_vint_at(data, 0, data.len()).ok()?; + if flags & 2 == 0 { + return None; + } + let (seconds, nanos) = if flags & 1 != 0 { + let seconds = + u32::from_le_bytes(data.get(at..at + 4)?.try_into().expect("four-byte slice")); + // Unix fractions follow ALL present whole-second values, not each value. + let count = (flags & 0x0e).count_ones() as usize; + // A vint occupies at most ten bytes; at most three u32 times follow. + let fraction_at = at + count * 4; + let nanos = if flags & 0x10 != 0 { + data.get(fraction_at..fraction_at + 4) + .map(|bytes| u32::from_le_bytes(bytes.try_into().expect("four-byte slice"))) + .filter(|nanos| *nanos < 1_000_000_000) + } else { + None + }; + (seconds, nanos) + } else { + let ticks = u64::from_le_bytes(data.get(at..at + 8)?.try_into().expect("eight-byte slice")); + let seconds = u32::try_from((ticks / 10_000_000).checked_sub(11_644_473_600)?).ok()?; + (seconds, Some(((ticks % 10_000_000) * 100) as u32)) + }; + Some(( + seconds, + nanos.map(|nanoseconds| crate::TimeRefinement { + add_second: false, + nanoseconds, + }), + )) +} + +fn parse_file_redirection_record(input: &[u8], range: Range) -> Result { + let (redirection_type, type_len) = read_vint_at(input, range.start, range.end)?; + let flags_start = range.start + type_len; + let (flags, flags_len) = read_vint_at(input, flags_start, range.end)?; + let name_len_start = flags_start + flags_len; + let (name_len, name_len_len) = read_vint_at(input, name_len_start, range.end)?; + let name_start = name_len_start + name_len_len; + let name_len = usize::try_from(name_len).map_err(|_| { + Error::InvalidHeader("RAR 5 file redirection target length overflows host address size") + })?; + let name_end = name_start + .checked_add(name_len) + .ok_or(Error::InvalidHeader( + "RAR 5 file redirection target length overflows", + ))?; + if name_end != range.end { + return Err(Error::InvalidHeader( + "RAR 5 file redirection record has trailing bytes", + )); + } + Ok(FileRedirection { + redirection_type, + flags, + target_name: input[name_start..name_end].to_vec(), + }) +} + +fn parse_file_encryption_record(input: &[u8], range: Range) -> Result { + let (version, version_len) = read_vint_at(input, range.start, range.end)?; + let flags_pos = range.start + version_len; + let (flags, flags_len) = read_vint_at(input, flags_pos, range.end)?; + let mut pos = flags_pos + flags_len; + if pos >= range.end { + return Err(Error::TooShort); + } + let kdf_count = input[pos]; + pos += 1; + let salt = read_array_at::<16>(input, &mut pos, range.end)?; + let iv = read_array_at::<16>(input, &mut pos, range.end)?; + let check_value = if flags & 0x0001 != 0 { + Some(read_array_at::<12>(input, &mut pos, range.end)?) + } else { + None + }; + if pos != range.end { + return Err(Error::InvalidHeader( + "RAR 5 file encryption record has trailing bytes", + )); + } + Ok(FileEncryption { + version, + flags, + kdf_count, + salt, + iv, + check_value, + }) +} + +fn parse_archive_encryption_header( + parsed: &ParsedBlockHeader, + password: Option<&[u8]>, +) -> Result<(Rar50Keys, bool)> { + #[cfg(not(feature = "encryption"))] + { + let _ = (parsed, password); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + crate::crypto::require_encryption()?; + + let mut reader = HeaderReader::new(&parsed.header, parsed.type_specific_range.clone()); + let version = reader.read_vint()?; + let flags = reader.read_vint()?; + // The first encrypted-header fields are decoded before password refusal. + let password = password.ok_or(Error::NeedPassword)?; + if version != 0 { + return Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 unknown header encryption version", + }); + } + let kdf_count = reader.read_byte()?; + let salt = reader.read_array::<16>()?; + let check_value = if flags & 0x0001 != 0 { + Some(reader.read_array::<12>()?) + } else { + None + }; + if reader.pos != reader.range.end { + return Err(Error::InvalidHeader( + "RAR 5 archive encryption header has trailing bytes", + )); + } + let keys = Rar50Keys::derive(password, salt, kdf_count).map_err(map_rar50_crypto_error)?; + if let Some(check_value) = check_value { + keys.check_password(&check_value) + .map_err(map_rar50_crypto_error)?; + } + Ok((keys, flags & !1 == 0)) + } +} + +fn attach_file_crypto(file: &mut FileHeader, password: Option<&[u8]>) -> Result<()> { + #[cfg(not(feature = "encryption"))] + { + let _ = (file, password); + Ok(()) + } + #[cfg(feature = "encryption")] + { + if !cfg!(feature = "encryption") { + return Ok(()); + } + + // Called once for each freshly parsed file or service header. + if !file.encrypted { + return Ok(()); + } + let Some(password) = password else { + return Ok(()); + }; + let encryption = file.encryption.as_ref().ok_or(Error::InvalidHeader( + "RAR 5 encrypted file is missing encryption record", + ))?; + if encryption.version != 0 { + return Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 unknown file encryption version", + }); + } + let keys = Rar50Keys::derive(password, encryption.salt, encryption.kdf_count) + .map_err(map_rar50_crypto_error)?; + if let Some(check_value) = encryption.check_value { + keys.check_password(&check_value) + .map_err(map_rar50_crypto_error)?; + } + file.crypto = Some(FileCryptoState { + keys, + iv: encryption.iv, + }); + Ok(()) + } +} + +fn attach_service_crypto(service: &mut FileHeader, password: Option<&[u8]>) -> Result<()> { + // WinRAR can emit encrypted QO metadata whose service-local password + // check does not validate with the archive password. QuickOpen is an + // optional cache, so keep archive parsing and file extraction independent + // from that service. + if service.name == b"QO" { + return Ok(()); + } + attach_file_crypto(service, password) +} + +#[cfg(feature = "encryption")] +fn map_rar50_crypto_error(error: crate::crypto::rar50::Error) -> Error { + match error { + crate::crypto::rar50::Error::FeatureDisabled => Error::FeatureDisabled { + feature: "encryption", + }, + crate::crypto::rar50::Error::KdfCountTooLarge => Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 KDF count", + }, + crate::crypto::rar50::Error::BadPassword => Error::WrongPasswordOrCorruptData, + crate::crypto::rar50::Error::UnalignedInput => { + Error::InvalidHeader("RAR 5 AES input is not block aligned") + } + } +} + +fn read_array_at(input: &[u8], pos: &mut usize, end: usize) -> Result<[u8; N]> { + if pos.checked_add(N).is_none_or(|next| next > end) { + return Err(Error::TooShort); + } + let mut out = [0; N]; + out.copy_from_slice(&input[*pos..*pos + N]); + *pos += N; + Ok(out) +} + +fn parse_archive_blocks( + archive_len: usize, + options: crate::ArchiveReadOptions<'_>, + mut read_block: F, + mut read_encrypted_block: G, +) -> Result<(MainHeader, Vec)> +where + F: FnMut(usize, &mut crate::parse_budget::ParseBudget) -> Result, + G: FnMut(usize, &Rar50Keys, &mut crate::parse_budget::ParseBudget) -> Result, +{ + options.check_cancelled()?; + // Budget refusals already identify the header; do not duplicate its context. + let at_offset = |error: Error, offset| match error { + Error::AtArchiveOffset { .. } => error, + error => error.at_archive_offset(offset), + }; + let password = options.password; + let mut budget = crate::parse_budget::ParseBudget::new(options); + let mut pos = RAR50_SIGNATURE.len(); + let first = read_block(pos, &mut budget).map_err(|error| at_offset(error, pos))?; + let (header_keys, header_metadata_complete) = if first.block.header_type == HEAD_CRYPT { + pos = first.next_offset; + let (keys, complete) = parse_archive_encryption_header(&first, password)?; + (Some(keys), complete) + } else { + (None, true) + }; + + let main_pos = pos; + let main_block; + let first = if let Some(keys) = &header_keys { + main_block = + read_encrypted_block(pos, keys, &mut budget).map_err(|error| at_offset(error, pos))?; + &main_block + } else { + &first + }; + if first.block.header_type != HEAD_MAIN { + return Err(Error::InvalidHeader("RAR 5 main header is missing")); + } + let mut main = parse_main_header_bytes(first).map_err(|error| at_offset(error, main_pos))?; + main.encrypted_headers = header_keys.is_some(); + main.rewrite_metadata_complete &= header_metadata_complete; + pos = first.next_offset; + + let mut blocks = Vec::new(); + while pos < archive_len { + let parsed = if let Some(keys) = &header_keys { + read_encrypted_block(pos, keys, &mut budget).map_err(|error| at_offset(error, pos))? + } else { + read_block(pos, &mut budget).map_err(|error| at_offset(error, pos))? + }; + let next = parsed.next_offset; + match parsed.block.header_type { + HEAD_FILE => { + let mut file = + parse_file_header_bytes(&parsed).map_err(|error| at_offset(error, pos))?; + attach_file_crypto(&mut file, password).map_err(|error| at_offset(error, pos))?; + blocks.push(Block::File(file)); + } + HEAD_SERVICE => { + let mut service = + parse_file_header_bytes(&parsed).map_err(|error| at_offset(error, pos))?; + attach_service_crypto(&mut service, password) + .map_err(|error| at_offset(error, pos))?; + blocks.push(Block::Service(service)); + } + HEAD_CRYPT => { + return Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 encrypted headers", + }); + } + HEAD_END => { + main.rewrite_metadata_complete &= next == archive_len; + // A block with no room for the vint reads as no flags rather + // than as a broken archive. Hand-built and truncated archives + // do turn up with an empty end block, and the field only says + // whether to look for another volume. + let flags = read_vint_at( + &parsed.header, + parsed.type_specific_range.start, + parsed.type_specific_range.end, + ) + .map(|(flags, _)| flags) + .unwrap_or(0); + blocks.push(Block::End(EndHeader { + block: parsed.block, + flags, + })); + break; + } + _ => blocks.push(Block::Unknown(parsed.block)), + } + pos = next; + } + + main.rewrite_metadata_complete &= matches!(blocks.last(), Some(Block::End(_))); + Ok((main, blocks)) +} + +/// Walks the records of a RAR 5 extra area, handing each one to `handle`. +/// +/// A record that does not fit the area ends the walk instead of failing the +/// archive. RAR 7.12 and unrar 7.20 both extract normally from headers whose +/// extra area ends in a record claiming more bytes than are left, a size vint +/// cut off by the end of the area, or a record too small to hold its own type +/// vint. Rejecting the archive would throw away file data that is intact. +/// +/// On a service header, a single byte left over after a `SUBDATA` record is +/// folded into that record. WinRAR 5.21 and earlier stored the `SUBDATA` size +/// one less than the payload they wrote, and `SUBDATA` is the last record in +/// those headers, so the shortfall surfaces as exactly one dangling byte. +/// Without this, `RR` loses its recovery percent and `STM` loses the last +/// character of the stream name. Note that the reference readers give no way +/// to see the recovered byte from the outside: they accept the short shape, +/// but neither prints the recovery percent, so the byte itself is unverified. +fn parse_extra_records( + input: &[u8], + range: Range, + is_service: bool, + control: &crate::read_control::ReadControl, + mut handle: F, +) -> Result +where + F: FnMut(u64, Range) -> Result<()>, +{ + let mut pos = range.start; + let mut poller = control.poller(); + while pos < range.end { + poller.check(pos)?; + let Ok((record_size, size_len)) = read_vint_at(input, pos, range.end) else { + break; + }; + let payload_start = pos + size_len; + let Ok(record_payload_len) = usize::try_from(record_size) else { + break; + }; + let Some(mut record_end) = payload_start.checked_add(record_payload_len) else { + break; + }; + if record_end > range.end || record_end <= payload_start { + break; + } + let Ok((record_type, type_len)) = read_vint_at(input, payload_start, record_end) else { + break; + }; + if is_service && record_type == FHEXTRA_SUBDATA && range.end - record_end == 1 { + record_end = range.end; + } + handle(record_type, payload_start + type_len..record_end)?; + pos = record_end; + } + Ok(pos == range.end) +} + +struct ParsedBlockHeader { + control: crate::read_control::ReadControl, + block: BlockHeader, + header: Vec, + type_specific_range: Range, + extra_range: Range, + next_offset: usize, +} + +struct HeaderPrefix { + crc: u32, + size: u64, + size_len: usize, +} + +fn parse_block_header_bytes( + input: &[u8], + offset: usize, + archive_len: usize, + sfx_offset: usize, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + let remaining = archive_len.saturating_sub(offset); + if remaining < 5 { + return Err(Error::TooShort); + } + // The physical archive slice contains this complete five-byte prefix. + let header_crc = u32::from_le_bytes([ + input[offset], + input[offset + 1], + input[offset + 2], + input[offset + 3], + ]); + let after_crc = offset + 4; + let (header_size, header_size_len) = read_vint_at(input, after_crc, archive_len)?; + let header_body_len = usize_from_u64(header_size, "RAR 5 header size overflows usize")?; + let header_total = 4usize + .checked_add(header_size_len) + .and_then(|size| size.checked_add(header_body_len)) + .ok_or(Error::InvalidHeader("RAR 5 header size overflows usize"))?; + if header_total > remaining { + return Err(Error::TooShort); + } + budget.admit(header_total, offset)?; + // header_total <= archive_len-offset and archive_len == input.len(). + let header_end = offset + header_total; + let header = input[offset..header_end].to_vec(); + parse_block_header_image( + header, + offset, + archive_len, + sfx_offset, + HeaderPrefix { + crc: header_crc, + size: header_size, + size_len: header_size_len, + }, + header_total, + &budget.control, + ) +} + +fn parse_encrypted_block_header_bytes( + input: &[u8], + offset: usize, + archive_len: usize, + sfx_offset: usize, + keys: &Rar50Keys, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + #[cfg(not(feature = "encryption"))] + { + let _ = (input, offset, archive_len, sfx_offset, keys, budget); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + let remaining = archive_len.saturating_sub(offset); + if remaining < 32 { + return Err(Error::TooShort); + } + budget.check_count(offset)?; + // archive_len is this physical slice's length; remaining admitted 32 bytes. + let first = &input[offset..offset + 32]; + let mut iv = [0; 16]; + iv.copy_from_slice(&first[..16]); + let mut first_plain = [0; 16]; + first_plain.copy_from_slice(&first[16..32]); + Rar50Cipher::new(keys.key, iv).decrypt_block(&mut first_plain); + let header_crc = u32::from_le_bytes([ + first_plain[0], + first_plain[1], + first_plain[2], + first_plain[3], + ]); + let (header_size, header_size_len) = read_vint_at(&first_plain, 4, first_plain.len())?; + let header_body_len = usize_from_u64(header_size, "RAR 5 header size overflows usize")?; + let header_total = 4usize + .checked_add(header_size_len) + .and_then(|size| size.checked_add(header_body_len)) + .ok_or(Error::InvalidHeader("RAR 5 header size overflows usize"))?; + let encrypted_len = checked_align16(header_total, "RAR 5 encrypted header size overflows")?; + let disk_header_len = 16usize + .checked_add(encrypted_len) + .ok_or(Error::InvalidHeader( + "RAR 5 encrypted header size overflows", + ))?; + if disk_header_len > remaining { + return Err(Error::TooShort); + } + budget.admit(header_total, offset)?; + // The complete IV plus ciphertext length was admitted above. + let encrypted = &input[offset + 16..offset + disk_header_len]; + let mut header = encrypted.to_vec(); + Rar50Cipher::new(keys.key, iv) + .decrypt_in_place(&mut header) + .map_err(map_rar50_crypto_error)?; + header.truncate(header_total); + + parse_block_header_image( + header, + offset, + archive_len, + sfx_offset, + HeaderPrefix { + crc: header_crc, + size: header_size, + size_len: header_size_len, + }, + disk_header_len, + &budget.control, + ) + } +} + +fn read_block_header_at( + file: &mut (impl Read + std::io::Seek), + offset: usize, + archive_len: usize, + sfx_offset: usize, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + let remaining = archive_len.saturating_sub(offset); + if remaining < 5 { + return Err(Error::TooShort); + } + let prefix_len = remaining.min(14); + let prefix = read_exact_at(file, sfx_offset + offset, prefix_len)?; + // An exact prefix read returned at least five bytes above. + let header_crc = u32::from_le_bytes([prefix[0], prefix[1], prefix[2], prefix[3]]); + let (header_size, header_size_len) = read_vint_at(&prefix, 4, prefix.len())?; + let header_body_len = usize_from_u64(header_size, "RAR 5 header size overflows usize")?; + let header_total = 4usize + .checked_add(header_size_len) + .and_then(|size| size.checked_add(header_body_len)) + .ok_or(Error::InvalidHeader("RAR 5 header size overflows usize"))?; + if header_total > remaining { + return Err(Error::TooShort); + } + + budget.admit(header_total, offset)?; + let header = read_exact_at(file, sfx_offset + offset, header_total)?; + parse_block_header_image( + header, + offset, + archive_len, + sfx_offset, + HeaderPrefix { + crc: header_crc, + size: header_size, + size_len: header_size_len, + }, + header_total, + &budget.control, + ) +} + +fn read_encrypted_block_header_at( + file: &mut (impl Read + std::io::Seek), + offset: usize, + archive_len: usize, + sfx_offset: usize, + keys: &Rar50Keys, + budget: &mut crate::parse_budget::ParseBudget, +) -> Result { + #[cfg(not(feature = "encryption"))] + { + let _ = (file, offset, archive_len, sfx_offset, keys, budget); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + let remaining = archive_len.saturating_sub(offset); + if remaining < 32 { + return Err(Error::TooShort); + } + budget.check_count(offset)?; + let first = read_exact_at(file, sfx_offset + offset, 32)?; + let mut iv = [0; 16]; + iv.copy_from_slice(&first[..16]); + let mut first_plain = [0; 16]; + first_plain.copy_from_slice(&first[16..32]); + Rar50Cipher::new(keys.key, iv).decrypt_block(&mut first_plain); + let header_crc = u32::from_le_bytes([ + first_plain[0], + first_plain[1], + first_plain[2], + first_plain[3], + ]); + let (header_size, header_size_len) = read_vint_at(&first_plain, 4, first_plain.len())?; + let header_body_len = usize_from_u64(header_size, "RAR 5 header size overflows usize")?; + let header_total = 4usize + .checked_add(header_size_len) + .and_then(|size| size.checked_add(header_body_len)) + .ok_or(Error::InvalidHeader("RAR 5 header size overflows usize"))?; + let encrypted_len = checked_align16(header_total, "RAR 5 encrypted header size overflows")?; + let disk_header_len = 16usize + .checked_add(encrypted_len) + .ok_or(Error::InvalidHeader( + "RAR 5 encrypted header size overflows", + ))?; + if disk_header_len > remaining { + return Err(Error::TooShort); + } + budget.admit(header_total, offset)?; + let encrypted = read_exact_at(file, sfx_offset + offset + 16, encrypted_len)?; + let mut header = encrypted; + Rar50Cipher::new(keys.key, iv) + .decrypt_in_place(&mut header) + .map_err(map_rar50_crypto_error)?; + header.truncate(header_total); + + parse_block_header_image( + header, + offset, + archive_len, + sfx_offset, + HeaderPrefix { + crc: header_crc, + size: header_size, + size_len: header_size_len, + }, + disk_header_len, + &budget.control, + ) + } +} + +fn parse_block_header_image( + header: Vec, + offset: usize, + archive_len: usize, + sfx_offset: usize, + prefix: HeaderPrefix, + disk_header_len: usize, + control: &crate::read_control::ReadControl, +) -> Result { + control.check()?; + let header_total = header.len(); + // All four callers decoded this same immutable size prefix before reading + // or decrypting the complete header image. + validate_block_header_crc(&header, prefix.crc)?; + let type_start = 4 + prefix.size_len; + let mut reader = SliceReader::new(&header, type_start, header_total); + let header_type = reader.read_vint()?; + let flags = reader.read_vint()?; + let extra_area_size = if flags & HFL_EXTRA != 0 { + Some(reader.read_vint()?) + } else { + None + }; + let data_size = if flags & HFL_DATA != 0 { + Some(reader.read_vint()?) + } else { + None + }; + let extra_len = extra_area_size + .map(|size| usize_from_u64(size, "RAR 5 extra area size overflows usize")) + .transpose()? + .unwrap_or(0); + if extra_len > header_total.saturating_sub(reader.pos) { + return Err(Error::TooShort); + } + let type_specific_end = header_total - extra_len; + let data_len = data_size + .map(|size| usize_from_u64(size, "RAR 5 data size overflows usize")) + .transpose()? + .unwrap_or(0); + let next_offset = offset + .checked_add(disk_header_len) + .and_then(|pos| pos.checked_add(data_len)) + .ok_or(Error::InvalidHeader("RAR 5 data size overflows usize"))?; + if next_offset > archive_len { + return Err(Error::TooShort); + } + let type_specific_start = reader.pos; + // Both source adapters set archive_len = source_len-sfx_offset. Since + // next_offset <= archive_len, these absolute positions fit the source. + let data_start = sfx_offset + offset + disk_header_len; + let data_end = data_start + data_len; + + Ok(ParsedBlockHeader { + control: control.clone(), + block: BlockHeader { + header_crc: prefix.crc, + header_size: prefix.size, + header_type, + flags, + extra_area_size, + data_size, + offset: sfx_offset + offset, + header_range: (offset + type_specific_start)..(offset + type_specific_end), + data_range: data_start..data_end, + }, + header, + type_specific_range: type_specific_start..type_specific_end, + extra_range: type_specific_end..header_total, + next_offset, + }) +} + +fn validate_block_header_crc(header: &[u8], expected: u32) -> Result<()> { + // Every admitted header has a four-byte CRC and at least one size byte. + let actual = crc32(&header[4..]); + if actual != expected { + return Err(Error::Crc32Mismatch { expected, actual }); + } + Ok(()) +} + +struct HeaderReader<'a> { + input: &'a [u8], + range: Range, + pos: usize, +} + +impl<'a> HeaderReader<'a> { + fn new(input: &'a [u8], range: Range) -> Self { + // parse_block_header_image bounds this range within the header image. + Self { + input, + pos: range.start, + range, + } + } + + fn read_vint(&mut self) -> Result { + let (value, len) = read_vint_at(self.input, self.pos, self.range.end)?; + self.pos += len; + Ok(value) + } + + fn read_u32(&mut self) -> Result { + self.read_array::<4>().map(u32::from_le_bytes) + } + + #[cfg(feature = "encryption")] + fn read_byte(&mut self) -> Result { + if self.pos >= self.range.end { + return Err(Error::TooShort); + } + let value = self.input[self.pos]; + self.pos += 1; + Ok(value) + } + + fn read_array(&mut self) -> Result<[u8; N]> { + read_array_at::(self.input, &mut self.pos, self.range.end) + } + + fn read_bytes(&mut self, len: usize) -> Result<&'a [u8]> { + let end = self + .pos + .checked_add(len) + .ok_or(Error::InvalidHeader("RAR 5 field size overflows usize"))?; + if end > self.range.end { + return Err(Error::TooShort); + } + let bytes = &self.input[self.pos..end]; + self.pos = end; + Ok(bytes) + } +} + +struct SliceReader<'a> { + input: &'a [u8], + end: usize, + pos: usize, +} + +impl<'a> SliceReader<'a> { + fn new(input: &'a [u8], pos: usize, end: usize) -> Self { + Self { input, pos, end } + } + + fn read_vint(&mut self) -> Result { + let (value, len) = read_vint_at(self.input, self.pos, self.end)?; + self.pos += len; + Ok(value) + } + + fn read_u32(&mut self) -> Result { + let bytes = self.read_bytes(4)?; + Ok(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + + fn read_u64(&mut self) -> Result { + let bytes = self.read_bytes(8)?; + Ok(u64::from_le_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ])) + } + + fn read_bytes(&mut self, len: usize) -> Result<&'a [u8]> { + let end = self + .pos + .checked_add(len) + .ok_or(Error::InvalidHeader("RAR 5 field size overflows usize"))?; + if end > self.end { + return Err(Error::TooShort); + } + let bytes = &self.input[self.pos..end]; + self.pos = end; + Ok(bytes) + } +} + +fn read_vint_at(input: &[u8], offset: usize, end: usize) -> Result<(u64, usize)> { + let mut value = 0u64; + let mut shift = 0u32; + for i in 0..10 { + // Each previous byte was strictly before `end`, so this addition + // cannot overflow even when a caller supplies an extreme offset. + let pos = offset + i; + if pos >= end { + return Err(Error::TooShort); + } + let byte = *input.get(pos).ok_or(Error::TooShort)?; + if shift == 63 && byte & 0x7e != 0 { + return Err(Error::InvalidHeader("RAR 5 vint overflows u64")); + } + // Each group occupies disjoint bits; the final group's range was + // checked above, so combining groups cannot overflow. + value |= ((byte & 0x7f) as u64) << shift; + if byte & 0x80 == 0 { + return Ok((value, i + 1)); + } + shift += 7; + } + Err(Error::InvalidHeader("RAR 5 vint is too long")) +} + +fn usize_from_u64(value: u64, message: &'static str) -> Result { + usize::try_from(value).map_err(|_| Error::InvalidHeader(message)) +} + +fn compression_method(compression_info: u64) -> u64 { + (compression_info >> 7) & 0x07 +} + +fn decode_compression_info(raw: u64) -> Result { + let algorithm_version = (raw & 0x3f) as u8; + if algorithm_version > 1 { + return Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 unknown compression algorithm version", + }); + } + + let dictionary_power = ((raw >> 10) & 0x1f) as u8; + let dictionary_fraction = ((raw >> 15) & 0x1f) as u8; + let rar5_compat = raw & 0x100000 != 0; + if algorithm_version == 0 && (dictionary_fraction != 0 || rar5_compat) { + return Err(Error::InvalidHeader( + "RAR 5 v0 compression info uses v1 dictionary fields", + )); + } + if algorithm_version == 0 && dictionary_power > 15 { + return Err(Error::InvalidHeader( + "RAR 5 v0 dictionary power exceeds 4 GiB limit", + )); + } + + // The wire fields are five bits each: v1 shifts by at most 43 and + // multiplies by at most 63. V0 is restricted above to a shift of 15. + let dictionary_size = if algorithm_version == 1 { + u64::from(dictionary_fraction + 32) << (u32::from(dictionary_power) + 12) + } else { + (128 * 1024_u64) << u32::from(dictionary_power) + }; + + Ok(CompressionInfo { + algorithm_version, + solid: raw & 0x40 != 0, + method: ((raw >> 7) & 0x07) as u8, + dictionary_power, + dictionary_fraction, + rar5_compat, + dictionary_size, + }) +} + +#[cfg(all(test, feature = "write"))] +mod tests { + #[test] + fn file_encryption_record_accepts_optional_check_and_rejects_truncation() { + let mut without_check = vec![0, 0, 0]; // Version, flags, KDF count. + without_check.extend_from_slice(&[7; 16]); // Salt. + without_check.extend_from_slice(&[9; 16]); // IV. + let plain = parse_file_encryption_record(&without_check, 0..without_check.len()).unwrap(); + assert_eq!(plain.check_value, None); + assert_eq!(plain.salt, [7; 16]); + assert_eq!(plain.iv, [9; 16]); + + let mut with_check = without_check.clone(); + with_check[1] = 1; + with_check.extend_from_slice(&[3; 12]); + let checked = parse_file_encryption_record(&with_check, 0..with_check.len()).unwrap(); + assert_eq!(checked.check_value, Some([3; 12])); + + assert!(matches!( + parse_file_encryption_record(&[0, 0], 0..2), + Err(Error::TooShort) + )); + assert!(matches!( + parse_file_encryption_record( + &without_check[..without_check.len() - 1], + 0..without_check.len() - 1 + ), + Err(Error::TooShort) + )); + let mut trailing = without_check; + trailing.push(0); + assert!(matches!( + parse_file_encryption_record(&trailing, 0..trailing.len()), + Err(Error::InvalidHeader( + "RAR 5 file encryption record has trailing bytes" + )) + )); + + #[cfg(feature = "encryption")] + { + let archive = build_archive_with_optional_comment(None); + let mut file = archive.files().next().unwrap().clone(); + file.encrypted = true; + file.encryption = Some(plain.clone()); + attach_file_crypto(&mut file, Some(b"password")).unwrap(); + assert!(file.crypto.is_some()); + + let mut unsupported = archive.files().next().unwrap().clone(); + unsupported.encrypted = true; + unsupported.encryption = Some(FileEncryption { + version: 1, + ..plain + }); + assert!(matches!( + attach_file_crypto(&mut unsupported, Some(b"password")), + Err(Error::UnsupportedFeature { + feature: "RAR 5 unknown file encryption version", + .. + }) + )); + } + } + + #[test] + fn duplicate_and_unknown_file_extras_disable_rewrite_preservation() { + let archive = build_archive_with_optional_comment(None); + let original = archive.files().next().unwrap(); + let control = crate::read_control::ReadControl::default(); + let mut hash_record = vec![34, FHEXTRA_HASH as u8, 0]; + hash_record.extend_from_slice(&[7; 32]); + + let mut single = original.clone(); + single.block.extra_area_size = Some(hash_record.len() as u64); + parse_file_extra_area( + &hash_record, + 0..hash_record.len(), + false, + &mut single, + &control, + ) + .unwrap(); + assert!(single.rewrite_metadata_complete); + assert!(single.hash.is_some()); + + let duplicate = [hash_record.as_slice(), hash_record.as_slice()].concat(); + let mut repeated = original.clone(); + repeated.block.extra_area_size = Some(duplicate.len() as u64); + parse_file_extra_area( + &duplicate, + 0..duplicate.len(), + false, + &mut repeated, + &control, + ) + .unwrap(); + assert!(!repeated.rewrite_metadata_complete); + assert!(repeated.hash.is_some()); + + let unknown = [1, 64]; + let mut future = original.clone(); + future.block.extra_area_size = Some(unknown.len() as u64); + parse_file_extra_area(&unknown, 0..unknown.len(), false, &mut future, &control).unwrap(); + assert!(!future.rewrite_metadata_complete); + } + + #[test] + fn preservation_preflight_rejects_unknown_parsed_file_and_comment_extras() { + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50) + .store(true) + .comment(Some(b"archive note".to_vec())); + builder + .add_bytes( + b"payload.txt".to_vec(), + b"payload bytes".to_vec(), + None, + None, + ) + .unwrap(); + let crate::Archive::Rar50Plus(seed) = + crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + else { + panic!("expected RAR5 archive") + }; + let baseline = crate::Archive::Rar50Plus(seed.clone()).rewrite_preservation_issues(); + assert!(baseline.is_empty(), "{baseline:?}"); + + // An extra byte in a CRC-valid main header is readable, but cannot + // be claimed to survive a rewrite. + let mut header = vec![4, HEAD_MAIN as u8, 0, 0, 0]; + let mut malformed = RAR50_SIGNATURE.to_vec(); + malformed.extend_from_slice(&crc32(&header).to_le_bytes()); + malformed.append(&mut header); + let parsed = Archive::parse(&malformed).unwrap(); + assert!(!parsed.main.rewrite_metadata_complete); + assert!(crate::Archive::Rar50Plus(parsed) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("main header metadata"))); + + let unknown = [1, 64]; + let control = crate::read_control::ReadControl::default(); + for service in [false, true] { + let mut candidate = seed.clone(); + let file = candidate + .blocks + .iter_mut() + .find_map(|block| match block { + Block::File(file) if !service => Some(file), + Block::Service(file) if service && file.name == b"CMT" => Some(file), + _ => None, + }) + .unwrap(); + file.block.extra_area_size = Some(unknown.len() as u64); + parse_file_extra_area(&unknown, 0..unknown.len(), service, file, &control).unwrap(); + assert!(!file.rewrite_metadata_complete); + + let issues = crate::Archive::Rar50Plus(candidate).rewrite_preservation_issues(); + let expected = if service { + "service record" + } else { + "member 0" + }; + assert!( + issues.iter().any(|issue| issue.contains(expected)), + "{issues:?}" + ); + } + + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50) + .store(true) + .archive_metadata(None, false, true) + .unwrap(); + builder + .add_bytes( + b"payload.txt".to_vec(), + b"payload bytes".to_vec(), + None, + None, + ) + .unwrap(); + let crate::Archive::Rar50Plus(mut indexed) = + crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + else { + panic!("expected RAR5 archive") + }; + assert!(crate::Archive::Rar50Plus(indexed.clone()) + .rewrite_preservation_issues() + .is_empty()); + let quick_open = indexed + .blocks + .iter_mut() + .find_map(|block| match block { + Block::Service(service) if service.name == b"QO" => Some(service), + _ => None, + }) + .unwrap(); + quick_open.block.extra_area_size = Some(unknown.len() as u64); + parse_file_extra_area(&unknown, 0..unknown.len(), true, quick_open, &control).unwrap(); + assert!(!quick_open.rewrite_metadata_complete); + assert!(crate::Archive::Rar50Plus(indexed) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("unsupported derived service"))); + } + + #[test] + fn file_extra_records_preserve_readable_metadata_and_limit_rewrites() { + let archive = build_archive_with_optional_comment(None); + let original = archive.files().next().unwrap(); + let control = crate::read_control::ReadControl::default(); + let parse = |kind: u8, payload: &[u8], service: bool, name: &[u8], mtime| { + let mut record = vec![(payload.len() + 1) as u8, kind]; + record.extend_from_slice(payload); + let mut file = original.clone(); + file.name = name.to_vec(); + file.mtime = mtime; + file.block.extra_area_size = Some(record.len() as u64); + parse_file_extra_area(&record, 0..record.len(), service, &mut file, &control).unwrap(); + file + }; + for (kind, size, complete) in [ + (0, 32, true), + (1, 32, false), + (0, 31, false), + (0, 33, false), + ] { + let mut payload = vec![kind]; + payload.extend(vec![7; size]); + let file = parse(FHEXTRA_HASH as u8, &payload, false, b"file", None); + assert_eq!(file.rewrite_metadata_complete, complete); + assert_eq!(file.hash.unwrap().data, vec![7; size]); + } + for (version, flags, complete) in [(0, 0, true), (0, 2, true), (0, 4, false), (1, 0, false)] + { + let mut payload = vec![version, flags, 0]; + payload.extend_from_slice(&[3; 16]); + payload.extend_from_slice(&[4; 16]); + let file = parse(FHEXTRA_CRYPT as u8, &payload, false, b"file", None); + assert!(file.encrypted); + assert_eq!(file.encryption.unwrap().flags, u64::from(flags)); + assert_eq!(file.rewrite_metadata_complete, complete); + } + for (service, kind, complete) in [(false, 1, true), (true, 1, false), (false, 6, false)] { + let file = parse( + FHEXTRA_REDIR as u8, + &[kind, 0, 1, b'x'], + service, + b"file", + None, + ); + assert_eq!(file.redirection.unwrap().target_name, b"x"); + assert_eq!(file.rewrite_metadata_complete, complete); + } + for (service, name, payload, complete) in [ + (false, b"file".as_slice(), b"".as_slice(), false), + (true, b"CMT".as_slice(), b"".as_slice(), true), + (true, b"CMT".as_slice(), b"x".as_slice(), false), + (true, b"RR".as_slice(), b"x".as_slice(), true), + ] { + let file = parse(FHEXTRA_SUBDATA as u8, payload, service, name, None); + assert_eq!(file.service_data.unwrap(), payload); + assert_eq!(file.rewrite_metadata_complete, complete); + } + for (flags, mtime, complete) in [(3, None, true), (3, Some(9), false), (5, Some(9), true)] { + let mut payload = vec![flags]; + payload.extend_from_slice(&123u32.to_le_bytes()); + let file = parse(FHEXTRA_HTIME as u8, &payload, false, b"file", mtime); + assert!(file.file_times.is_some()); + assert_eq!(file.htime_mtime, (flags == 3).then_some(123)); + assert_eq!(file.rewrite_metadata_complete, complete); + } + for payload in [vec![], vec![0x80], vec![3, 1], vec![0x23, 1, 0, 0, 0]] { + let file = parse(FHEXTRA_HTIME as u8, &payload, false, b"file", None); + assert!(file.file_times.is_none()); + assert!(!file.rewrite_metadata_complete); + } + let mut no_extras = original.clone(); + no_extras.block.extra_area_size = None; + parse_file_extra_area(&[0x80], 0..1, false, &mut no_extras, &control).unwrap(); + assert!(no_extras.rewrite_metadata_complete); + } + + #[test] + fn htime_mtime_retains_valid_seconds_at_format_boundaries() { + for bytes in [vec![], vec![0x80], vec![1], vec![5, 1, 0, 0, 0], vec![3, 1]] { + assert_eq!(parse_htime_mtime(&bytes, 0..bytes.len()), None); + } + for flags in [0x13u8, 0x17, 0x1b, 0x1f] { + for nanos in [0u32, 999_999_999] { + let mut bytes = vec![flags]; + for _ in 0..(flags & 0x0e).count_ones() { + bytes.extend_from_slice(&u32::MAX.to_le_bytes()); + } + bytes.extend_from_slice(&nanos.to_le_bytes()); + let (seconds, detail) = parse_htime_mtime(&bytes, 0..bytes.len()).unwrap(); + assert_eq!(seconds, u32::MAX); + assert_eq!(detail.unwrap().nanoseconds, nanos); + } + } + let epoch = 116_444_736_000_000_000u64; + for ticks in [epoch - 1, epoch + (u64::from(u32::MAX) + 1) * 10_000_000] { + let mut bytes = vec![2]; + bytes.extend_from_slice(&ticks.to_le_bytes()); + assert_eq!(parse_htime_mtime(&bytes, 0..bytes.len()), None); + // The lossless metadata representation still preserves FILETIME + // when the convenient u32 Unix-seconds view cannot represent it. + let archive = build_archive_with_optional_comment(None); + let mut file = archive.files().next().unwrap().clone(); + let mut record = vec![(bytes.len() + 1) as u8, FHEXTRA_HTIME as u8]; + record.extend_from_slice(&bytes); + file.block.extra_area_size = Some(record.len() as u64); + parse_file_extra_area( + &record, + 0..record.len(), + false, + &mut file, + &crate::read_control::ReadControl::default(), + ) + .unwrap(); + assert!(file.rewrite_metadata_complete); + assert_eq!( + file.file_times.unwrap().modified, + Some(crate::FileTimestamp::WindowsFiletime(ticks)) + ); + } + } + + #[test] + fn compression_dictionary_fields_obey_version_and_wire_limits() { + for raw in [2, 63] { + assert!(matches!( + decode_compression_info(raw), + Err(Error::UnsupportedFeature { .. }) + )); + } + for raw in [1 << 15, 0x100000, 31 << 10] { + assert!(matches!( + decode_compression_info(raw), + Err(Error::InvalidHeader(_)) + )); + } + for power in 0..=31u64 { + for fraction in 0..=31u64 { + let raw = 1 | (power << 10) | (fraction << 15); + assert_eq!( + decode_compression_info(raw).unwrap().dictionary_size, + (fraction + 32) * (1u64 << (power + 12)) + ); + } + if power <= 15 { + assert_eq!( + decode_compression_info(power << 10) + .unwrap() + .dictionary_size, + 131_072 * (1u64 << power) + ); + } + } + } + + #[test] + fn audit_redirection_identity_and_volume_metadata() { + for (kind, flags, name, supported) in [ + (1, 0, b"target".as_slice(), true), + (1, 2, b"target", false), + (4, 1, b"target", false), + (4, 0, b"target", true), + (1, 0, b"", false), + (1, 0, b"a\0b", false), + (1, 0, b"\xff", false), + (6, 0, b"target", false), + ] { + assert_eq!( + FileRedirection { + redirection_type: kind, + flags, + target_name: name.to_vec() + } + .is_supported(), + supported + ); + } + for (kind, flags, host, attr, directory, expected) in [ + (1, 0, 1, 0o120777, false, true), + (2, 0, 0, 0x400, false, true), + (3, 1, 0, 0x410, true, true), + (4, 0, 0, 0, false, true), + (5, 0, 1, 0o100644, false, true), + (6, 0, 1, 0o120777, false, false), + ] { + let link = FileRedirection { + redirection_type: kind, + flags, + target_name: b"target".to_vec(), + }; + assert_eq!(link.supports_header(host, attr, directory), expected); + } + let mut archive = build_archive_with_optional_comment(None); + assert!(!archive.main.is_locked()); + archive.main.archive_flags |= MHFL_LOCKED; + assert!(archive.main.is_locked()); + archive.main.extras = vec![MainExtraRecord::ArchiveMetadata(ArchiveMetadataRecord { + flags: 0, + name: None, + creation_time: None, + })]; + assert!(archive.main.locator().is_none()); + archive.main.extras = vec![MainExtraRecord::Locator(LocatorRecord { + flags: 0, + quick_open_offset: None, + recovery_record_offset: None, + })]; + assert!(archive.main.locator().is_some()); + for block in &mut archive.blocks { + if let Block::End(end) = block { + assert!(!end.has_next_volume()); + end.flags |= EFL_NEXT_VOLUME; + assert!(end.has_next_volume()); + } + } + } + + #[test] + fn explicit_integrity_checks_cover_absent_invalid_and_encrypted_records() { + let archive = build_archive_with_optional_comment(None); + let mut file = archive.files().next().unwrap().clone(); + let data = b"payload bytes"; + file.verify_integrity(data).unwrap(); + file.verify_crc32(data).unwrap(); + assert!(matches!( + file.verify_crc32(b"damaged"), + Err(Error::Crc32Mismatch { .. }) + )); + file.data_crc32 = None; + file.verify_crc32(b"anything").unwrap(); + file.hash = None; + file.verify_hash(b"anything").unwrap(); + file.hash = Some(FileHash { + hash_type: 0, + data: blake2sp::hash(data).to_vec(), + }); + file.verify_hash(data).unwrap(); + assert!(matches!( + file.verify_hash(b"damaged"), + Err(Error::HashMismatch { hash_type: 0 }) + )); + file.hash.as_mut().unwrap().data.pop(); + assert!(matches!( + file.verify_hash(data), + Err(Error::InvalidHeader(_)) + )); + file.hash.as_mut().unwrap().hash_type = 1; + assert!(matches!( + file.verify_hash(data), + Err(Error::UnsupportedFeature { .. }) + )); + file.encryption = Some(FileEncryption { + version: 0, + flags: 2, + kdf_count: 0, + salt: [0; 16], + iv: [0; 16], + check_value: None, + }); + assert!(matches!( + file.verify_hash(data), + Err(Error::InvalidHeader(_)) + )); + file.data_crc32 = Some(crc32(data)); + assert!(matches!( + file.verify_crc32(data), + Err(Error::InvalidHeader(_)) + )); + } + + #[test] + fn combined_integrity_check_uses_authoritative_blake2sp_record() { + // The fixture is independently accepted by RAR and UnRAR despite its + // deliberately wrong CRC32 beside the valid BLAKE2sp record. + let archive = Archive::parse(include_bytes!( + "../tests/fixtures/rar50/crc32_wrong_beside_blake2sp.rar" + )) + .unwrap(); + let file = archive.files().next().unwrap(); + let data = file.packed_data(&archive).unwrap(); + assert!(matches!( + file.verify_crc32(&data), + Err(Error::Crc32Mismatch { .. }) + )); + file.verify_hash(&data).unwrap(); + file.verify_integrity(&data).unwrap(); + } + + #[test] + fn recovery_record_distinguishes_other_services_and_rejects_missing_or_extra_data() { + let archive = build_archive_with_optional_comment(None); + let mut file = archive.files().next().unwrap().clone(); + assert_eq!(file.recovery_record().unwrap(), None); + + file.name = b"RR".to_vec(); + assert!(matches!( + file.recovery_record(), + Err(Error::InvalidHeader( + "RAR 5 recovery service is missing service data" + )) + )); + file.service_data = Some(Vec::new()); + assert_eq!(file.recovery_record(), Err(Error::TooShort)); + file.service_data = Some(vec![5, 0]); + assert!(matches!( + file.recovery_record(), + Err(Error::InvalidHeader( + "RAR 5 recovery service data has trailing bytes" + )) + )); + file.service_data = Some(vec![5]); + let record = file.recovery_record().unwrap().unwrap(); + assert_eq!(record.percent, 5); + assert_eq!(record.payload_size, file.packed_size()); + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_entry_points_preserve_missing_record_and_precancellation() { + let archive = build_archive_with_optional_comment(None); + assert!(matches!( + archive.repair_recovery_to(&mut Vec::new()), + Err(Error::InvalidHeader( + "RAR 5 archive does not contain an inline recovery record" + )) + )); + let cancelled = crate::ReadCancellation::new(); + cancelled.cancel(); + let options = crate::ArchiveReadOptions::new().with_cancellation(&cancelled); + let mut output = Vec::new(); + assert!(matches!( + archive.repair_recovery_to_with_options(&mut output, options), + Err(Error::Cancelled) + )); + assert!(output.is_empty()); + assert!(matches!( + repair_inline_recovery_bytes_with_options(RAR50_SIGNATURE, options), + Err(Error::Cancelled) + )); + assert!(matches!( + recovery_end_header(RAR50_SIGNATURE, options), + Err(Error::Cancelled) + )); + assert!(matches!( + repair_inline_recovery_bytes_with_options( + RAR50_SIGNATURE, + crate::ArchiveReadOptions::new() + ), + Err(Error::Rar5Recovery( + crate::recovery::rar5::Error::BadRecoveryChunk + )) + )); + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_end_header_preserves_split_volume_continuation() { + let first = include_bytes!("../tests/fixtures/rar50/multivol.part1.rar"); + let last = include_bytes!("../tests/fixtures/rar50/multivol.part3.rar"); + for (bytes, expected) in [(first.as_slice(), EFL_NEXT_VOLUME), (last.as_slice(), 0)] { + let archive = Archive::parse(bytes).unwrap(); + assert_eq!( + archive.files().last().unwrap().is_split_after(), + expected != 0 + ); + let end = recovery_end_header(bytes, crate::ArchiveReadOptions::new()).unwrap(); + let parsed = parse_block_header_bytes( + &end, + 0, + end.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ) + .unwrap(); + assert_eq!(parsed.block.header_type, HEAD_END); + let mut reader = HeaderReader::new(&parsed.header, parsed.type_specific_range); + assert_eq!(reader.read_vint().unwrap(), expected); + } + + assert!(matches!( + recovery_end_header(RAR50_SIGNATURE, crate::ArchiveReadOptions::new()), + Err(Error::TooShort) + )); + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_end_header_keeps_parse_cancellation_distinct() { + assert!(matches!( + recovery_end_parse_error(Error::Cancelled), + Err(Error::Cancelled) + )); + assert_eq!(recovery_end_parse_error(Error::TooShort).unwrap(), 0); + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_end_header_requires_keys_for_encrypted_archive() { + let bytes = include_bytes!("../tests/fixtures/rar50/header_encrypted.rar"); + #[cfg(feature = "encryption")] + { + assert!(matches!( + recovery_end_header(bytes, crate::ArchiveReadOptions::new()), + Err(Error::NeedPassword) + )); + assert!(matches!( + recovery_end_header(bytes, crate::ArchiveReadOptions::with_password(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + assert!(!recovery_end_header( + bytes, + crate::ArchiveReadOptions::with_password(b"password") + ) + .unwrap() + .is_empty()); + } + #[cfg(not(feature = "encryption"))] + for options in [ + crate::ArchiveReadOptions::new(), + crate::ArchiveReadOptions::with_password(b"wrong"), + crate::ArchiveReadOptions::with_password(b"password"), + ] { + let error = recovery_end_header(bytes, options).unwrap_err(); + assert!( + matches!( + error.root_cause(), + Error::FeatureDisabled { + feature: "encryption" + } + ), + "{error:?}" + ); + } + } + + #[test] + fn header_fields_reject_truncation_overflow_and_unexpected_encryption() { + let image = |body: &[u8]| { + assert!(body.len() < 128); + let mut bytes = vec![body.len() as u8]; + bytes.extend_from_slice(body); + let mut header = crc32(&bytes).to_le_bytes().to_vec(); + header.extend_from_slice(&bytes); + header + }; + let parse = |bytes: &[u8]| { + parse_block_header_bytes( + bytes, + 0, + bytes.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ) + }; + for flags in [FHFL_MTIME, FHFL_CRC32] { + let header = image(&[HEAD_FILE as u8, 0, flags as u8, 0, 0]); + let parsed = parse(&header).unwrap(); + assert!(matches!( + parse_file_header_bytes(&parsed), + Err(Error::TooShort) + )); + } + let header = image(&[HEAD_FILE as u8, 0, 0, 0, 0, 0, 0, 10]); + assert!(matches!( + parse_file_header_bytes(&parse(&header).unwrap()), + Err(Error::TooShort) + )); + let header = image(&[HEAD_FILE as u8, HFL_DATA as u8, 1]); + assert!(matches!(parse(&header), Err(Error::TooShort))); + let mut empty = RAR50_SIGNATURE.to_vec(); + empty.extend_from_slice(&image(&[HEAD_MAIN as u8, 0, 0])); + assert!(Archive::parse(&empty).unwrap().blocks.is_empty()); + empty = RAR50_SIGNATURE.to_vec(); + empty.extend_from_slice(&image(&[HEAD_MAIN as u8, 0, 0, 0])); + assert!( + !Archive::parse(&empty) + .unwrap() + .main + .rewrite_metadata_complete + ); + let maximum = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 1]; + let mut body = vec![HEAD_FILE as u8, 0, 0, 0, 0, 0, 0]; + body.extend_from_slice(&maximum); + let header = image(&body); + let expected_name_error = if usize::BITS < 64 { + "RAR 5 file name length overflows usize" + } else { + "RAR 5 field size overflows usize" + }; + assert!(matches!( + parse_file_header_bytes(&parse(&header).unwrap()), + Err(Error::InvalidHeader(message)) if message == expected_name_error + )); + let header = image(&[HEAD_FILE as u8, HFL_EXTRA as u8, 20, 0]); + assert!(matches!(parse(&header), Err(Error::TooShort))); + let mut body = vec![HEAD_FILE as u8, HFL_DATA as u8]; + body.extend_from_slice(&maximum); + assert!(matches!(parse(&image(&body)), Err(Error::InvalidHeader(_)))); + + empty = RAR50_SIGNATURE.to_vec(); + empty.extend_from_slice(&image(&[HEAD_MAIN as u8, HFL_EXTRA as u8, 1, 0, 0])); + assert!( + !Archive::parse(&empty) + .unwrap() + .main + .rewrite_metadata_complete + ); + + #[cfg(feature = "encryption")] + { + // A crypt header with a valid CRC can still truncate the KDF count, + // salt or password-check field. Test every field boundary explicitly. + for flags in [0, 1] { + let mut body = vec![HEAD_CRYPT as u8, 0, 0, flags, 0]; + body.extend_from_slice(&[0; 16]); + if flags == 1 { + body.extend_from_slice(&[0; 12]); + } + for end in 4..body.len() { + let header = image(&body[..end]); + let parsed = parse(&header).unwrap(); + assert!( + matches!( + parse_archive_encryption_header(&parsed, Some(b"pw")), + Err(Error::TooShort) + ), + "flags {flags}, end {end}" + ); + } + if flags == 0 { + let header = image(&body); + parse_archive_encryption_header(&parse(&header).unwrap(), Some(b"pw")).unwrap(); + body.push(0); + let header = image(&body); + assert!(matches!( + parse_archive_encryption_header(&parse(&header).unwrap(), Some(b"pw")), + Err(Error::InvalidHeader( + "RAR 5 archive encryption header has trailing bytes" + )) + )); + } + } + } + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&image(&[HEAD_MAIN as u8, 0, 0])); + bytes.extend_from_slice(&image(&[HEAD_CRYPT as u8, 0])); + assert!(matches!( + Archive::parse(&bytes).unwrap_err().root_cause(), + Error::UnsupportedFeature { + feature: "RAR 5 encrypted headers", + .. + } + )); + } + + #[test] + fn oversized_header_declarations_refuse_before_payload_access() { + fn vint(mut value: u64) -> Vec { + let mut bytes = Vec::new(); + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + bytes.push(byte | if value != 0 { 0x80 } else { 0 }); + if value == 0 { + return bytes; + } + } + } + + #[cfg(feature = "encryption")] + let keys = Rar50Keys::derive(b"pw", [0; 16], 0).unwrap(); + for (declared, _encrypted_expected) in [ + (u64::MAX, "RAR 5 header size overflows usize"), + (u64::MAX - 28, "RAR 5 encrypted header size overflows"), + (u64::MAX - 34, "RAR 5 encrypted header size overflows"), + ] { + let mut prefix = vec![0; 4]; + prefix.extend_from_slice(&vint(declared)); + assert_eq!(prefix.len(), 14); + let memory = parse_block_header_bytes( + &prefix, + 0, + prefix.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ); + let seekable = read_block_header_at( + &mut std::io::Cursor::new(&prefix), + 0, + prefix.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ); + for result in [memory, seekable] { + if usize::BITS == 32 || declared == u64::MAX { + assert!(matches!( + result, + Err(Error::InvalidHeader("RAR 5 header size overflows usize")) + )); + } else { + assert!(matches!(result, Err(Error::TooShort))); + } + } + + #[cfg(feature = "encryption")] + { + let mut first_plain = prefix; + first_plain.resize(16, 0); + Rar50Cipher::new(keys.key, [0; 16]) + .encrypt_in_place(&mut first_plain) + .unwrap(); + let mut encrypted = vec![0; 16]; + encrypted.extend_from_slice(&first_plain); + let memory = parse_encrypted_block_header_bytes( + &encrypted, + 0, + encrypted.len(), + 0, + &keys, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ); + let seekable = read_encrypted_block_header_at( + &mut std::io::Cursor::new(&encrypted), + 0, + encrypted.len(), + 0, + &keys, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ); + for result in [memory, seekable] { + let expected = if usize::BITS == 32 { + "RAR 5 header size overflows usize" + } else { + _encrypted_expected + }; + assert!( + matches!(result, Err(Error::InvalidHeader(message)) if message == expected) + ); + } + } + } + } + + #[test] + fn malformed_header_size_vints_and_optional_lengths_reach_both_adapters() { + let mut overlong = vec![0; 4]; + overlong.extend_from_slice(&[0x80; 10]); + for result in [ + parse_block_header_bytes( + &overlong, + 0, + overlong.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + read_block_header_at( + &mut std::io::Cursor::new(&overlong), + 0, + overlong.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + ] { + assert!(matches!( + result, + Err(Error::InvalidHeader("RAR 5 vint is too long")) + )); + } + #[cfg(feature = "encryption")] + { + let keys = Rar50Keys::derive(b"pw", [0; 16], 0).unwrap(); + let mut first_plain = overlong; + first_plain.resize(16, 0); + Rar50Cipher::new(keys.key, [0; 16]) + .encrypt_in_place(&mut first_plain) + .unwrap(); + let mut encrypted = vec![0; 16]; + encrypted.extend_from_slice(&first_plain); + for result in [ + parse_encrypted_block_header_bytes( + &encrypted, + 0, + encrypted.len(), + 0, + &keys, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + read_encrypted_block_header_at( + &mut std::io::Cursor::new(&encrypted), + 0, + encrypted.len(), + 0, + &keys, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + ] { + assert!(matches!( + result, + Err(Error::InvalidHeader("RAR 5 vint is too long")) + )); + } + } + let maximum_vint = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 1]; + for (flag, expected) in [ + (HFL_EXTRA, "RAR 5 extra area size overflows usize"), + (HFL_DATA, "RAR 5 data size overflows usize"), + ] { + let mut body = vec![HEAD_FILE as u8, flag as u8]; + body.extend_from_slice(&maximum_vint); + let mut encoded = vec![body.len() as u8]; + encoded.extend_from_slice(&body); + let mut header = crc32(&encoded).to_le_bytes().to_vec(); + header.extend_from_slice(&encoded); + for result in [ + parse_block_header_bytes( + &header, + 0, + header.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + read_block_header_at( + &mut std::io::Cursor::new(&header), + 0, + header.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + ] { + if flag == HFL_EXTRA && usize::BITS == 64 { + assert!(matches!(result, Err(Error::TooShort))); + } else { + assert!( + matches!(result, Err(Error::InvalidHeader(message)) if message == expected) + ); + } + } + } + } + + #[test] + fn file_header_field_boundaries_agree_across_reader_adapters() { + fn image(body: &[u8]) -> Vec { + let mut encoded = vec![body.len() as u8]; + encoded.extend_from_slice(body); + let mut header = crc32(&encoded).to_le_bytes().to_vec(); + header.extend_from_slice(&encoded); + header + } + // Type, block flags, file flags, unpacked size, attributes, + // compression, host, name length and one name byte. + let complete = [HEAD_FILE as u8, 0, 0, 0, 0, 0, 0, 1, b'x']; + for included in 0..=complete.len() { + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&image(&[HEAD_MAIN as u8, 0, 0])); + bytes.extend_from_slice(&image(&complete[..included])); + for result in [ + Archive::parse(&bytes), + Archive::parse_file_backed( + &mut std::io::Cursor::new(&bytes), + bytes.len(), + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ), + ] { + if included < complete.len() { + assert!( + matches!(result.unwrap_err().root_cause(), Error::TooShort), + "field prefix {included}" + ); + } else { + let archive = result.unwrap(); + assert_eq!(archive.files().next().unwrap().name, b"x"); + } + } + } + } + + #[test] + fn optional_block_header_fields_need_their_own_vints() { + for (body, complete) in [ + (vec![HEAD_FILE as u8, HFL_EXTRA as u8], false), + (vec![HEAD_FILE as u8, HFL_DATA as u8], false), + ( + vec![HEAD_FILE as u8, (HFL_EXTRA | HFL_DATA) as u8, 0], + false, + ), + ( + vec![HEAD_FILE as u8, (HFL_EXTRA | HFL_DATA) as u8, 0, 0], + true, + ), + ] { + let mut encoded = vec![body.len() as u8]; + encoded.extend_from_slice(&body); + let mut header = crc32(&encoded).to_le_bytes().to_vec(); + header.extend_from_slice(&encoded); + for result in [ + parse_block_header_bytes( + &header, + 0, + header.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + read_block_header_at( + &mut std::io::Cursor::new(&header), + 0, + header.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ), + ] { + if complete { + let parsed = result.unwrap(); + assert_eq!(parsed.block.extra_area_size, Some(0)); + assert_eq!(parsed.block.data_size, Some(0)); + } else { + assert!(matches!(result, Err(Error::TooShort))); + } + } + } + } + + #[test] + fn file_extra_field_failures_reach_both_archive_readers() { + fn image(body: &[u8]) -> Vec { + let mut encoded = vec![body.len() as u8]; + encoded.extend_from_slice(body); + let mut header = crc32(&encoded).to_le_bytes().to_vec(); + header.extend_from_slice(&encoded); + header + } + let mut records = Vec::new(); + for mut crypt in [vec![0, 0, 0], vec![0, 1, 0]] { + crypt.extend_from_slice(&[0; 32]); // Salt and IV. + if crypt[1] == 1 { + crypt.extend_from_slice(&[0; 12]); // Optional password check. + } + for included in 0..=crypt.len() { + records.push(( + FHEXTRA_CRYPT as u8, + crypt[..included].to_vec(), + included == crypt.len(), + )); + } + } + records.extend([ + (FHEXTRA_HASH as u8, vec![], false), + (FHEXTRA_HASH as u8, vec![0], true), + (FHEXTRA_REDIR as u8, vec![], false), + (FHEXTRA_REDIR as u8, vec![1], false), + (FHEXTRA_REDIR as u8, vec![1, 0], false), + (FHEXTRA_REDIR as u8, vec![1, 0, 1], false), + (FHEXTRA_REDIR as u8, vec![1, 0, 1, b'y'], true), + ]); + for (kind, record, valid) in records { + let mut extra = vec![(record.len() + 1) as u8, kind]; + extra.extend_from_slice(&record); + let mut file_body = vec![ + HEAD_FILE as u8, + HFL_EXTRA as u8, + extra.len() as u8, + 0, + 0, + 0, + 0, + 0, + 1, + b'x', + ]; + file_body.extend_from_slice(&extra); + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&image(&[HEAD_MAIN as u8, 0, 0])); + let offset = bytes.len(); + bytes.extend_from_slice(&image(&file_body)); + for result in [ + Archive::parse(&bytes), + Archive::parse_file_backed( + &mut std::io::Cursor::new(&bytes), + bytes.len(), + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ), + ] { + if valid { + assert_eq!(result.unwrap().files().count(), 1); + } else { + let error = result.unwrap_err(); + assert!( + matches!(error, Error::AtArchiveOffset { offset: actual, .. } if actual == offset) + ); + assert!(matches!( + error.root_cause(), + Error::TooShort | Error::InvalidHeader(_) + )); + } + } + } + } + + #[test] + fn main_volume_and_first_crypt_fields_reject_truncation() { + fn image(body: &[u8]) -> Vec { + let mut encoded = vec![body.len() as u8]; + encoded.extend_from_slice(body); + let mut header = crc32(&encoded).to_le_bytes().to_vec(); + header.extend_from_slice(&encoded); + header + } + for body in [ + &[HEAD_MAIN as u8, 0, MHFL_VOLUME_NUMBER as u8][..], + &[HEAD_CRYPT as u8, 0][..], + &[HEAD_CRYPT as u8, 0, 0][..], + ] { + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&image(body)); + for result in [ + Archive::parse(&bytes), + Archive::parse_file_backed( + &mut std::io::Cursor::new(&bytes), + bytes.len(), + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + crate::ArchiveReadOptions::new(), + ), + ] { + let error = result.unwrap_err(); + let expected = if body[0] == HEAD_CRYPT as u8 && !cfg!(feature = "encryption") { + Error::FeatureDisabled { + feature: "encryption", + } + } else { + Error::TooShort + }; + assert_eq!(error.root_cause(), &expected); + } + } + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&image(&[HEAD_MAIN as u8, 0, MHFL_VOLUME_NUMBER as u8, 3])); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.main.volume_number, Some(3)); + } + + #[test] + fn seekable_parser_preserves_input_faults_and_cancellation() { + use std::io::{Cursor, Read, Seek, SeekFrom}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FaultReader { + input: Cursor>, + operations: Arc, + stop_at: usize, + cancellation: Option, + } + impl FaultReader { + fn check(&self) -> std::io::Result<()> { + let operation = self.operations.fetch_add(1, Ordering::Relaxed); + if operation != self.stop_at { + return Ok(()); + } + if let Some(token) = &self.cancellation { + token.cancel(); + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "RAR5 input fault", + )) + } + } + } + impl Read for FaultReader { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + self.check()?; + self.input.read(output) + } + } + impl Seek for FaultReader { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.check()?; + self.input.seek(from) + } + } + let source = build_archive_with_optional_comment(Some(b"comment")); + let bytes = source.read_range(0..source.source_len().unwrap()).unwrap(); + let operations = Arc::new(AtomicUsize::new(0)); + let parse = |stop_at, cancellation: Option<&crate::ReadCancellation>| { + operations.store(0, Ordering::Relaxed); + Archive::parse_file_backed( + &mut FaultReader { + input: Cursor::new(bytes.clone()), + operations: operations.clone(), + stop_at, + cancellation: cancellation.cloned(), + }, + bytes.len(), + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + cancellation.map_or_else(crate::ArchiveReadOptions::new, |token| { + crate::ArchiveReadOptions::new().with_cancellation(token) + }), + ) + }; + assert_eq!(parse(usize::MAX, None).unwrap().blocks, source.blocks); + let count = operations.load(Ordering::Relaxed); + assert!(count > 4); + let pre_cancelled = crate::ReadCancellation::new(); + pre_cancelled.cancel(); + assert!(matches!( + parse(usize::MAX, Some(&pre_cancelled)) + .unwrap_err() + .root_cause(), + Error::Cancelled + )); + assert_eq!(operations.load(Ordering::Relaxed), 0); + for operation in 0..count { + let error = parse(operation, None).unwrap_err(); + assert!(matches!(error.root_cause(), Error::Io(source) + if source.kind == std::io::ErrorKind::PermissionDenied && source.message == "RAR5 input fault")); + assert_eq!(operations.load(Ordering::Relaxed), operation + 1); + let cancelled = crate::ReadCancellation::new(); + let error = parse(operation, Some(&cancelled)).unwrap_err(); + assert!( + matches!(error.root_cause(), Error::Cancelled), + "operation {operation}: {error:?}" + ); + assert!(cancelled.is_cancelled()); + } + + #[cfg(feature = "encryption")] + { + let keys = Rar50Keys::derive(b"pw", [0; 16], 0).unwrap(); + let mut encoded = vec![3, HEAD_MAIN as u8, 0, 0]; + let mut first_plain = crc32(&encoded).to_le_bytes().to_vec(); + first_plain.append(&mut encoded); + first_plain.resize(16, 0); + Rar50Cipher::new(keys.key, [0; 16]) + .encrypt_in_place(&mut first_plain) + .unwrap(); + let mut encrypted = vec![0; 16]; + encrypted.extend_from_slice(&first_plain); + let parse_encrypted = |stop_at, cancellation: Option<&crate::ReadCancellation>| { + operations.store(0, Ordering::Relaxed); + read_encrypted_block_header_at( + &mut FaultReader { + input: Cursor::new(encrypted.clone()), + operations: operations.clone(), + stop_at, + cancellation: cancellation.cloned(), + }, + 0, + encrypted.len(), + 0, + &keys, + &mut crate::parse_budget::ParseBudget::new( + cancellation.map_or_else(crate::ArchiveReadOptions::new, |token| { + crate::ArchiveReadOptions::new().with_cancellation(token) + }), + ), + ) + }; + assert_eq!( + parse_encrypted(usize::MAX, None).unwrap().block.header_type, + HEAD_MAIN + ); + let count = operations.load(Ordering::Relaxed); + assert!(count > 1); + for operation in 0..count { + let error = parse_encrypted(operation, None) + .err() + .expect("input fault refused"); + assert!(matches!(error.root_cause(), Error::Io(source) + if source.kind == std::io::ErrorKind::PermissionDenied && source.message == "RAR5 input fault")); + assert_eq!(operations.load(Ordering::Relaxed), operation + 1); + let cancelled = crate::ReadCancellation::new(); + let error = parse_encrypted(operation, Some(&cancelled)) + .err() + .expect("cancellation observed"); + assert!( + matches!(error.root_cause(), Error::Cancelled), + "encrypted operation {operation}: {error:?}" + ); + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encryption_key_failures_retain_specific_diagnostics() { + let mut encoded = vec![HEAD_CRYPT as u8, 0, 0, 0, 25]; + encoded.extend_from_slice(&[0; 16]); + let mut image = vec![encoded.len() as u8]; + image.extend_from_slice(&encoded); + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&crc32(&image).to_le_bytes()); + bytes.extend_from_slice(&image); + let options = crate::ArchiveReadOptions::with_password(b"pw"); + for result in [ + Archive::parse_with_options(&bytes, options), + Archive::parse_file_backed( + &mut std::io::Cursor::new(&bytes), + bytes.len(), + 0, + ArchiveSource::Memory(Arc::from(bytes.clone())), + options, + ), + ] { + assert!(matches!( + result.unwrap_err().root_cause(), + Error::UnsupportedFeature { + feature: "RAR 5 KDF count", + .. + } + )); + } + + let archive = build_archive_with_optional_comment(None); + let mut file = archive.files().next().unwrap().clone(); + file.encrypted = true; + file.encryption = None; + assert!(matches!( + attach_file_crypto(&mut file, Some(b"pw")), + Err(Error::InvalidHeader( + "RAR 5 encrypted file is missing encryption record" + )) + )); + file.encryption = Some(FileEncryption { + version: 0, + flags: 1, + kdf_count: 25, + salt: [0; 16], + iv: [0; 16], + check_value: Some([0; 12]), + }); + assert!(matches!( + attach_file_crypto(&mut file, Some(b"pw")), + Err(Error::UnsupportedFeature { + feature: "RAR 5 KDF count", + .. + }) + )); + file.encryption.as_mut().unwrap().kdf_count = 0; + // WinRAR 5.21 and earlier wrote an all-zero password check, which + // deliberately defers verification to the data checksum. + attach_file_crypto(&mut file, Some(b"pw")).unwrap(); + let mut bad_check = file.clone(); + bad_check.encryption.as_mut().unwrap().check_value = Some([1; 12]); + assert!(matches!( + attach_file_crypto(&mut bad_check, Some(b"pw")), + Err(Error::WrongPasswordOrCorruptData) + )); + } + + #[test] + #[cfg(feature = "encryption")] + fn unknown_archive_encryption_flags_disable_rewrite_preservation() { + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50) + .store(true) + .password(Some(b"pw".to_vec())) + .header_encryption(true); + builder + .add_bytes(b"x".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let mut bytes = builder.to_bytes().unwrap(); + let parsed = parse_block_header_bytes( + &bytes, + RAR50_SIGNATURE.len(), + bytes.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ) + .unwrap(); + assert_eq!(parsed.block.header_type, HEAD_CRYPT); + let options = crate::ArchiveReadOptions::with_password(b"pw"); + assert!( + Archive::parse_with_options(&bytes, options) + .unwrap() + .main + .rewrite_metadata_complete + ); + + let flags_at = RAR50_SIGNATURE.len() + parsed.type_specific_range.start + 1; + bytes[flags_at] |= 2; // Retain the check-value bit and add a future flag. + let crc_at = RAR50_SIGNATURE.len(); + let end = crc_at + parsed.header.len(); + let crc = crc32(&bytes[crc_at + 4..end]); + bytes[crc_at..crc_at + 4].copy_from_slice(&crc.to_le_bytes()); + let archive = Archive::parse_with_options(&bytes, options).unwrap(); + assert!(!archive.main.rewrite_metadata_complete); + assert_eq!(archive.files().next().unwrap().name, b"x"); + } + + #[test] + fn unknown_header_remains_readable_and_budget_error_keeps_one_offset() { + let header = |body: &[u8]| { + let mut encoded = vec![body.len() as u8]; + encoded.extend_from_slice(body); + let mut header = crc32(&encoded).to_le_bytes().to_vec(); + header.extend_from_slice(&encoded); + header + }; + let mut bytes = RAR50_SIGNATURE.to_vec(); + bytes.extend_from_slice(&header(&[HEAD_MAIN as u8, 0, 0])); + bytes.extend_from_slice(&header(&[127, 0])); + bytes.extend_from_slice(&header(&[HEAD_END as u8, 0, 0])); + + let archive = Archive::parse(&bytes).unwrap(); + assert!(matches!( + archive.blocks.as_slice(), + [Block::Unknown(_), Block::End(_)] + )); + + let error = Archive::parse_with_options( + &bytes, + crate::ArchiveReadOptions::new().with_max_header_bytes(0), + ) + .unwrap_err(); + let Error::AtArchiveOffset { offset, source } = error else { + panic!("header refusal needs its physical offset"); + }; + assert_eq!(offset, RAR50_SIGNATURE.len()); + assert!(matches!(*source, Error::HeaderBytesLimitExceeded { .. })); + } + + #[test] + fn header_reader_u32_obeys_type_specific_boundary() { + // Extra-area bytes remain physically available in the header image, + // but must not satisfy a field in the type-specific part. + let input = [99, 99, 1, 2, 3, 4, 77]; + for available in 0..=4 { + let mut reader = HeaderReader::new(&input, 2..2 + available); + let result = reader.read_u32(); + if available < 4 { + assert!( + matches!(result, Err(Error::TooShort)), + "available {available}" + ); + assert_eq!(reader.pos, 2); + } else { + assert_eq!(result.unwrap(), 0x0403_0201); + assert_eq!(reader.pos, 6); + } + } + } + + #[test] + fn main_extra_records_keep_future_and_duplicate_layouts_readable() { + let control = crate::read_control::ReadControl::default(); + // Locator zero offsets are retained as wire values; they do not + // indicate an actual service block at the main header's location. + for (data, quick_open, recovery, complete) in [ + (vec![0], None, None, true), + (vec![1, 0], Some(0), None, true), + (vec![1, 127], Some(127), None, true), + (vec![2, 0x80, 1], None, Some(128), true), + (vec![3, 12, 34], Some(12), Some(34), true), + (vec![4], None, None, false), + (vec![4, 99], None, None, false), + (vec![0, 99], None, None, false), + ] { + let extra = [vec![(1 + data.len()) as u8, MHEXTRA_LOCATOR as u8], data].concat(); + let (records, is_complete) = + parse_main_extra_area(&extra, 0..extra.len(), &control).unwrap(); + assert_eq!(is_complete, complete); + assert!( + matches!(records.as_slice(), [MainExtraRecord::Locator(record)] + if record.quick_open_offset == quick_open && record.recovery_record_offset == recovery) + ); + } + for extra in [ + vec![2, MHEXTRA_LOCATOR as u8, 0, 2, MHEXTRA_LOCATOR as u8, 0], + vec![ + 2, + MHEXTRA_ARCHIVE_METADATA as u8, + 0, + 2, + MHEXTRA_ARCHIVE_METADATA as u8, + 0, + ], + ] { + let (records, complete) = + parse_main_extra_area(&extra, 0..extra.len(), &control).unwrap(); + assert_eq!(records.len(), 2); + assert!( + !complete, + "duplicate records must not be silently rewritten" + ); + } + for (extra, count) in [ + (vec![1, 64], 0), + (vec![2, MHEXTRA_ARCHIVE_METADATA as u8, 16], 1), + (vec![1, 64, 2, MHEXTRA_ARCHIVE_METADATA as u8, 16], 1), + ] { + let (records, complete) = + parse_main_extra_area(&extra, 0..extra.len(), &control).unwrap(); + assert!(!complete); + assert_eq!(records.len(), count); + if count != 0 { + assert!( + matches!(records.as_slice(), [MainExtraRecord::ArchiveMetadata(record)] + if record.flags == 16 && record.name.is_none() && record.creation_time.is_none()) + ); + } + } + let extra = [2, MHEXTRA_LOCATOR as u8, 0, 0x80]; + let (records, complete) = parse_main_extra_area(&extra, 0..extra.len(), &control).unwrap(); + assert_eq!(records.len(), 1); + assert!( + !complete, + "a malformed tail must not erase preceding records" + ); + } + + #[test] + fn main_extra_records_reject_truncated_known_fields_and_metadata_trailing_bytes() { + let control = crate::read_control::ReadControl::default(); + let truncated = [ + vec![1, MHEXTRA_LOCATOR as u8], // Missing locator flags. + vec![2, MHEXTRA_LOCATOR as u8, 1], // Missing quick-open offset. + vec![2, MHEXTRA_LOCATOR as u8, 2], // Missing recovery offset. + vec![1, MHEXTRA_ARCHIVE_METADATA as u8], // Missing metadata flags. + vec![2, MHEXTRA_ARCHIVE_METADATA as u8, 1], // Missing name length. + vec![4, MHEXTRA_ARCHIVE_METADATA as u8, 1, 2, b'a'], // Short name. + vec![5, MHEXTRA_ARCHIVE_METADATA as u8, 6, 1, 2, 3], // Short Unix seconds. + vec![9, MHEXTRA_ARCHIVE_METADATA as u8, 2, 1, 2, 3, 4, 5, 6, 7], // Short FILETIME. + vec![9, MHEXTRA_ARCHIVE_METADATA as u8, 14, 1, 2, 3, 4, 5, 6, 7], // Short Unix nanoseconds. + ]; + for extra in truncated { + // Bytes following the extra area must never satisfy a field read. + let input = [extra.clone(), vec![0; 16]].concat(); + assert!( + matches!( + parse_main_extra_area(&input, 0..extra.len(), &control), + Err(Error::TooShort) + ), + "extra {extra:?}" + ); + } + let trailing = [3, MHEXTRA_ARCHIVE_METADATA as u8, 0, 99]; + assert!(matches!( + parse_main_extra_area(&trailing, 0..trailing.len(), &control), + Err(Error::InvalidHeader( + "RAR 5 archive metadata record has trailing bytes" + )) + )); + let huge_name = [ + 12, + MHEXTRA_ARCHIVE_METADATA as u8, + 1, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 0xff, + 1, + ]; + let expected = if usize::BITS < 64 { + "RAR 5 archive metadata name length overflows usize" + } else { + "RAR 5 field size overflows usize" + }; + assert!( + matches!(parse_main_extra_area(&huge_name, 0..huge_name.len(), &control), + Err(Error::InvalidHeader(message)) if message == expected) + ); + } + + #[test] + fn main_extra_records_preserve_reserved_name_buffers_and_following_times() { + let control = crate::read_control::ReadControl::default(); + let time = 0x01d9_0000_0000_0000u64; + // RARLAB documents padding after renamed archives and a leading zero + // when the final name did not fit the buffer reserved during archiving. + for name in [ + b"original.rar".as_slice(), + b"short.rar\0\0\0", + b"\0old-name.rar", + b"", + ] { + let mut extra = vec![ + (3 + name.len() + 8) as u8, + MHEXTRA_ARCHIVE_METADATA as u8, + 3, + name.len() as u8, + ]; + extra.extend_from_slice(name); + extra.extend_from_slice(&time.to_le_bytes()); + let (records, complete) = + parse_main_extra_area(&extra, 0..extra.len(), &control).unwrap(); + assert!(complete); + assert!( + matches!(records.as_slice(), [MainExtraRecord::ArchiveMetadata(record)] + if record.name.as_deref() == Some(name) && record.creation_time == Some(time)) + ); + } + } + + #[test] + fn archive_metadata_time_flags_follow_rar5_wire_widths() { + let control = crate::read_control::ReadControl::default(); + // RARLAB technote: 0x04 selects Unix time, and 0x08 widens Unix + // seconds to nanoseconds. Without 0x04, time is a FILETIME u64. + for (flags, time, expected, complete) in [ + ( + 0x02, + 0x01d9_0000_0000_0000u64.to_le_bytes().to_vec(), + 0x01d9_0000_0000_0000, + true, + ), + ( + 0x06, + 1_700_000_000u32.to_le_bytes().to_vec(), + 1_700_000_000, + true, + ), + ( + 0x0e, + 1_700_000_000_123_456_789u64.to_le_bytes().to_vec(), + 1_700_000_000_123_456_789, + true, + ), + ( + 0x0a, + 1_700_000_000_123_456_789u64.to_le_bytes().to_vec(), + 1_700_000_000_123_456_789, + false, + ), + ] { + let mut extra = vec![ + (2 + time.len()) as u8, + MHEXTRA_ARCHIVE_METADATA as u8, + flags, + ]; + extra.extend_from_slice(&time); + let (records, is_complete) = + parse_main_extra_area(&extra, 0..extra.len(), &control).unwrap(); + assert_eq!(is_complete, complete, "flags {flags:#x}"); + assert!( + matches!(records.as_slice(), [MainExtraRecord::ArchiveMetadata(record)] + if record.flags == u64::from(flags) && record.creation_time == Some(expected)) + ); + } + } + + #[test] + fn malformed_extra_tail_keeps_preceding_record_and_marks_area_incomplete() { + let valid = [1, 2]; // Record type 2 with no data. + let overflowing_size = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]; + let tails: [&[u8]; 5] = [&[0x80], &overflowing_size, &[1, 0x80], &[0], &[9, 2]]; + let control = crate::read_control::ReadControl::default(); + + for tail in tails { + let input = [valid.as_slice(), tail].concat(); + let mut records = Vec::new(); + let complete = super::parse_extra_records( + &input, + 0..input.len(), + false, + &control, + |record_type, data| { + records.push((record_type, input[data].to_vec())); + Ok(()) + }, + ) + .unwrap(); + assert!(!complete, "tail {tail:?} must be incomplete"); + assert_eq!(records, [(2, Vec::new())]); + } + } + + #[test] + fn cancellation_interrupts_metadata_record_iteration() { + let input = [2, 127, 0].repeat(10000); + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(1); + let mut handled = 0; + let err = super::parse_extra_records(&input, 0..input.len(), false, &control, |_, _| { + handled += 1; + Ok(()) + }) + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::Cancelled); + assert!(handled > 0 && handled < 10000); + } + #[test] + fn file_extra_cancellation_propagates_from_the_metadata_phase() { + let archive = build_archive_with_optional_comment(None); + let file = archive.files().next().unwrap(); + let bytes = archive + .read_range(file.block.offset..file.block.data_range.end) + .unwrap(); + let mut parsed = parse_block_header_bytes( + &bytes, + 0, + bytes.len(), + 0, + &mut crate::parse_budget::ParseBudget::new(crate::ArchiveReadOptions::new()), + ) + .unwrap(); + assert!(!parsed.extra_range.is_empty()); + let token = crate::ReadCancellation::new(); + parsed.control = crate::read_control::ReadControl::new(Some(&token)); + token.cancel(); + assert_eq!( + parse_file_header_bytes(&parsed).unwrap_err().kind(), + crate::ErrorKind::Cancelled + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_header_extent_rejects_short_prefix_and_declared_ciphertext() { + use crate::parse_budget::ParseBudget; + let keys = crate::crypto::rar50::Rar50Keys::derive(b"secret", [0; 16], 0).unwrap(); + let mut plain = vec![0; 16]; + plain[4] = 60; + crate::crypto::rar50::Rar50Cipher::new(keys.key, [0; 16]) + .encrypt_in_place(&mut plain) + .unwrap(); + let mut bytes = vec![0; 16]; + bytes.extend_from_slice(&plain); + for len in [31, 32] { + let input = &bytes[..len]; + let memory = super::parse_encrypted_block_header_bytes( + input, + 0, + len, + 0, + &keys, + &mut ParseBudget::new(crate::ArchiveReadOptions::new()), + ) + .map(|_| ()) + .unwrap_err(); + let seekable = super::read_encrypted_block_header_at( + &mut std::io::Cursor::new(input), + 0, + len, + 0, + &keys, + &mut ParseBudget::new(crate::ArchiveReadOptions::new()), + ) + .map(|_| ()) + .unwrap_err(); + assert!(matches!(memory, crate::Error::TooShort)); + assert!(matches!(seekable, crate::Error::TooShort)); + } + } + + #[test] + fn header_budget_refuses_full_reads_after_plain_and_encrypted_prefixes() { + use crate::parse_budget::{ParseBudget, PrefixReader}; + let mut plain = vec![0u8; 16]; + plain[4] = 60; // 65 plaintext header bytes, larger than the prefix. + let mut reader = PrefixReader::new(plain[..14].to_vec()); + let options = crate::ArchiveReadOptions::new().with_max_header_bytes(64); + let e = super::read_block_header_at(&mut reader, 0, 128, 0, &mut ParseBudget::new(options)) + .map(|_| ()) + .expect_err("header budget must refuse"); + assert!(matches!( + e.root_cause(), + crate::Error::HeaderBytesLimitExceeded { required: 65, .. } + )); + assert_eq!(reader.reads, [14]); + + #[cfg(feature = "encryption")] + { + let keys = crate::crypto::rar50::Rar50Keys::derive(b"secret", [0; 16], 0).unwrap(); + crate::crypto::rar50::Rar50Cipher::new(keys.key, [0; 16]) + .encrypt_in_place(&mut plain) + .unwrap(); + let mut prefix = vec![0; 16]; + prefix.extend_from_slice(&plain); + let mut reader = PrefixReader::new(prefix); + let e = super::read_encrypted_block_header_at( + &mut reader, + 0, + 128, + 0, + &keys, + &mut ParseBudget::new(options), + ) + .map(|_| ()) + .expect_err("header budget must refuse"); + assert!(matches!( + e.root_cause(), + crate::Error::HeaderBytesLimitExceeded { required: 65, .. } + )); + assert_eq!(reader.reads, [32]); + } + } + use super::*; + + #[test] + fn read_vint_at_honors_logical_end_before_decoding() { + assert_eq!(read_vint_at(&[0x01], 0, 0), Err(Error::TooShort)); + assert_eq!(read_vint_at(&[0x81, 0x01], 0, 1), Err(Error::TooShort)); + assert_eq!(read_vint_at(&[0x81, 0x01], 0, 2).unwrap(), (129, 2)); + assert_eq!( + read_vint_at(&[], usize::MAX, usize::MAX), + Err(Error::TooShort) + ); + } + + #[test] + fn read_vint_at_rejects_overflow_and_overlong_continuations() { + let max = [0xff; 9].into_iter().chain([0x01]).collect::>(); + assert_eq!(read_vint_at(&max, 0, max.len()).unwrap(), (u64::MAX, 10)); + + let overflow = [0xff; 9].into_iter().chain([0x02]).collect::>(); + assert_eq!( + read_vint_at(&overflow, 0, overflow.len()), + Err(Error::InvalidHeader("RAR 5 vint overflows u64")) + ); + + let overlong = [0x80; 10]; + assert_eq!( + read_vint_at(&overlong, 0, overlong.len()), + Err(Error::InvalidHeader("RAR 5 vint is too long")) + ); + } + + #[test] + fn parses_file_redirection_extra_record() { + let input = [1, 1, 6, b't', b'a', b'r', b'g', b'e', b't']; + let record = parse_file_redirection_record(&input, 0..input.len()).unwrap(); + + assert_eq!(record.redirection_type, 1); + assert_eq!(record.flags, 1); + assert_eq!(record.target_name, b"target"); + } + + #[test] + fn redirection_target_length_cannot_overflow_its_record() { + let input = [ + 1, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 1, + ]; + let expected = if usize::BITS == 64 { + "RAR 5 file redirection target length overflows" + } else { + "RAR 5 file redirection target length overflows host address size" + }; + assert!( + matches!(parse_file_redirection_record(&input, 0..input.len()), + Err(Error::InvalidHeader(message)) if message == expected) + ); + } + + #[test] + fn oversized_extra_record_length_keeps_preceding_metadata() { + let mut extra = vec![2, MHEXTRA_LOCATOR as u8, 0]; + // A valid vint can declare more bytes than a 32-bit process can + // address; on 64-bit the physical record still cannot contain them. + extra.extend_from_slice(&[0xff; 9]); + extra.push(1); + let (records, complete) = parse_main_extra_area( + &extra, + 0..extra.len(), + &crate::read_control::ReadControl::default(), + ) + .unwrap(); + assert!(!complete); + assert!(matches!(records.as_slice(), [MainExtraRecord::Locator(_)])); + } + + #[test] + fn rejects_file_redirection_record_with_trailing_bytes() { + let input = [1, 0, 3, b'f', b'o', b'o', 0]; + + assert!(matches!( + parse_file_redirection_record(&input, 0..input.len()), + Err(Error::InvalidHeader( + "RAR 5 file redirection record has trailing bytes" + )) + )); + } + + #[test] + fn htime_precision_respects_record_bounds_and_time_layout() { + let mut unix = vec![0x1f]; // mtime, ctime, atime, then their fractions + for value in [123u32, 456, 789, 987_654_321, 111, 222] { + unix.extend_from_slice(&value.to_le_bytes()); + } + let (seconds, detail) = parse_htime_mtime(&unix, 0..unix.len()).unwrap(); + assert_eq!(seconds, 123); + assert_eq!(detail.unwrap().nanoseconds, 987_654_321); + assert_eq!(parse_htime_mtime(&unix, 0..2), None); + assert_eq!(parse_htime_mtime(&unix, 0..14), Some((123, None))); + unix[13..17].copy_from_slice(&1_000_000_000u32.to_le_bytes()); + assert_eq!(parse_htime_mtime(&unix, 0..unix.len()), Some((123, None))); + + let mut filetime = vec![2]; + let ticks = (11_644_473_600u64 + 123) * 10_000_000 + 7_040_883; + filetime.extend_from_slice(&ticks.to_le_bytes()); + let (seconds, detail) = parse_htime_mtime(&filetime, 0..filetime.len()).unwrap(); + assert_eq!(seconds, 123); + assert_eq!(detail.unwrap().nanoseconds, 704_088_300); + assert_eq!(parse_htime_mtime(&filetime, 0..8), None); + } + + #[test] + fn file_header_name_bytes_preserve_non_utf8_names() { + let file = FileHeader { + block: BlockHeader { + header_crc: 0, + header_size: 0, + header_type: HEAD_FILE, + flags: 0, + extra_area_size: None, + data_size: Some(0), + offset: 0, + header_range: 0..0, + data_range: 0..0, + }, + file_flags: 0, + rewrite_metadata_complete: true, + unpacked_size: 0, + attributes: 0, + mtime: None, + htime_mtime: None, + htime_mtime_refinement: None, + file_times: None, + data_crc32: None, + compression_info: 0, + host_os: 0, + name: vec![0xff, b'.', b'b', b'i', b'n'], + hash: None, + redirection: None, + service_data: None, + encrypted: false, + encryption: None, + crypto: None, + }; + + assert_eq!(file.name_bytes(), [0xff, b'.', b'b', b'i', b'n']); + assert_eq!(file.name_lossy(), "\u{fffd}.bin"); + } + + /// Builds a member from bytes the test already holds. + fn rar50_entry(name: &[u8], data: &[u8]) -> crate::rar50::ArchiveEntry { + crate::rar50::ArchiveEntry::new( + name.to_vec(), + crate::EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data.to_vec())), + ) + } + + fn build_archive_with_optional_comment(comment: Option<&[u8]>) -> Archive { + use crate::FeatureSet; + let features = FeatureSet::store_only(); + let entries = [rar50_entry(b"payload.txt", b"payload bytes") + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = crate::rar50::Rar50Writer::new( + crate::rar50::WriterOptions::new(crate::version::ArchiveVersion::Rar50, features) + .with_compression_level(0), + ) + .entries(entries.to_vec()) + .archive_comment(comment) + .finish() + .unwrap(); + Archive::parse(&bytes).unwrap() + } + + #[test] + fn parser_metadata_failures_retain_physical_header_offsets() { + fn image(body: &[u8]) -> Vec { + assert!(body.len() < 128); + let mut bytes = vec![body.len() as u8]; + bytes.extend_from_slice(body); + let mut result = crc32(&bytes).to_le_bytes().to_vec(); + result.extend_from_slice(&bytes); + result + } + let main = image(&[HEAD_MAIN as u8, 0, 0]); + let mut cases = vec![( + image(&[HEAD_MAIN as u8, 0]), + true, + crate::ErrorKind::InvalidArchive, + )]; + for kind in [HEAD_FILE, HEAD_SERVICE] { + cases.push(( + image(&[kind as u8, 0]), + false, + crate::ErrorKind::InvalidArchive, + )); + // Structurally complete encryption record, unsupported version. + let mut record = vec![FHEXTRA_CRYPT as u8, 1, 0, 0]; + record.extend_from_slice(&[0; 32]); + let mut body = vec![ + kind as u8, + HFL_EXTRA as u8, + (record.len() + 1) as u8, + 0, + 0, + 0, + 0, + 0, + 1, + b'x', + record.len() as u8, + ]; + body.extend_from_slice(&record); + cases.push((image(&body), false, crate::ErrorKind::UnsupportedFeature)); + } + for (header, is_main, expected_kind) in cases { + let mut bytes = RAR50_SIGNATURE.to_vec(); + if !is_main { + bytes.extend_from_slice(&main); + } + let offset = bytes.len(); + bytes.extend_from_slice(&header); + let options = crate::ArchiveReadOptions::with_password(b"secret"); + let memory = Archive::parse_with_options(&bytes, options); + let seekable = Archive::parse_file_backed( + &mut std::io::Cursor::new(&bytes), + bytes.len(), + 0, + ArchiveSource::Memory(std::sync::Arc::from(bytes.clone())), + options, + ); + if expected_kind == crate::ErrorKind::UnsupportedFeature + && !cfg!(feature = "encryption") + { + // Without crypto, an unknown encryption version remains readable metadata. + for archive in [memory, seekable] { + let archive = archive.unwrap(); + let file = match archive.blocks.last().unwrap() { + Block::File(file) | Block::Service(file) => file, + _ => panic!("expected encryption metadata"), + }; + assert_eq!(file.encryption.as_ref().unwrap().version, 1); + assert!(file.crypto.is_none()); + assert!(!file.rewrite_metadata_complete); + for password in [None, Some(b"secret".as_slice())] { + let mut output = b"untouched".to_vec(); + assert_eq!( + file.write_to(&archive, password, &mut output) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "encryption" + } + ); + assert_eq!(output, b"untouched"); + } + } + continue; + } + for error in [memory, seekable] { + let error = error.unwrap_err(); + assert_eq!(error.kind(), expected_kind, "{error}"); + assert!( + matches!(error, Error::AtArchiveOffset { offset: actual, .. } if actual == offset), + "{error}" + ); + } + } + } + + #[test] + fn memory_and_file_backed_parsers_reject_a_non_main_first_header() { + let archive = build_archive_with_optional_comment(None); + let mut bytes = archive + .read_range(0..archive.source_len().unwrap()) + .unwrap(); + let start = RAR50_SIGNATURE.len(); + let (body_size, size_len) = read_vint_at(&bytes, start + 4, bytes.len()).unwrap(); + let header_end = start + 4 + size_len + body_size as usize; + let header_type = start + 4 + size_len; + assert_eq!(bytes[header_type], HEAD_MAIN as u8); + bytes[header_type] = HEAD_FILE as u8; + let crc = crc32(&bytes[start + 4..header_end]); + bytes[start..start + 4].copy_from_slice(&crc.to_le_bytes()); + + let memory = Archive::parse(&bytes).unwrap_err(); + let mut cursor = std::io::Cursor::new(&bytes); + let file_backed = Archive::parse_file_backed( + &mut cursor, + bytes.len(), + 0, + ArchiveSource::Memory(std::sync::Arc::from(bytes.clone().into_boxed_slice())), + crate::ArchiveReadOptions::default(), + ) + .unwrap_err(); + + assert!(matches!( + memory, + Error::InvalidHeader("RAR 5 main header is missing") + )); + assert!(matches!( + file_backed, + Error::InvalidHeader("RAR 5 main header is missing") + )); + } + + #[test] + fn memory_and_file_backed_parsers_reject_truncated_first_header() { + let archive = build_archive_with_optional_comment(None); + let original = archive + .read_range(0..archive.source_len().unwrap()) + .unwrap(); + let first_header = RAR50_SIGNATURE.len(); + + for len in [first_header + 4, first_header + 5] { + let bytes = original[..len].to_vec(); + let memory = Archive::parse(&bytes).unwrap_err(); + let mut cursor = std::io::Cursor::new(&bytes); + let file_backed = Archive::parse_file_backed( + &mut cursor, + bytes.len(), + 0, + ArchiveSource::Memory(std::sync::Arc::from(bytes.clone().into_boxed_slice())), + crate::ArchiveReadOptions::default(), + ) + .unwrap_err(); + + assert!( + matches!(memory.root_cause(), Error::TooShort), + "length {len}" + ); + assert!( + matches!(file_backed.root_cause(), Error::TooShort), + "length {len}" + ); + } + } + + #[test] + fn archive_comment_returns_none_for_archive_without_a_cmt_service() { + let mut archive = build_archive_with_optional_comment(None); + assert!(archive.archive_comment().unwrap().is_none()); + for block in &mut archive.blocks { + if let Block::File(file) = block { + let mut service = file.clone(); + service.name = b"STM".to_vec(); + *block = Block::Service(service); + } + } + assert!(archive.archive_comment().unwrap().is_none()); + } + + #[test] + fn archive_comment_decodes_the_cmt_service_payload_text() { + let comment_text = b"archive comment from rars unit test\n"; + let archive = build_archive_with_optional_comment(Some(comment_text)); + let comment = archive.archive_comment().unwrap(); + assert_eq!(comment.as_deref(), Some(&comment_text[..])); + } + + #[test] + fn archive_comment_ignores_cmt_services_attached_to_files() { + // Service blocks that follow a File block belong to that file, not the + // archive — archive_comment should not surface them. + use crate::FeatureSet; + let entry = rar50_entry(b"payload.txt", b"payload bytes") + .with_attributes(0x20) + .with_host_os(3) + .with_service(crate::rar50::ServiceEntry::new( + b"CMT".to_vec(), + b"per-file comment".to_vec(), + )); + let features = FeatureSet::store_only(); + let bytes = crate::rar50::Rar50Writer::new(crate::rar50::WriterOptions::new( + crate::version::ArchiveVersion::Rar50, + features, + )) + .entry(entry) + .finish() + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert!(archive.archive_comment().unwrap().is_none()); + } +} + +#[cfg(all(test, feature = "write"))] +#[test] +fn redirection_preservation_refuses_inconsistent_header_metadata() { + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"link".to_vec(), Vec::new(), None, None) + .unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let original = archive.members().next().unwrap(); + for (kind, flags, host, attr, directory, supported) in [ + (2, 0, 0, 0x400, false, true), + (2, 0, 1, 0x400, false, false), + (2, 0, 0, 0, false, false), + (2, 0, 0, 0x410, false, false), + (2, 1, 0, 0x400, false, false), + (3, 0, 0, 0x400, false, false), + (3, 1, 0, 0x410, true, true), + (4, 0, 0, 0, false, true), + (4, 0, 0, 0, true, false), + (5, 0, 1, 0o100600, false, true), + (0, 0, 0, 0, false, false), + (6, 0, 0, 0, false, false), + ] { + let mut member = original.clone(); + member.meta.host_os = Some(host); + member.meta.file_attr = attr; + member.meta.is_directory = directory; + member.meta.is_redirection = true; + let crate::ArchiveMemberDetail::Rar50Plus { redirection, .. } = &mut member.detail else { + panic!("expected RAR5 member"); + }; + *redirection = Some(FileRedirection { + redirection_type: kind, + flags, + target_name: b"target".to_vec(), + }); + assert_eq!( + member.supported_redirection().is_some(), + supported, + "kind={kind}, flags={flags}, host={host}, attr={attr:#x}, directory={directory}" + ); + if !supported { + let mut output = crate::Builder::new(crate::ArchiveVersion::Rar50); + assert_eq!( + output.add_archive_redirection(&member), + Err(Error::InvalidArgument("unsupported redirection metadata")) + ); + assert!(output.is_empty()); + } + } +} diff --git a/arca-rar/vendor/rars/src/rar50/blake2sp.rs b/arca-rar/vendor/rars/src/rar50/blake2sp.rs new file mode 100644 index 00000000..5cfa9894 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/blake2sp.rs @@ -0,0 +1,517 @@ +const BLOCK_BYTES: usize = 64; +const OUT_BYTES: usize = 32; +const PARALLELISM: usize = 8; +const BUFFER_BYTES: usize = BLOCK_BYTES * PARALLELISM; + +const IV: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]; + +const SIGMA: [[usize; 16]; 10] = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], + [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], + [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8], + [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], + [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9], + [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], + [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10], + [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], + [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0], +]; + +struct Blake2s { + h: [u32; 8], + t: u64, + buf: [u8; BLOCK_BYTES], + buflen: usize, + last_node: bool, +} + +impl Blake2s { + fn new(node_offset: u64, node_depth: u8, last_node: bool) -> Self { + let mut param = [0u8; 32]; + param[0] = OUT_BYTES as u8; + param[2] = PARALLELISM as u8; + param[3] = 2; + param[8..14].copy_from_slice(&node_offset.to_le_bytes()[..6]); + param[14] = node_depth; + param[15] = OUT_BYTES as u8; + + let mut h = IV; + for (word, chunk) in h.iter_mut().zip(param.as_chunks::<4>().0.iter()) { + *word ^= u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + } + + Self { + h, + t: 0, + buf: [0; BLOCK_BYTES], + buflen: 0, + last_node, + } + } + + fn update(&mut self, mut input: &[u8]) { + // Stripe updates are one 64-byte block, final leaf tails are 1..=64 + // bytes, and the root consumes one 32-byte leaf digest at a time. + debug_assert!(!input.is_empty() && input.len() <= BLOCK_BYTES); + if self.buflen > 0 { + let fill = BLOCK_BYTES - self.buflen; + if input.len() > fill { + self.buf[self.buflen..].copy_from_slice(&input[..fill]); + self.t = self.t.wrapping_add(BLOCK_BYTES as u64); + let block = self.buf; + self.compress(&block, false); + self.buflen = 0; + input = &input[fill..]; + } else { + self.buf[self.buflen..self.buflen + input.len()].copy_from_slice(input); + self.buflen += input.len(); + return; + } + } + self.buf[..input.len()].copy_from_slice(input); + self.buflen = input.len(); + } + + fn finalize(mut self) -> [u8; OUT_BYTES] { + self.t = self.t.wrapping_add(self.buflen as u64); + self.buf[self.buflen..].fill(0); + let block = self.buf; + self.compress(&block, true); + + let mut out = [0u8; OUT_BYTES]; + for (chunk, word) in out.as_chunks_mut::<4>().0.iter_mut().zip(self.h) { + chunk.copy_from_slice(&word.to_le_bytes()); + } + out + } + + fn compress(&mut self, block: &[u8; BLOCK_BYTES], last_block: bool) { + let mut m = [0u32; 16]; + for (word, chunk) in m.iter_mut().zip(block.as_chunks::<4>().0.iter()) { + *word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + } + + let mut v = [0u32; 16]; + v[..8].copy_from_slice(&self.h); + v[8..].copy_from_slice(&IV); + v[12] ^= self.t as u32; + v[13] ^= (self.t >> 32) as u32; + if last_block { + v[14] = !v[14]; + } + if last_block && self.last_node { + v[15] = !v[15]; + } + + for s in SIGMA { + g(&mut v, 0, 4, 8, 12, m[s[0]], m[s[1]]); + g(&mut v, 1, 5, 9, 13, m[s[2]], m[s[3]]); + g(&mut v, 2, 6, 10, 14, m[s[4]], m[s[5]]); + g(&mut v, 3, 7, 11, 15, m[s[6]], m[s[7]]); + g(&mut v, 0, 5, 10, 15, m[s[8]], m[s[9]]); + g(&mut v, 1, 6, 11, 12, m[s[10]], m[s[11]]); + g(&mut v, 2, 7, 8, 13, m[s[12]], m[s[13]]); + g(&mut v, 3, 4, 9, 14, m[s[14]], m[s[15]]); + } + + for i in 0..8 { + self.h[i] ^= v[i] ^ v[i + 8]; + } + } +} + +pub(crate) struct Hasher { + leaves: [Blake2s; PARALLELISM], + buffer: [u8; BUFFER_BYTES], + buffer_len: usize, +} + +impl Hasher { + pub(crate) fn new() -> Self { + Self { + leaves: [ + Blake2s::new(0, 0, false), + Blake2s::new(1, 0, false), + Blake2s::new(2, 0, false), + Blake2s::new(3, 0, false), + Blake2s::new(4, 0, false), + Blake2s::new(5, 0, false), + Blake2s::new(6, 0, false), + Blake2s::new(7, 0, true), + ], + buffer: [0; BUFFER_BYTES], + buffer_len: 0, + } + } + + pub(crate) fn update(&mut self, mut input: &[u8]) { + if input.is_empty() { + return; + } + + if self.buffer_len > 0 { + let fill = BUFFER_BYTES - self.buffer_len; + if input.len() >= fill { + self.buffer[self.buffer_len..].copy_from_slice(&input[..fill]); + let group = self.buffer; + self.update_group(&group); + self.buffer_len = 0; + input = &input[fill..]; + } else { + self.buffer[self.buffer_len..self.buffer_len + input.len()].copy_from_slice(input); + self.buffer_len += input.len(); + return; + } + } + + let (groups, tail) = input.as_chunks::(); + for group in groups { + self.update_group(group); + } + self.buffer[..tail.len()].copy_from_slice(tail); + self.buffer_len = tail.len(); + } + + pub(crate) fn finalize(mut self) -> [u8; OUT_BYTES] { + if self.buffer_len > 0 { + let tail = &self.buffer[..self.buffer_len]; + for (leaf_index, leaf) in self.leaves.iter_mut().enumerate() { + let start = leaf_index * BLOCK_BYTES; + if tail.len() > start { + let end = (start + BLOCK_BYTES).min(tail.len()); + leaf.update(&tail[start..end]); + } + } + } + + let mut root = Blake2s::new(0, 1, true); + for leaf in self.leaves { + root.update(&leaf.finalize()); + } + root.finalize() + } + + fn update_group(&mut self, group: &[u8]) { + // Keep each leaf's final block pending, just as Blake2s::update does. + // A new full stripe proves the previous stripe is not final, so its + // eight independent compression states can run together. + if self.leaves[0].buflen == BLOCK_BYTES { + debug_assert!(self.leaves.iter().all(|leaf| leaf.buflen == BLOCK_BYTES)); + self.compress_leaf_group(); + for (leaf, block) in self + .leaves + .iter_mut() + .zip(group.as_chunks::().0.iter()) + { + leaf.buf.copy_from_slice(block); + } + return; + } + for (leaf, block) in self + .leaves + .iter_mut() + .zip(group.as_chunks::().0.iter()) + { + leaf.update(block); + } + } + + fn compress_leaf_group(&mut self) { + // Word-major lanes expose independent leaf work to the optimizer + // without CPU-specific intrinsics or changing the persistent state. + let mut message = [[0u32; PARALLELISM]; 16]; + let mut state = [[0u32; PARALLELISM]; 16]; + for (lane, leaf) in self.leaves.iter_mut().enumerate() { + leaf.t = leaf.t.wrapping_add(BLOCK_BYTES as u64); + for (word, bytes) in leaf.buf.as_chunks::<4>().0.iter().enumerate() { + message[word][lane] = u32::from_le_bytes(*bytes); + } + for word in 0..8 { + state[word][lane] = leaf.h[word]; + state[word + 8][lane] = IV[word]; + } + state[12][lane] ^= leaf.t as u32; + state[13][lane] ^= (leaf.t >> 32) as u32; + } + for schedule in SIGMA { + g_lanes::<0, 4, 8, 12>(&mut state, &message[schedule[0]], &message[schedule[1]]); + g_lanes::<1, 5, 9, 13>(&mut state, &message[schedule[2]], &message[schedule[3]]); + g_lanes::<2, 6, 10, 14>(&mut state, &message[schedule[4]], &message[schedule[5]]); + g_lanes::<3, 7, 11, 15>(&mut state, &message[schedule[6]], &message[schedule[7]]); + g_lanes::<0, 5, 10, 15>(&mut state, &message[schedule[8]], &message[schedule[9]]); + g_lanes::<1, 6, 11, 12>(&mut state, &message[schedule[10]], &message[schedule[11]]); + g_lanes::<2, 7, 8, 13>(&mut state, &message[schedule[12]], &message[schedule[13]]); + g_lanes::<3, 4, 9, 14>(&mut state, &message[schedule[14]], &message[schedule[15]]); + } + for (lane, leaf) in self.leaves.iter_mut().enumerate() { + for word in 0..8 { + leaf.h[word] ^= state[word][lane] ^ state[word + 8][lane]; + } + } + } +} + +#[inline(always)] +fn g_lanes( + state: &mut [[u32; PARALLELISM]; 16], + x: &[u32; PARALLELISM], + y: &[u32; PARALLELISM], +) { + for lane in 0..PARALLELISM { + state[A][lane] = state[A][lane] + .wrapping_add(state[B][lane]) + .wrapping_add(x[lane]); + state[D][lane] = (state[D][lane] ^ state[A][lane]).rotate_right(16); + state[C][lane] = state[C][lane].wrapping_add(state[D][lane]); + state[B][lane] = (state[B][lane] ^ state[C][lane]).rotate_right(12); + state[A][lane] = state[A][lane] + .wrapping_add(state[B][lane]) + .wrapping_add(y[lane]); + state[D][lane] = (state[D][lane] ^ state[A][lane]).rotate_right(8); + state[C][lane] = state[C][lane].wrapping_add(state[D][lane]); + state[B][lane] = (state[B][lane] ^ state[C][lane]).rotate_right(7); + } +} + +pub(crate) fn hash(input: &[u8]) -> [u8; OUT_BYTES] { + let mut hasher = Hasher::new(); + hasher.update(input); + hasher.finalize() +} + +fn g(v: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, x: u32, y: u32) { + v[a] = v[a].wrapping_add(v[b]).wrapping_add(x); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(12); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(y); + v[d] = (v[d] ^ v[a]).rotate_right(8); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(7); +} + +#[cfg(test)] +mod tests { + use super::{hash, Blake2s, Hasher, BLOCK_BYTES, PARALLELISM}; + + #[test] + fn matches_public_blake2sp_vectors() { + assert_eq!( + hex(&hash(b"")), + "dd0e891776933f43c7d032b08a917e25741f8aa9a12c12e1cac8801500f2ca4f" + ); + assert_eq!( + hex(&hash(b"abc")), + "70f75b58f1fecab821db43c88ad84edde5a52600616cd22517b7bb14d440a7d5" + ); + } + + #[test] + fn block_and_stripe_boundaries_match_independent_tree_hash_vectors() { + // Python hashlib.blake2s oracle: distribute consecutive 64-byte + // blocks over eight leaves, fanout=8, depth=2, inner_size=32, + // node_offset=leaf index, node_depth=0, last_node on leaf 7. Hash + // their concatenated digests at node_depth=1, last_node=true. + // The same oracle reproduces both public vectors above. + for (length, expected) in [ + ( + 1, + "a6b9eecc25227ad788c99d3f236debc8da408849e9a5178978727a81457f7239", + ), + ( + 63, + "1024c940be7341449b5010522b509f65bbdc1287b455c2bb7f72b2c92fd0d189", + ), + ( + 64, + "52603b6cbfad4966cb044cb267568385cf35f21e6c45cf30aed19832cb51e9f5", + ), + ( + 65, + "fff24d3cc729d395daf978b0157306cb495797e6c8dca1731d2f6f81b849baae", + ), + ( + 127, + "a626543c271fccc3e4450b48d66bc9cbdeb25e5d077a6213cd90cbbd0fd22076", + ), + ( + 511, + "8e1e8ee1ffa0a01028fff3bff0ae9df2565a82e55a04e9541bb78b9c4778336f", + ), + ( + 512, + "8d9e357863298dd8364b7caf4234317f8a49f180d788b7abffb521925f1e1ff1", + ), + ( + 513, + "8a4bc3330497e681f15daf24fc496044a1c32bf0a837a210399e1ae4af7e92be", + ), + ( + 1023, + "0db3cb64828effe5b2aa6be5e865121f03226cca6423b8841b1019cfad09ac09", + ), + ( + 1024, + "48467549502e2d3f422870bfb1d09bce71a065735763bf654582cf46a5112793", + ), + ( + 1025, + "04e03e65b8f19a5f46288802b2a515bab73363262caa300ae75c0eb29c016e5a", + ), + ( + 4097, + "055d3f5f5440fe6c8dd0dcceb503b93caa03a533b71b1e47b4fdcb5577b8762a", + ), + ] { + let input: Vec<_> = (0..length).map(|i| (i % 251) as u8).collect(); + assert_eq!(hex(&hash(&input)), expected, "length={length}"); + for chunk_size in [1, 31, 32, 63, 64, 65, 127, 511, 512, 513] { + let mut hasher = Hasher::new(); + hasher.update(&[]); + for chunk in input.chunks(chunk_size) { + hasher.update(chunk); + hasher.update(&[]); + } + assert_eq!( + hex(&hasher.finalize()), + expected, + "length={length}, chunk_size={chunk_size}" + ); + } + } + } + + #[test] + fn streaming_hasher_matches_one_shot_hash() { + let input: Vec = (0..4097).map(|i| (i % 251) as u8).collect(); + let mut hasher = Hasher::new(); + for chunk in input.chunks(37) { + hasher.update(chunk); + } + assert_eq!(hasher.finalize(), hash(&input)); + } + + #[test] + fn batched_stripes_match_independent_long_tree_hash_vectors() { + // Same independent hashlib tree parameters as the boundary vectors, + // with enough complete stripes to repeatedly enter the batched path. + for (length, expected) in [ + ( + 65_535, + "ab141a58697b388f28149af484e466beea9f28dbd5aed8e649f02fa8d0afcb0d", + ), + ( + 65_536, + "f31ebaad05d2ead9e476110facfaaf38141a58d53da972a09be1d9e22e0a94ad", + ), + ( + 65_537, + "60e42e849f6f22218ae9faeb17cf9eaf00fd9bc140a4b392109d6378557a4192", + ), + ( + 1_048_576, + "65f05ea2b52b252474eadbcc69d159009c3b7b98e2c630a3056561269d78656e", + ), + ] { + let input: Vec<_> = (0..length).map(|i| (i % 251) as u8).collect(); + assert_eq!(hex(&hash(&input)), expected, "length={length}"); + for chunk_size in [63, 64, 65, 511, 512, 513, 4097, 65_536] { + let mut hasher = Hasher::new(); + for chunk in input.chunks(chunk_size) { + hasher.update(chunk); + } + assert_eq!( + hex(&hasher.finalize()), + expected, + "length={length}, chunk={chunk_size}" + ); + } + } + } + + #[test] + fn batched_updates_match_scalar_leaves_for_varied_data_and_tails() { + let mut random = 0x1234_5678u32; + let input: Vec<_> = (0..65_537) + .map(|_| { + random = random.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (random >> 24) as u8 + }) + .collect(); + for length in [ + 0, 1, 63, 64, 65, 511, 512, 513, 1023, 1024, 1025, 4095, 4096, 4097, 65_535, 65_536, + 65_537, + ] { + let data = &input[..length]; + let mut leaves: [Blake2s; PARALLELISM] = + std::array::from_fn(|lane| Blake2s::new(lane as u64, 0, lane == PARALLELISM - 1)); + for (block_index, block) in data.chunks(BLOCK_BYTES).enumerate() { + leaves[block_index % PARALLELISM].update(block); + } + let mut root = Blake2s::new(0, 1, true); + for leaf in leaves { + root.update(&leaf.finalize()); + } + let expected = root.finalize(); + assert_eq!(hash(data), expected, "length={length}"); + for chunk_size in [1, 67, 511, 512, 513, 4097] { + let mut hasher = Hasher::new(); + for chunk in data.chunks(chunk_size) { + hasher.update(chunk); + } + assert_eq!( + hasher.finalize(), + expected, + "length={length}, chunk={chunk_size}" + ); + } + } + } + + #[test] + fn batched_compression_keeps_each_leaf_counter_and_wrapping_arithmetic() { + for counter in [0, 1, u64::MAX - 64, u64::MAX - 63, u64::MAX - 32] { + let prepare = || { + let mut hasher = Hasher::new(); + for (lane, leaf) in hasher.leaves.iter_mut().enumerate() { + leaf.t = counter.wrapping_add(lane as u64); + leaf.buflen = BLOCK_BYTES; + for (byte_index, byte) in leaf.buf.iter_mut().enumerate() { + *byte = (byte_index * 17 + lane * 41) as u8; + } + for (word, value) in leaf.h.iter_mut().enumerate() { + *value ^= u32::MAX.wrapping_sub((word * 17 + lane * 41) as u32); + } + } + hasher + }; + let mut actual = prepare(); + let mut expected = prepare(); + actual.compress_leaf_group(); + for leaf in &mut expected.leaves { + leaf.t = leaf.t.wrapping_add(BLOCK_BYTES as u64); + let block = leaf.buf; + leaf.compress(&block, false); + } + for (actual, expected) in actual.leaves.iter().zip(expected.leaves) { + assert_eq!(actual.h, expected.h); + assert_eq!(actual.t, expected.t); + assert_eq!(actual.buf, expected.buf); + assert_eq!(actual.buflen, expected.buflen); + } + } + } + + fn hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + out + } +} diff --git a/arca-rar/vendor/rars/src/rar50/extract.rs b/arca-rar/vendor/rars/src/rar50/extract.rs new file mode 100644 index 00000000..5186406f --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/extract.rs @@ -0,0 +1,4148 @@ +#[path = "scratch.rs"] +mod scratch; +use super::{blake2sp, Archive, ExtractedEntryMeta, FileHeader, FileRedirection}; +use crate::codec::rar50::{ + DecodeMode, DecodedChunk, HistoryRetention, ReaderState, StreamDecodeError, +}; +use crate::codec::workspace::{Allowance, Boxed, Budget, Buffer}; +use crate::crc32::{crc32, Crc32}; +#[cfg(feature = "encryption")] +use crate::crypto::rar50::Rar50Cipher; +use crate::crypto::rar50::Rar50Keys; +use crate::error::{Error, Result}; +use crate::volume_extract::{ChainedReader, SplitVolumeState, SplitVolumeStep}; +use std::io::{Read, Write}; + +// Filtered RAR5 members still need whole-member byte transforms. Members at or +// below this boundary use the buffered path, while larger members stream once +// and reject filtered streams through the codec's typed sentinel. +#[cfg(not(test))] +const BUFFERED_DECODE_LIMIT: u64 = 512 * 1024 * 1024; +#[cfg(test)] +const BUFFERED_DECODE_LIMIT: u64 = 1024; + +#[cfg(all(test, feature = "write"))] +#[path = "streaming_checks_tests.rs"] +mod streaming_checks_tests; + +impl FileHeader { + fn check_output_limit(&self, budget: &crate::output_limit::OutputBudget) -> Result<()> { + if !budget.is_limited() || self.is_directory() || self.redirection.is_some() { + return Ok(()); + } + let size = self.known_unpacked_size().ok_or_else(|| { + self.entry_error( + "limiting output", + Error::UnsupportedFeature { + version: crate::ArchiveVersion::Rar50, + feature: "output-limited extraction of an unknown-size member", + }, + ) + })?; + budget.check(size, &self.name) + } + + fn check_dictionary_limit(&self, limit: Option) -> Result<()> { + let Some(limit) = limit else { + return Ok(()); + }; + if self.is_stored() || self.is_directory() || self.redirection.is_some() { + return Ok(()); + } + let required = self + .decoded_compression_info() + .map_err(|error| self.entry_error("checking dictionary limit", error))? + .dictionary_size; + if required > limit { + return Err(self.entry_error( + "checking dictionary limit", + Error::Rar50DictionaryLimitExceeded { limit, required }, + )); + } + Ok(()) + } + + fn encryption_keys(&self, password: Option<&[u8]>) -> Result { + #[cfg(not(feature = "encryption"))] + { + let _ = (self, password); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + crate::crypto::require_encryption()?; + + // Both callers have already selected an encrypted member. + if let Some(crypto) = &self.crypto { + return Ok(crypto.keys.clone()); + } + let password = password.ok_or(Error::NeedPassword)?; + let encryption = self.encryption.as_ref().ok_or(Error::InvalidHeader( + "RAR 5 encrypted file is missing encryption record", + ))?; + if encryption.version != 0 { + return Err(Error::UnsupportedFeature { + version: crate::version::ArchiveVersion::Rar50, + feature: "RAR 5 unknown file encryption version", + }); + } + let keys = Rar50Keys::derive(password, encryption.salt, encryption.kdf_count) + .map_err(super::map_rar50_crypto_error)?; + if let Some(check_value) = encryption.check_value { + keys.check_password(&check_value) + .map_err(super::map_rar50_crypto_error)?; + } + Ok(keys) + } + } + + fn encryption_iv(&self) -> Result<[u8; 16]> { + if let Some(crypto) = &self.crypto { + return Ok(crypto.iv); + } + self.encryption + .as_ref() + .map(|encryption| encryption.iv) + .ok_or(Error::InvalidHeader( + "RAR 5 encrypted file is missing encryption record", + )) + } + + fn packed_data_with_allowance( + &self, + archive: &Archive, + password: Option<&[u8]>, + control: &crate::read_control::ReadControl, + allowance: &B, + ) -> Result<(Buffer, Option)> { + control.check()?; + let (reader, keys) = self.packed_reader_with_allowance(archive, password, allowance)?; + let mut reader = control.reader(reader); + let mut packed = Buffer::new(allowance); + packed.read_to_end(&mut reader)?; + Ok((packed, keys)) + } + + fn packed_reader_with_allowance<'a, B: Budget>( + &self, + archive: &'a Archive, + password: Option<&[u8]>, + allowance: &B, + ) -> Result<( + PackedReader, B>, + Option, + )> { + let reader = archive.range_reader(self.block.data_range.clone())?; + if !self.encrypted { + return Ok((PackedReader::Plain(reader), None)); + } + if !self.packed_size().is_multiple_of(16) { + return Err(Error::InvalidHeader( + "RAR 5 encrypted file payload is not block aligned", + )); + } + let keys = self.encryption_keys(password)?; + let iv = self.encryption_iv()?; + let reader = Boxed::try_new( + || Rar50DecryptingReader::with_keys(reader, keys.key, iv), + allowance, + )?; + Ok((PackedReader::Encrypted(reader), Some(keys))) + } + + #[cfg(all(test, feature = "write", feature = "encryption"))] + fn packed_reader_with_password<'a>( + &self, + archive: &'a Archive, + password: Option<&[u8]>, + ) -> Result<( + PackedReader>, + Option, + )> { + self.packed_reader_with_allowance(archive, password, &Allowance::default()) + } + + /// True when the header carries a BLAKE2sp record this build can check. + /// + /// That record is the authoritative check, and the header's CRC32 is not + /// evaluated beside it. Measured on RAR 7.12 and unrar 7.20 with a file + /// header carrying both: a deliberately wrong CRC32 next to a correct + /// BLAKE2sp tests clean in both readers, while a correct CRC32 next to a + /// corrupted BLAKE2sp is rejected by both. rars used to check both fields + /// and so rejected the first archive. + /// + /// WinRAR writes one or the other, never both, so this only shows up on + /// archives from somewhere else. + fn blake2sp_supersedes_crc32(&self) -> bool { + self.hash + .as_ref() + .is_some_and(|hash| hash.hash_type == 0 && hash.data.len() == 32) + } + + fn needs_crc32(&self) -> bool { + self.data_crc32.is_some() && !self.blake2sp_supersedes_crc32() + } + + pub(super) fn verify_integrity_with_keys( + &self, + data: &[u8], + keys: Option<&Rar50Keys>, + ) -> Result<()> { + if let Some(expected) = self + .data_crc32 + .filter(|_| !self.blake2sp_supersedes_crc32()) + { + let actual = crc32(data); + let actual = if self.uses_hash_mac() { + let keys = keys.ok_or(Error::InvalidHeader( + "RAR 5 encrypted hash MAC needs encryption keys", + ))?; + keys.checked_crc_mac(actual)? + } else { + actual + }; + if actual != expected { + return Err(Error::Crc32Mismatch { expected, actual }); + } + } + + let Some(hash) = &self.hash else { + return Ok(()); + }; + match hash.hash_type { + 0 if hash.data.len() == 32 => { + let actual = blake2sp::hash(data); + let actual = if self.uses_hash_mac() { + let keys = keys.ok_or(Error::InvalidHeader( + "RAR 5 encrypted hash MAC needs encryption keys", + ))?; + keys.checked_hash_mac(actual)? + } else { + actual + }; + if constant_time_eq(&hash.data, &actual) { + Ok(()) + } else { + Err(Error::HashMismatch { hash_type: 0 }) + } + } + 0 => Err(Error::InvalidHeader( + "RAR 5 BLAKE2sp hash record has invalid length", + )), + _ => Ok(()), + } + } + + fn verify_streaming_integrity( + &self, + crc: Crc32, + hash: Option<([u8; 32], blake2sp::Hasher)>, + keys: Option<&Rar50Keys>, + ) -> Result<()> { + if let Some(expected) = self + .data_crc32 + .filter(|_| !self.blake2sp_supersedes_crc32()) + { + let actual = if self.uses_hash_mac() { + let keys = keys.ok_or(Error::InvalidHeader( + "RAR 5 encrypted hash MAC needs encryption keys", + ))?; + keys.checked_crc_mac(crc.finish())? + } else { + crc.finish() + }; + if actual != expected { + return Err(Error::Crc32Mismatch { expected, actual }); + } + } + + if let Some((expected, hasher)) = hash { + let actual = if self.uses_hash_mac() { + let keys = keys.ok_or(Error::InvalidHeader( + "RAR 5 encrypted hash MAC needs encryption keys", + ))?; + keys.checked_hash_mac(hasher.finalize())? + } else { + hasher.finalize() + }; + if !constant_time_eq(&expected, &actual) { + return Err(Error::HashMismatch { hash_type: 0 }); + } + } + Ok(()) + } + + /// Modification time in Unix seconds, using extraction's established + /// base-header precedence and falling back to the extended time record. + /// Unlike filesystem extraction metadata, this retains absent versus epoch. + pub fn modification_time(&self) -> Option { + self.mtime.or(self.htime_mtime) + } + + pub fn modification_time_refinement(&self) -> Option { + // Do not attach a fraction from the lower-priority extended timestamp + // to a different base-header timestamp. + self.mtime + .is_none() + .then_some(self.htime_mtime_refinement) + .flatten() + } + + pub fn metadata(&self) -> ExtractedEntryMeta { + ExtractedEntryMeta { + name: self.name.clone(), + file_time: self.modification_time(), + mtime_refinement: self.modification_time_refinement(), + attr: self.attributes, + host_os: self.host_os, + is_directory: self.is_directory(), + } + } + + pub fn write_to( + &self, + archive: &Archive, + password: Option<&[u8]>, + out: &mut impl Write, + ) -> Result<()> { + let mut session = DecoderSession::new_with_password(password, BUFFERED_DECODE_LIMIT); + session.write_file_to(archive, self, out) + } + + #[cfg(all(test, feature = "write"))] + pub(crate) fn decoded_data_unverified( + &self, + archive: &Archive, + password: Option<&[u8]>, + ) -> Result> { + let mut decoder = ReaderState::new(&Allowance::default()); + Ok(self + .decoded_data_with_decoder(archive, &mut decoder, password)? + .data + .into_vec()) + } + + #[cfg(feature = "recovery")] + pub(super) fn decoded_recovery_data( + &self, + archive: &Archive, + password: Option<&[u8]>, + control: &crate::read_control::ReadControl, + ) -> Result> { + let mut decoder = ReaderState::new(&Allowance::default()); + decoder.read_control = control.clone(); + let decoded = + control.finish(self.decoded_data_with_decoder(archive, &mut decoder, password))?; + control.check()?; + Ok(decoded.data.into_vec()) + } + + pub(super) fn decoded_comment_with_options( + &self, + archive: &Archive, + options: crate::ArchiveReadOptions<'_>, + ) -> Result> { + if let Some(limit) = options.max_reader_workspace_bytes { + return self.decoded_comment_with_allowance( + archive, + options, + &Allowance::limited(limit), + ); + } + self.decoded_comment_with_allowance(archive, options, &Allowance::default()) + } + + fn decoded_comment_with_allowance( + &self, + archive: &Archive, + options: crate::ArchiveReadOptions<'_>, + allowance: &B, + ) -> Result> { + let mut budget = crate::output_limit::OutputBudget::new(options); + if budget.is_limited() && self.known_unpacked_size().is_none() { + return Err(Error::UnsupportedFeature { + version: crate::ArchiveVersion::Rar50, + feature: "output-limited decoding of an unknown-size comment", + }); + } + budget.check(self.unpacked_size, &self.name)?; + if !self.is_stored() { + if let Some(limit) = options.rar50_dictionary_size_limit { + let required = self.decoded_compression_info()?.dictionary_size; + if required > limit { + return Err(self.entry_error( + "checking dictionary limit", + Error::Rar50DictionaryLimitExceeded { limit, required }, + )); + } + } + } + // Comment services historically decode without checking payload hashes. + // Reuse bounded member decoding while retaining that checksum contract. + let payload = FileHeader { + block: self.block.clone(), + file_flags: self.file_flags, + rewrite_metadata_complete: self.rewrite_metadata_complete, + unpacked_size: self.unpacked_size, + attributes: self.attributes, + mtime: self.mtime, + htime_mtime: self.htime_mtime, + htime_mtime_refinement: self.htime_mtime_refinement, + file_times: self.file_times, + data_crc32: None, + compression_info: self.compression_info, + host_os: self.host_os, + name: self.name.clone(), + hash: None, + redirection: None, + // Parsed service payloads belong to the archive. Decoding reads its + // source range; copying service_data would create uncharged staging. + service_data: None, + encrypted: self.encrypted, + encryption: self.encryption.clone(), + crypto: self.crypto.clone(), + }; + let mut session = DecoderSession::with_allowance( + options.password, + options.rar50_buffered_decode_limit.unwrap_or(u64::MAX), + allowance, + ); + session.decoder.read_control = budget.control.clone(); + session.scratch = options.rar50_scratch; + let mut data = Vec::new(); + budget.run(&self.name, &mut data, |writer| { + if payload.is_stored() { + let decoded = payload.decoded_data_with_decoder( + archive, + &mut session.decoder, + options.password, + )?; + writer.write_all(&decoded.data)?; + Ok(()) + } else { + session.write_file_to(archive, &payload, writer) + } + })?; + Ok(data) + } + + fn decoded_data_with_decoder( + &self, + archive: &Archive, + decoder: &mut ReaderState, + password: Option<&[u8]>, + ) -> Result> { + self.decoded_data_with_decoder_and_history( + archive, + decoder, + password, + HistoryRetention::Retain, + ) + } + + fn decoded_data_with_decoder_and_history( + &self, + archive: &Archive, + decoder: &mut ReaderState, + password: Option<&[u8]>, + history: HistoryRetention, + ) -> Result> { + let (packed, keys) = self.packed_data_with_allowance( + archive, + password, + &decoder.read_control, + &decoder.allowance(), + )?; + let data = self.decode_packed_owned(&packed, decoder, history)?; + Ok(DecodedData { data, keys }) + } + + fn decode_packed_owned( + &self, + packed: &[u8], + decoder: &mut ReaderState, + history: HistoryRetention, + ) -> Result> { + if self.is_stored() { + if self.encrypted { + let unpacked_size = usize::try_from(self.unpacked_size).map_err(|_| { + Error::InvalidHeader("RAR 5 unpacked size overflows host address size") + })?; + if packed.len() < unpacked_size { + return Err(Error::InvalidHeader( + "RAR 5 encrypted stored file is shorter than unpacked size", + )); + } + if packed[unpacked_size..].iter().any(|&byte| byte != 0) { + return Err(Error::InvalidHeader( + "RAR 5 encrypted stored file has non-zero padding", + )); + } + return Buffer::copied(&packed[..unpacked_size], &decoder.allowance()) + .map_err(Error::from); + } + if packed.len() as u64 != self.unpacked_size { + return Err(Error::InvalidHeader( + "RAR 5 stored file has mismatched packed and unpacked sizes", + )); + } + return Buffer::copied(packed, &decoder.allowance()).map_err(Error::from); + } + // Empty compressed members can legitimately omit the packed stream. + // Any other member must decode successfully, even without CRC/hash. + if self.unpacked_size == 0 && packed.is_empty() { + return Ok(Buffer::new(&decoder.allowance())); + } + + let info = self.decoded_compression_info()?; + let dictionary_size = usize::try_from(info.dictionary_size).map_err(|_| { + Error::InvalidHeader("RAR 5 dictionary size overflows host address size") + })?; + let output_size = checked_unpacked_size(self.unpacked_size)?; + decoder + .decode_member_with_dictionary_and_history( + packed, + info.algorithm_version, + output_size, + dictionary_size, + info.solid, + DecodeMode::Lz, + history, + ) + .map_err(Error::from) + } + + fn stream_packed_with_decoder( + &self, + packed: &mut R, + keys: Option<&Rar50Keys>, + decoder: &mut ReaderState, + buffered_decode_limit: u64, + scratch_policy: Option<&crate::Rar50Scratch>, + writer: &mut dyn Write, + ) -> Result<()> { + // Split members retain their existing checksum work. Only ordinary + // members select the optional policy, independently of scratch decoding. + self.stream_packed_with_crc_policy::( + packed, + keys, + decoder, + buffered_decode_limit, + scratch_policy, + writer, + ) + } + + fn stream_packed_with_crc_policy( + &self, + packed: &mut R, + keys: Option<&Rar50Keys>, + decoder: &mut ReaderState, + buffered_decode_limit: u64, + scratch_policy: Option<&crate::Rar50Scratch>, + writer: &mut dyn Write, + ) -> Result<()> { + if let Some(policy) = scratch_policy { + return scratch::decode(self, packed, keys, decoder, policy, writer); + } + + let info = self.decoded_compression_info()?; + let dictionary_size = usize::try_from(info.dictionary_size).map_err(|_| { + Error::InvalidHeader("RAR 5 dictionary size overflows host address size") + })?; + let output_size = usize::try_from(self.unpacked_size) + .map_err(|_| Error::InvalidHeader("RAR 5 unpacked size overflows host address size"))?; + let mut crc = Crc32::new(); + let check_crc = !OPTIONAL_CRC || self.needs_crc32(); + let mut hash = streaming_hash_verifier(self)?; + decoder + .decode_member_from_reader_with_dictionary_to_sink( + packed, + info.algorithm_version, + output_size, + dictionary_size, + info.solid, + |chunk| match chunk { + DecodedChunk::Bytes(chunk) => { + if check_crc { + #[cfg(all(test, feature = "write"))] + streaming_checks_tests::record_crc_work(chunk.len(), false); + crc.update(chunk); + } + if let Some((_, hasher)) = &mut hash { + hasher.update(chunk); + } + writer.write_all(chunk) + } + DecodedChunk::Repeated { byte, len } => { + if check_crc { + write_repeated_chunk(writer, &mut crc, &mut hash, byte, len) + } else { + write_repeated_chunk_with_optional_crc( + writer, None, &mut hash, byte, len, + ) + } + } + }, + ) + .map_err(|error| match error { + StreamDecodeError::Decode(error) => Error::from(error), + StreamDecodeError::FilteredMember => Error::Rar50BufferedDecodeLimitExceeded { + limit: buffered_decode_limit, + required: self.unpacked_size, + }, + StreamDecodeError::Sink(error) => Error::from(error), + })?; + self.verify_streaming_integrity(crc, hash, keys) + } + + fn write_stored_with_allowance( + &self, + archive: &Archive, + password: Option<&[u8]>, + writer: &mut dyn Write, + allowance: &B, + ) -> Result<()> { + if !self.encrypted && self.packed_size() != self.unpacked_size { + return Err(self.entry_error( + "decoding", + Error::InvalidHeader("RAR 5 stored file has mismatched packed and unpacked sizes"), + )); + } + let (mut reader, keys) = self + .packed_reader_with_allowance(archive, password, allowance) + .map_err(|error| self.entry_error("decoding", error))?; + let mut crc = Crc32::new(); + let check_crc = self.needs_crc32(); + let mut hash = + streaming_hash_verifier(self).map_err(|error| self.entry_error("decoding", error))?; + let mut written = 0u64; + let mut buf = [0u8; 64 * 1024]; + + loop { + let count = reader + .read(&mut buf) + .map_err(Error::from) + .map_err(|error| self.entry_error("decoding", error))?; + if count == 0 { + break; + } + let remaining = + usize::try_from(self.unpacked_size.saturating_sub(written)).unwrap_or(usize::MAX); + let chunk_len = count.min(remaining); + let chunk = &buf[..chunk_len]; + if self.encrypted && buf[chunk_len..count].iter().any(|&byte| byte != 0) { + return Err(self.entry_error( + "decoding", + Error::InvalidHeader("RAR 5 encrypted stored file has non-zero padding"), + )); + } + // chunk_len is bounded by unpacked_size - written, so this sum + // cannot exceed the declared u64 size, including u64::MAX. + written += chunk.len() as u64; + if check_crc { + #[cfg(all(test, feature = "write"))] + streaming_checks_tests::record_crc_work(chunk.len(), false); + crc.update(chunk); + } + if let Some((_, hasher)) = &mut hash { + hasher.update(chunk); + } + writer + .write_all(chunk) + .map_err(Error::from) + .map_err(|error| self.entry_error("writing", error))?; + } + + if written != self.unpacked_size { + return Err(self.entry_error( + "decoding", + Error::InvalidHeader("RAR 5 stored file has mismatched packed and unpacked sizes"), + )); + } + self.verify_streaming_integrity(crc, hash, keys.as_ref()) + .map_err(|error| self.entry_error("verifying", error)) + } + + #[cfg(all(test, feature = "write"))] + fn decode_packed_with_decoder( + &self, + packed: &[u8], + decoder: &mut ReaderState, + ) -> Result> { + self.decode_packed_owned(packed, decoder, HistoryRetention::Retain) + .map(Buffer::into_vec) + } + + fn entry_error(&self, operation: &'static str, error: Error) -> Error { + error.at_entry(self.name.clone(), operation) + } +} + +fn write_repeated_chunk( + writer: &mut dyn Write, + crc: &mut Crc32, + hash: &mut Option<([u8; 32], blake2sp::Hasher)>, + byte: u8, + len: usize, +) -> std::io::Result<()> { + write_repeated_chunk_with_optional_crc(writer, Some(crc), hash, byte, len) +} + +fn write_repeated_chunk_with_optional_crc( + writer: &mut dyn Write, + mut crc: Option<&mut Crc32>, + hash: &mut Option<([u8; 32], blake2sp::Hasher)>, + byte: u8, + mut len: usize, +) -> std::io::Result<()> { + let buffer = [byte; 64 * 1024]; + while len > 0 { + let take = len.min(buffer.len()); + let chunk = &buffer[..take]; + writer.write_all(chunk)?; + #[cfg(all(test, feature = "write"))] + streaming_checks_tests::record_repeated_output(take, byte == 0); + if let Some(crc) = crc.as_mut() { + #[cfg(all(test, feature = "write"))] + streaming_checks_tests::record_crc_work(take, byte == 0); + if byte == 0 { + crc.update_zeroes(take as u64); + } else { + crc.update(chunk); + } + } + if let Some((_, hasher)) = hash.as_mut() { + hasher.update(chunk); + } + len -= take; + } + Ok(()) +} + +impl Archive { + pub fn extract_to(&self, options: crate::ArchiveReadOptions<'_>, mut open: F) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + self.extract_to_impl(options, &mut open, &mut |_, _| Ok(()), false, None, None) + .map(|_| ()) + } + + pub fn extract_to_with_redirections( + &self, + options: crate::ArchiveReadOptions<'_>, + mut open: F, + mut redirect: R, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, + { + options.check_cancelled()?; + self.extract_to_impl(options, &mut open, &mut redirect, true, None, None) + .map(|_| ()) + } + + pub(crate) fn extract_controlled( + &self, + options: crate::ArchiveReadOptions<'_>, + selector: &mut crate::extraction_control::Selector<'_>, + on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + ) -> Result { + self.extract_to_impl( + options, + &mut |_| unreachable!("controlled selection supplies writer"), + &mut |_, _| Ok(()), + false, + Some(selector), + on_error, + ) + } + + fn extract_to_impl( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + redirect: &mut R, + emit_redirections: bool, + selector: Option<&mut crate::extraction_control::Selector<'_>>, + on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + ) -> Result + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, + { + if let Some(limit) = options.max_reader_workspace_bytes { + return self.extract_with_allowance( + options, + open, + redirect, + emit_redirections, + selector, + on_error, + &crate::codec::workspace::Allowance::limited(limit), + ); + } + self.extract_with_allowance( + options, + open, + redirect, + emit_redirections, + selector, + on_error, + &Allowance::default(), + ) + } + + #[allow(clippy::too_many_arguments)] + fn extract_with_allowance( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + redirect: &mut R, + emit_redirections: bool, + mut selector: Option<&mut crate::extraction_control::Selector<'_>>, + mut on_error: Option<&mut crate::extraction_control::ErrorHandler<'_>>, + allowance: &B, + ) -> Result + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, + { + options.check_cancelled()?; + let mut budget = crate::output_limit::OutputBudget::new(options); + let buffered_decode_limit = rar50_buffered_decode_limit(options); + let mut session = + DecoderSession::with_allowance(options.password, buffered_decode_limit, allowance); + session.decoder.read_control = budget.control.clone(); + session.scratch = options.rar50_scratch; + let solid = selector.is_some() + && (self.main.is_solid() || self.files().any(|file| file.compression_info & 0x40 != 0)); + for file in self.files() { + let selected = crate::extraction_control::select( + &mut selector, + options, + || crate::rar50_member(file), + solid, + )?; + let selected_writer = match selected { + Some(crate::ExtractionDecision::Skip) => continue, + Some(crate::ExtractionDecision::Stop) => { + return Ok(crate::ExtractionOutcome::Stopped); + } + Some(crate::ExtractionDecision::Extract(writer)) => Some(writer), + None => None, + }; + let result = (|| { + options.check_cancelled()?; + file.check_dictionary_limit(options.rar50_dictionary_size_limit)?; + if let Some(redirection) = &file.redirection { + if selected_writer.is_some() { + return Err(file.entry_error( + "selecting", + Error::UnsupportedFeature { + version: crate::ArchiveVersion::Rar50, + feature: "extracting redirections through extraction control", + }, + )); + } + if emit_redirections { + options.check_cancelled()?; + redirect(&file.metadata(), redirection)?; + options.check_cancelled()?; + } + return Ok(()); + } + if file.is_split_before() || file.is_split_after() { + return Err(Error::InvalidHeader( + "RAR 5 split entry requires multivolume extraction", + )); + } + file.check_output_limit(&budget)?; + let meta = file.metadata(); + options.check_cancelled()?; + let mut writer = match selected_writer { + Some(writer) => writer, + None => open(&meta)?, + }; + options.check_cancelled()?; + if !meta.is_directory { + budget.run(&file.name, &mut writer, |writer| { + session.write_file_to(self, file, writer) + })?; + } + Ok(()) + })(); + if crate::extraction_control::finish_member( + &mut on_error, + options, + || crate::rar50_member(file), + solid, + result, + )? { + session = DecoderSession::with_allowance( + options.password, + buffered_decode_limit, + allowance, + ); + session.decoder.read_control = budget.control.clone(); + session.scratch = options.rar50_scratch; + } + } + options.check_cancelled()?; + Ok(crate::ExtractionOutcome::Complete) + } + + /// Decodes independent members in bounded batches, emitting in archive order. + /// Completed batches can have been emitted when a later batch fails. The + /// buffering limit bounds retained payload bytes, not aggregate decoder state. + /// A configured total output ceiling selects sequential extraction. + pub fn extract_to_parallel_buffered( + &self, + options: crate::ArchiveReadOptions<'_>, + mut open: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + self.parallel_to_impl(options, &mut open, false) + } + + pub(crate) fn test_to( + &self, + options: crate::ArchiveReadOptions<'_>, + mut tested: F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result<()>, + { + self.parallel_to_impl( + options, + &mut |meta| { + tested(meta)?; + Ok(Box::new(std::io::sink()) as Box) + }, + true, + ) + } + + fn parallel_to_impl( + &self, + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + verification: bool, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + options.check_cancelled()?; + // Admit and charge total-limited work in archive order before dispatch. + if options.rar50_scratch.is_some() + || options.max_total_output_bytes.is_some() + || self.main.is_solid() + || self.files().any(|file| { + file.is_split_before() + || file.is_split_after() + || file.unpacked_size > rar50_buffered_decode_limit(options) + || file.decoded_compression_info().is_ok_and(|info| info.solid) + }) + { + return self.extract_to(options, open); + } + + if let Some(limit) = options.max_reader_workspace_bytes { + let ledger = Allowance::limited(limit); + let publication = crate::read_control::ReadControl::new(options.cancellation); + return crate::codec::workspace::coordinator::extract_windowed( + self.files(), + &ledger, + |file| { + if file.is_directory() || file.redirection.is_some() { + 0 + } else if file.is_stored() { + if verification { + 0 + } else { + file.unpacked_size + } + } else { + file.unpacked_size + .saturating_add(file.packed_size()) + .saturating_add(8192) + } + }, + |file, local| { + options.check_cancelled()?; + decode_parallel_entry_with_allowance( + self, + file, + options.password, + rar50_buffered_decode_limit(options), + options, + verification, + local, + ) + }, + |entry| write_parallel_entry(entry, open, &mut |_, _| Ok(()), &publication), + ); + } + + let password = options.password; + let buffered_decode_limit = rar50_buffered_decode_limit(options); + let mut files = self.files().peekable(); + let window = crate::parallel::default_window().max(1); + let publication = crate::read_control::ReadControl::new(options.cancellation); + while files.peek().is_some() { + options.check_cancelled()?; + let mut batch = Vec::new(); + let mut remaining = buffered_decode_limit; + while batch.len() < window { + let Some(file) = files.peek() else { break }; + if file.unpacked_size > remaining { + break; + } + remaining -= file.unpacked_size; + batch.push(files.next().expect("peeked file")); + } + // Write the verified members ahead of a failure before reporting it. + let (entries, failure) = crate::parallel::map_prefix(batch, |file| { + decode_parallel_entry( + self, + file, + password, + buffered_decode_limit, + options, + verification, + ) + }); + for entry in entries { + write_parallel_entry(entry, open, &mut |_, _| Ok(()), &publication)?; + } + if let Some(error) = failure { + return Err(error); + } + } + options.check_cancelled()?; + Ok(()) + } +} + +fn decode_parallel_entry( + archive: &Archive, + file: &FileHeader, + password: Option<&[u8]>, + buffered_decode_limit: u64, + options: crate::ArchiveReadOptions<'_>, + verification: bool, +) -> Result { + decode_parallel_entry_with_allowance( + archive, + file, + password, + buffered_decode_limit, + options, + verification, + &Allowance::default(), + ) +} + +enum ParallelExtractedEntry { + Directory(ExtractedEntryMeta), + Verified(ExtractedEntryMeta), + File { + meta: ExtractedEntryMeta, + data: Buffer, + }, + Redirection { + meta: ExtractedEntryMeta, + redirection: FileRedirection, + }, +} + +fn decode_parallel_entry_with_allowance( + archive: &Archive, + file: &FileHeader, + password: Option<&[u8]>, + buffered_decode_limit: u64, + options: crate::ArchiveReadOptions<'_>, + verification: bool, + allowance: &B, +) -> Result> { + options.check_cancelled()?; + let mut budget = crate::output_limit::OutputBudget::new(options); + file.check_dictionary_limit(options.rar50_dictionary_size_limit)?; + file.check_output_limit(&budget)?; + if let Some(redirection) = &file.redirection { + return Ok(ParallelExtractedEntry::Redirection { + meta: file.metadata(), + redirection: redirection.clone(), + }); + } + // The only caller dispatches workers after excluding split members. + let meta = file.metadata(); + if meta.is_directory { + return Ok(ParallelExtractedEntry::Directory(meta)); + } + let mut session = DecoderSession::with_allowance(password, buffered_decode_limit, allowance); + session.decoder.read_control = budget.control.clone(); + session.scratch = options.rar50_scratch; + // Only a disposable buffered worker can omit the final dictionary copy. + // Compression metadata is still validated at its original decode boundary; + // ReaderState also forces retention for solid and limited decoding. + let history = if !B::LIMITED + && !archive.main.is_solid() + && !file.is_split_before() + && !file.is_split_after() + && !file.is_stored() + && session.scratch.is_none() + && !file.should_stream_decode(buffered_decode_limit) + { + HistoryRetention::DiscardIndependent + } else { + HistoryRetention::Retain + }; + if verification { + budget.run(&file.name, std::io::sink(), |writer| { + session.write_file_to_with_history(archive, file, writer, history) + })?; + return Ok(ParallelExtractedEntry::Verified(meta)); + } + let mut data = Buffer::new(allowance); + budget.run(&file.name, &mut data, |writer| { + session.write_file_to_with_history(archive, file, writer, history) + })?; + Ok(ParallelExtractedEntry::File { meta, data }) +} + +fn write_parallel_entry( + entry: ParallelExtractedEntry, + open: &mut F, + redirect: &mut R, + control: &crate::read_control::ReadControl, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, +{ + control.check()?; + match entry { + ParallelExtractedEntry::Directory(meta) | ParallelExtractedEntry::Verified(meta) => { + let _ = open(&meta)?; + } + ParallelExtractedEntry::File { meta, data } => { + let mut writer = open(&meta)?; + control.check()?; + control + .write_all(&mut writer, &data) + .map_err(|error| Error::from(error).at_entry(meta.name.clone(), "extracting"))?; + } + ParallelExtractedEntry::Redirection { meta, redirection } => { + redirect(&meta, &redirection)?; + } + } + control.check()?; + Ok(()) +} + +struct DecodedData { + data: Buffer, + keys: Option, +} + +struct DecoderSession<'a, B: Budget = Allowance> { + decoder: ReaderState, + allowance: B, + password: Option<&'a [u8]>, + buffered_decode_limit: u64, + scratch: Option<&'a crate::Rar50Scratch>, +} + +impl<'a, B: Budget> DecoderSession<'a, B> { + fn with_allowance( + password: Option<&'a [u8]>, + buffered_decode_limit: u64, + allowance: &B, + ) -> Self { + Self { + decoder: ReaderState::new(allowance), + allowance: allowance.clone(), + password, + buffered_decode_limit, + scratch: None, + } + } + + fn write_file_to( + &mut self, + archive: &Archive, + file: &FileHeader, + writer: &mut dyn Write, + ) -> Result<()> { + self.write_file_to_with_history(archive, file, writer, HistoryRetention::Retain) + } + + fn write_file_to_with_history( + &mut self, + archive: &Archive, + file: &FileHeader, + writer: &mut dyn Write, + history: HistoryRetention, + ) -> Result<()> { + if file.is_stored() { + return file.write_stored_with_allowance( + archive, + self.password, + writer, + &self.allowance, + ); + } + if file.should_stream_decode(self.buffered_decode_limit) { + return self.stream_file_to(archive, file, writer); + } + let decoded = self + .decoded_file_data(archive, file, history) + .map_err(|error| file.entry_error("decoding", error))?; + file.verify_integrity_with_keys(&decoded.data, decoded.keys.as_ref()) + .map_err(|error| file.entry_error("verifying", error))?; + writer + .write_all(&decoded.data) + .map_err(Error::from) + .map_err(|error| file.entry_error("writing", error)) + } + + fn stream_file_to( + &mut self, + archive: &Archive, + file: &FileHeader, + writer: &mut dyn Write, + ) -> Result<()> { + let mut streaming_decoder = self.decoder.try_clone_for_stream()?; + let (mut packed, keys) = file + .packed_reader_with_allowance(archive, self.password, &self.allowance) + .map_err(|error| file.entry_error("reading", error))?; + file.stream_packed_with_crc_policy::( + &mut packed, + keys.as_ref(), + &mut streaming_decoder, + self.buffered_decode_limit, + self.scratch, + writer, + ) + .map_err(|error| file.entry_error("decoding", error))?; + self.decoder = streaming_decoder; + Ok(()) + } + + fn decoded_file_data( + &mut self, + archive: &Archive, + file: &FileHeader, + history: HistoryRetention, + ) -> Result> { + file.decoded_data_with_decoder_and_history( + archive, + &mut self.decoder, + self.password, + history, + ) + } + + fn split_decryptor( + &self, + split: &PendingSplitRefs, + volumes: &[Archive], + ) -> Result> { + split.split_decryptor(volumes, self.password) + } + + fn decode_split( + &mut self, + volumes: &[Archive], + split: &PendingSplitRefs, + final_file: &FileHeader, + decryptor: Option<&SplitDecryptor>, + ) -> Result> { + final_file.decode_split_with_decoder(volumes, split, &mut self.decoder, decryptor) + } + + fn stream_split_to( + &mut self, + volumes: &[Archive], + split: &PendingSplitRefs, + final_file: &FileHeader, + decryptor: Option<&SplitDecryptor>, + writer: &mut dyn Write, + ) -> Result<()> { + // Keep solid state only after successful emission and integrity checks, + // just as for a non-split streaming member. + self.decoder.read_control.check()?; + let mut decoder = self.decoder.try_clone_for_stream()?; + let control = decoder.read_control.clone(); + let mut packed = + split.fragment_reader_with_allowance(volumes, decryptor, &self.allowance)?; + control + .finish(final_file.stream_packed_with_decoder( + &mut packed, + decryptor.map(|decryptor| &decryptor.keys), + &mut decoder, + self.buffered_decode_limit, + self.scratch, + writer, + )) + .map_err(|error| match error { + // Sink/reader I/O and the intentional filter limit must retain their + // own meaning. Fragment diagnostics are for decode/integrity failure. + error + if matches!( + error.kind(), + crate::ErrorKind::Io + | crate::ErrorKind::ResourceLimit + | crate::ErrorKind::Cancelled + | crate::ErrorKind::UnsupportedFeature + ) => + { + error + } + error => split.checksum_error(volumes).unwrap_or(error), + })?; + self.decoder = decoder; + Ok(()) + } +} + +impl<'a> DecoderSession<'a, Allowance> { + fn new_with_password(password: Option<&'a [u8]>, buffered_decode_limit: u64) -> Self { + Self::with_allowance(password, buffered_decode_limit, &Allowance::default()) + } +} + +impl FileHeader { + fn should_stream_decode(&self, buffered_decode_limit: u64) -> bool { + self.unpacked_size > buffered_decode_limit + } +} + +fn rar50_buffered_decode_limit(options: crate::ArchiveReadOptions<'_>) -> u64 { + options + .rar50_buffered_decode_limit + .unwrap_or(BUFFERED_DECODE_LIMIT) +} + +/// Streams a RAR 5 multivolume archive set to caller-provided writers. +/// Compressed logical members obey the same buffered-decode threshold as ordinary +/// members. Above it, filtered members return a typed limit error. Direct sinks +/// can receive data before an integrity or decoding failure; callers requiring +/// verified publication must stage output until extraction succeeds. +pub fn extract_volumes_to( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + mut open: F, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, +{ + options.check_cancelled()?; + extract_volumes_to_impl(volumes, options, &mut open, &mut |_, _| Ok(()), false) +} + +pub(crate) fn test_volumes_to( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + mut tested: F, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result<()>, +{ + extract_volumes_to(volumes, options, |meta| { + tested(meta)?; + Ok(Box::new(std::io::sink()) as Box) + }) +} + +pub fn extract_volumes_to_with_redirections( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + mut open: F, + mut redirect: R, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, +{ + options.check_cancelled()?; + extract_volumes_to_impl(volumes, options, &mut open, &mut redirect, true) +} + +fn extract_volumes_to_impl( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + redirect: &mut R, + emit_redirections: bool, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, +{ + if let Some(limit) = options.max_reader_workspace_bytes { + return extract_volumes_with_allowance( + volumes, + options, + open, + redirect, + emit_redirections, + &Allowance::limited(limit), + ); + } + extract_volumes_with_allowance( + volumes, + options, + open, + redirect, + emit_redirections, + &Allowance::default(), + ) +} + +fn extract_volumes_with_allowance( + volumes: &[Archive], + options: crate::ArchiveReadOptions<'_>, + open: &mut F, + redirect: &mut R, + emit_redirections: bool, + allowance: &B, +) -> Result<()> +where + F: FnMut(&ExtractedEntryMeta) -> Result>, + R: FnMut(&ExtractedEntryMeta, &FileRedirection) -> Result<()>, +{ + options.check_cancelled()?; + if volumes.is_empty() { + return Err(Error::InvalidHeader("RAR 5 volume set is empty")); + } + + let password = options.password; + let mut budget = crate::output_limit::OutputBudget::new(options); + let mut split = SplitVolumeState::new(); + let buffered_decode_limit = rar50_buffered_decode_limit(options); + let mut session = DecoderSession::with_allowance(password, buffered_decode_limit, allowance); + session.decoder.read_control = budget.control.clone(); + session.scratch = options.rar50_scratch; + + for (volume_index, archive) in volumes.iter().enumerate() { + for (file_index, file) in archive.files().enumerate() { + options.check_cancelled()?; + // Admission precedes split decryption and checksum-error preference: + // a resource refusal must not read payloads or become a checksum error. + file.check_dictionary_limit(options.rar50_dictionary_size_limit)?; + match split.advance(file.is_split_before(), file.is_split_after()) { + SplitVolumeStep::Regular => { + if let Some(redirection) = &file.redirection { + if emit_redirections { + options.check_cancelled()?; + redirect(&file.metadata(), redirection)?; + options.check_cancelled()?; + } + continue; + } + file.check_output_limit(&budget)?; + let meta = file.metadata(); + options.check_cancelled()?; + let mut writer = open(&meta)?; + options.check_cancelled()?; + if !meta.is_directory { + budget.run(&file.name, &mut writer, |writer| { + session.write_file_to(archive, file, writer) + })?; + } + } + SplitVolumeStep::Start => { + validate_split_fragment(file, password)?; + split.begin(PendingSplitRefs::with_allowance( + file, + volume_index, + file_index, + allowance, + )?); + } + SplitVolumeStep::Continue(current) => { + validate_split_continuation_refs(current, file, password)?; + current.append(volume_index, file_index)?; + } + SplitVolumeStep::Finish(mut completed) => { + validate_split_continuation_refs(&completed, file, password)?; + completed.append(volume_index, file_index)?; + file.check_output_limit(&budget)?; + completed.write_to(volumes, file, &mut session, &mut budget, &mut *open)?; + } + SplitVolumeStep::MissingFirst => { + return Err(Error::InvalidHeader( + "RAR 5 split entry is missing its first part", + )); + } + SplitVolumeStep::Interrupted => { + return Err(Error::InvalidHeader( + "RAR 5 split entry is interrupted by a regular entry", + )); + } + } + } + } + + if split.is_pending() { + return Err(Error::InvalidHeader("RAR 5 split entry is incomplete")); + } + + options.check_cancelled()?; + Ok(()) +} + +fn validate_split_fragment(file: &FileHeader, password: Option<&[u8]>) -> Result<()> { + if file.is_directory() { + return Err(Error::InvalidHeader( + "RAR 5 split directory entry is invalid", + )); + } + if file.encrypted { + crate::crypto::require_encryption()?; + } + if file.encrypted && password.is_none() && file.crypto.is_none() { + return Err(Error::NeedPassword); + } + Ok(()) +} + +fn validate_split_continuation_refs( + pending: &PendingSplitRefs, + file: &FileHeader, + password: Option<&[u8]>, +) -> Result<()> { + validate_split_fragment(file, password)?; + if file.name != pending.name { + return Err(Error::InvalidHeader("RAR 5 split entry name changed")); + } + if file.compression_info != pending.compression_info { + return Err(Error::InvalidHeader( + "RAR 5 split entry compression info changed", + )); + } + if file.encrypted != pending.encrypted { + return Err(Error::InvalidHeader( + "RAR 5 split entry encryption flag changed", + )); + } + Ok(()) +} + +struct PendingSplitRefs { + name: Vec, + fragments: Buffer<(usize, usize), B>, + file_time: Option, + mtime_refinement: Option, + attr: u64, + host_os: u64, + compression_info: u64, + encrypted: bool, +} + +impl PendingSplitRefs { + fn with_allowance( + file: &FileHeader, + volume_index: usize, + file_index: usize, + allowance: &B, + ) -> Result { + Ok(Self { + name: file.name.clone(), + fragments: Buffer::copied(&[(volume_index, file_index)], allowance)?, + file_time: file.modification_time(), + mtime_refinement: file.modification_time_refinement(), + attr: file.attributes, + host_os: file.host_os, + compression_info: file.compression_info, + encrypted: file.encrypted, + }) + } + + fn append(&mut self, volume_index: usize, file_index: usize) -> Result<()> { + self.fragments.try_push((volume_index, file_index))?; + Ok(()) + } + + fn write_to( + self, + volumes: &[Archive], + final_file: &FileHeader, + session: &mut DecoderSession<'_, B>, + budget: &mut crate::output_limit::OutputBudget, + open: &mut F, + ) -> Result<()> + where + F: FnMut(&ExtractedEntryMeta) -> Result>, + { + budget.control.check()?; + let decryptor = session.split_decryptor(&self, volumes)?; + budget.control.check()?; + let meta = ExtractedEntryMeta { + name: self.name.clone(), + file_time: self.file_time, + mtime_refinement: self.mtime_refinement, + attr: self.attr, + host_os: self.host_os, + is_directory: false, + }; + let mut writer = open(&meta)?; + budget.run(&final_file.name, &mut writer, |mut writer| { + // Decode/integrity failures can gain fragment context. I/O failures + // (including the output guard sentinel) must retain their meaning. + if final_file.is_stored() { + return self + .write_stored_to(volumes, final_file, decryptor.as_ref(), &mut writer) + .map_err(|error| { + if error.kind() == crate::ErrorKind::Io { + error + } else { + self.checksum_error(volumes).unwrap_or(error) + } + }) + .map_err(|error| final_file.entry_error("extracting", error)); + } + + if final_file.should_stream_decode(session.buffered_decode_limit) { + return session + .stream_split_to(volumes, &self, final_file, decryptor.as_ref(), &mut writer) + .map_err(|error| final_file.entry_error("decoding", error)); + } + + let data = session + .decode_split(volumes, &self, final_file, decryptor.as_ref()) + .map_err(|error| final_file.entry_error("decoding", error))?; + final_file + .verify_integrity_with_keys( + &data, + decryptor.as_ref().map(|decryptor| &decryptor.keys), + ) + .map_err(|error| self.checksum_error(volumes).unwrap_or(error)) + .map_err(|error| final_file.entry_error("verifying", error))?; + writer + .write_all(&data) + .map_err(Error::from) + .map_err(|error| final_file.entry_error("writing", error))?; + Ok(()) + }) + } + + fn write_stored_to( + &self, + volumes: &[Archive], + final_file: &FileHeader, + decryptor: Option<&SplitDecryptor>, + writer: &mut dyn Write, + ) -> Result<()> { + let mut reader = + self.fragment_reader_with_allowance(volumes, decryptor, &self.fragments.allowance())?; + let mut crc = Crc32::new(); + let mut hash = streaming_hash_verifier(final_file)?; + let mut written = 0u64; + let mut buf = [0u8; 64 * 1024]; + + loop { + let count = reader.read(&mut buf)?; + if count == 0 { + break; + } + let chunk = if final_file.encrypted { + let remaining = usize::try_from(final_file.unpacked_size.saturating_sub(written)) + .unwrap_or(usize::MAX); + let chunk_len = count.min(remaining); + if buf[chunk_len..count].iter().any(|&byte| byte != 0) { + return Err(Error::InvalidHeader( + "RAR 5 encrypted stored split file has non-zero padding", + )); + } + &buf[..chunk_len] + } else { + &buf[..count] + }; + written = written + .checked_add(chunk.len() as u64) + .ok_or(Error::InvalidHeader("RAR 5 stored split size overflows"))?; + crc.update(chunk); + if let Some((_, hasher)) = &mut hash { + hasher.update(chunk); + } + writer.write_all(chunk)?; + } + + if written != final_file.unpacked_size { + return Err(Error::InvalidHeader( + "RAR 5 stored split file has mismatched packed and unpacked sizes", + )); + } + if let Some(expected) = final_file + .data_crc32 + .filter(|_| !final_file.blake2sp_supersedes_crc32()) + { + // Gate on the tweaked-checksum flag, not on "is encrypted". + // An encrypted file whose crypt record leaves flag 0x0002 clear + // stores a plain CRC32, and MACing ours fails a sound archive. + let actual = if final_file.uses_hash_mac() { + let decryptor = decryptor.ok_or(Error::InvalidHeader( + "RAR 5 encrypted split CRC needs encryption keys", + ))?; + decryptor.keys.checked_crc_mac(crc.finish())? + } else { + crc.finish() + }; + if actual != expected { + return Err(Error::Crc32Mismatch { expected, actual }); + } + } + if let Some((expected, hasher)) = hash { + let actual = if final_file.uses_hash_mac() { + let decryptor = decryptor.ok_or(Error::InvalidHeader( + "RAR 5 encrypted split hash needs encryption keys", + ))?; + decryptor.keys.checked_hash_mac(hasher.finalize())? + } else { + hasher.finalize() + }; + if !constant_time_eq(&expected, &actual) { + return Err(Error::HashMismatch { hash_type: 0 }); + } + } + Ok(()) + } + + fn split_decryptor( + &self, + volumes: &[Archive], + password: Option<&[u8]>, + ) -> Result> { + if !self.encrypted { + return Ok(None); + } + // Fragment indices come from this immutable volume slice. + let (volume_index, file_index) = self.fragments[0]; + let archive = &volumes[volume_index]; + let file = archive + .files() + .nth(file_index) + .expect("split fragment index comes from archive enumeration"); + let keys = file.encryption_keys(password)?; + Ok(Some(SplitDecryptor { + keys, + iv: file.encryption_iv()?, + })) + } + + /// Re-reads every fragment that carries a checksum over its own stored + /// bytes and reports the first that disagrees. + /// + /// Only consulted once a member has already failed. Checking as the + /// fragments are read would reject an archive whose fragment checksums say + /// something other than what rars and WinRAR put there, and unrar and RAR + /// 7.12 both extract those. Reading the set a second time to explain a + /// failure is worth it; refusing a member that would have come out intact + /// is not. + fn checksum_error(&self, volumes: &[Archive]) -> Option { + let last = self.fragments.len().saturating_sub(1); + for (index, &(volume_index, file_index)) in self.fragments.iter().enumerate() { + if index == last { + break; + } + // Indices originate in extract_volumes_to_impl's enumeration of + // this same immutable volume slice; neither collection can change. + let archive = &volumes[volume_index]; + let file = archive + .files() + .nth(file_index) + .expect("split fragment index comes from archive enumeration"); + let Some(expected) = file.data_crc32 else { + continue; + }; + let mut crc = Crc32::new(); + if archive + .copy_range_to(file.block.data_range.clone(), &mut CrcSink(&mut crc)) + .is_err() + { + continue; + } + let actual = crc.finish(); + if actual != expected { + return Some(Error::InVolume { + number: volume_index + 1, + source: Box::new(Error::Crc32Mismatch { expected, actual }), + }); + } + } + None + } + + fn fragment_reader_with_allowance<'a>( + &self, + volumes: &'a [Archive], + decryptor: Option<&SplitDecryptor>, + allowance: &B, + ) -> Result, B>, B>> { + let mut readers = Buffer::with_capacity(self.fragments.len(), allowance)?; + for &(volume_index, file_index) in &self.fragments { + let archive = &volumes[volume_index]; + let file = archive + .files() + .nth(file_index) + .expect("split fragment index comes from archive enumeration"); + readers.push_admitted(archive.range_reader(file.block.data_range.clone())?); + } + let chained = ChainedReader::with_readers(readers); + if let Some(decryptor) = decryptor { + Ok(PackedReader::Encrypted(Boxed::try_new( + || Rar50DecryptingReader::with_keys(chained, decryptor.keys.key, decryptor.iv), + allowance, + )?)) + } else { + Ok(PackedReader::Plain(chained)) + } + } +} + +#[cfg(all(test, feature = "write"))] +impl PendingSplitRefs { + fn new(file: &FileHeader, volume_index: usize, file_index: usize) -> Self { + Self::with_allowance(file, volume_index, file_index, &Allowance::default()) + .expect("unlimited split bookkeeping") + } +} + +struct SplitDecryptor { + keys: Rar50Keys, + iv: [u8; 16], +} + +fn streaming_hash_verifier(file: &FileHeader) -> Result> { + let Some(hash) = &file.hash else { + return Ok(None); + }; + match hash.hash_type { + 0 if hash.data.len() == 32 => { + let mut expected = [0u8; 32]; + expected.copy_from_slice(&hash.data); + Ok(Some((expected, blake2sp::Hasher::new()))) + } + 0 => Err(Error::InvalidHeader( + "RAR 5 BLAKE2sp hash record has invalid length", + )), + _ => Ok(None), + } +} + +fn checked_unpacked_size(size: u64) -> Result { + usize::try_from(size) + .map_err(|_| Error::InvalidHeader("RAR 5 unpacked size overflows host address size")) +} + +fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + if left.len() != right.len() { + return false; + } + let mut diff = 0u8; + for (&left, &right) in left.iter().zip(right) { + diff |= left ^ right; + } + diff == 0 +} + +impl FileHeader { + fn decode_split_with_decoder( + &self, + volumes: &[Archive], + split: &PendingSplitRefs, + decoder: &mut ReaderState, + decryptor: Option<&SplitDecryptor>, + ) -> Result> { + // PendingSplitRefs::write_to dispatches stored members to + // write_stored_to before calling this compressed-member decoder. + let info = self.decoded_compression_info()?; + let dictionary_size = usize::try_from(info.dictionary_size).map_err(|_| { + Error::InvalidHeader("RAR 5 dictionary size overflows host address size") + })?; + let mut reader = + split.fragment_reader_with_allowance(volumes, decryptor, &decoder.allowance())?; + let output_size = checked_unpacked_size(self.unpacked_size)?; + let decoded = decoder + .decode_member_from_reader_with_dictionary( + &mut reader, + info.algorithm_version, + output_size, + dictionary_size, + info.solid, + DecodeMode::Lz, + ) + .map_err(Error::from); + // A damaged volume usually derails the decoder before the fragment it + // sits in is read to its end, so the reader's own check never fires and + // the member fails as "truncated" instead. Checking the fragments here + // turns that into the checksum mismatch it is. + match decoded { + Err(error) + if matches!( + error.kind(), + crate::ErrorKind::ResourceLimit + | crate::ErrorKind::Cancelled + | crate::ErrorKind::Io + | crate::ErrorKind::UnsupportedFeature + ) => + { + Err(error) + } + Err(error) => Err(split.checksum_error(volumes).unwrap_or(error)), + decoded => decoded, + } + } +} + +/// Feeds what it is given to a running CRC32 and keeps none of it. +struct CrcSink<'a>(&'a mut Crc32); + +impl Write for CrcSink<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +enum PackedReader { + Plain(R), + Encrypted(Boxed, B>), +} +impl Read for PackedReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self { + Self::Plain(reader) => reader.read(out), + Self::Encrypted(reader) => reader.read(out), + } + } +} + +#[cfg(not(feature = "encryption"))] +type Rar50DecryptingReader = crate::crypto::unavailable::Reader; + +#[cfg(feature = "encryption")] +struct Rar50DecryptingReader { + inner: R, + cipher: Rar50Cipher, + buffer: [u8; 16], + encrypted_len: usize, + pos: usize, + len: usize, +} + +#[cfg(feature = "encryption")] +impl Rar50DecryptingReader { + fn with_keys(inner: R, key: [u8; 32], iv: [u8; 16]) -> Result { + Ok(Self::new(inner, key, iv)) + } + + fn new(inner: R, key: [u8; 32], iv: [u8; 16]) -> Self { + Self { + inner, + cipher: Rar50Cipher::new(key, iv), + buffer: [0; 16], + encrypted_len: 0, + pos: 0, + len: 0, + } + } + + fn fill_buffer(&mut self) -> std::io::Result { + // Preserve bytes already consumed if the source returns an error. + // In particular, Read::read_to_end retries Interrupted automatically. + while self.encrypted_len < self.buffer.len() { + let count = self.inner.read(&mut self.buffer[self.encrypted_len..])?; + if count == 0 { + if self.encrypted_len == 0 { + return Ok(false); + } + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "truncated RAR 5 encrypted stream", + )); + } + self.encrypted_len += count; + } + self.cipher.decrypt_block(&mut self.buffer); + self.encrypted_len = 0; + self.pos = 0; + self.len = self.buffer.len(); + Ok(true) + } +} + +#[cfg(feature = "encryption")] +impl Read for Rar50DecryptingReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if out.is_empty() { + return Ok(0); + } + if self.pos == self.len && !self.fill_buffer()? { + return Ok(0); + } + let count = out.len().min(self.len - self.pos); + out[..count].copy_from_slice(&self.buffer[self.pos..self.pos + count]); + self.pos += count; + Ok(count) + } +} + +#[cfg(all(test, feature = "write"))] +mod tests { + #[test] + fn checksum_sink_flush_preserves_accumulated_checksum() { + let mut crc = Crc32::new(); + { + let mut sink = CrcSink(&mut crc); + sink.flush().unwrap(); + assert_eq!(sink.write(b"").unwrap(), 0); + assert_eq!(sink.write(b"a").unwrap(), 1); + sink.flush().unwrap(); + sink.write_all(b"bc").unwrap(); + sink.flush().unwrap(); + sink.flush().unwrap(); + } + assert_eq!(crc.finish(), 0x3524_41c2); + } + + #[test] + fn reader_workspace_split_descriptor_growth_is_admitted_before_mutation() { + let first = stored_split_archive(b"a", b"a", crc32(b"a"), HFL_SPLIT_AFTER) + .files() + .next() + .unwrap() + .clone(); + let bytes = std::mem::size_of::<(usize, usize)>() as u64; + let quota = Allowance::limited(bytes); + let mut pending = PendingSplitRefs::with_allowance(&first, 0, 0, "a).unwrap(); + assert_eq!(quota.used(), bytes); + let error = pending.append(1, 0).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(&pending.fragments[..], &[(0, 0)]); + assert_eq!(quota.used(), bytes); + drop(pending); + assert_eq!(quota.used(), 0); + } + + #[test] + fn compressed_split_reader_releases_each_refused_allocation() { + let volumes = [ + include_bytes!("../../tests/fixtures/rar50/multivol.part1.rar").as_slice(), + include_bytes!("../../tests/fixtures/rar50/multivol.part2.rar").as_slice(), + include_bytes!("../../tests/fixtures/rar50/multivol.part3.rar").as_slice(), + ] + .map(|bytes| Archive::parse(bytes).unwrap()); + let first = volumes[0].files().next().unwrap(); + let last = volumes[2].files().next().unwrap(); + assert!(!last.is_stored()); + refuse_each_reader_session_allocation(|budget| { + let mut split = PendingSplitRefs::with_allowance(first, 0, 0, budget)?; + split.append(1, 0)?; + split.append(2, 0)?; + let mut decoder = ReaderState::new(budget); + let decoded = last.decode_split_with_decoder(&volumes, &split, &mut decoder, None)?; + assert_eq!(decoded.len(), 4096); + assert_eq!(crc32(&decoded), 0xb9c5_4415); + Ok(()) + }); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_split_reader_releases_descriptor_and_cipher_allocations() { + let volumes = [ + include_bytes!("../../tests/fixtures/rar50/encrypted_multivol.part1.rar").as_slice(), + include_bytes!("../../tests/fixtures/rar50/encrypted_multivol.part2.rar").as_slice(), + include_bytes!("../../tests/fixtures/rar50/encrypted_multivol.part3.rar").as_slice(), + ] + .map(|bytes| Archive::parse_with_password(bytes, Some(b"password")).unwrap()); + let first = volumes[0].files().next().unwrap(); + assert!(first.encrypted); + let decryptor = SplitDecryptor { + keys: first.encryption_keys(Some(b"password")).unwrap(), + iv: first.encryption_iv().unwrap(), + }; + refuse_each_reader_session_allocation(|budget| { + let mut split = PendingSplitRefs::with_allowance(first, 0, 0, budget)?; + split.append(1, 0)?; + split.append(2, 0)?; + let _reader = + split.fragment_reader_with_allowance(&volumes, Some(&decryptor), budget)?; + Ok(()) + }); + } + + #[test] + fn reader_session_workspace_refusals_release_rar5_scratch_filter_buffers_and_files() { + let dir = crate::scratch::case("reader-workspace-rar5-scratch"); + let policy = crate::Rar50Scratch::new(&*dir, 4096); + let data = b"scratch Delta quota buffers scratch Delta quota buffers".repeat(2); + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + ) + .entry(entry(b"filtered.bin", &data)) + .filter_policy(FilterPolicy::Explicit(crate::FilterSpec::whole( + FilterKind::Delta { channels: 2 }, + ))) + .finish() + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + refuse_each_reader_session_allocation(|budget| { + let result = (|| { + let mut session = DecoderSession::with_allowance(None, 1, budget); + session.scratch = Some(&policy); + let mut out = Buffer::new(budget); + session.write_file_to(&archive, file, &mut out)?; + assert_eq!(&out[..], &data); + Ok(()) + })(); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + result + }); + let budget = Allowance::limited(1); + let mut session = DecoderSession::with_allowance(None, 1, &budget); + session.scratch = Some(&policy); + let error = session + .write_file_to(&archive, file, &mut std::io::sink()) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(budget.used(), 0); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } + + fn refuse_each_reader_session_allocation( + mut run: impl FnMut(&crate::codec::workspace::RefusingBudget) -> Result<()>, + ) { + use crate::codec::workspace::RefusingBudget; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline).unwrap(); + assert_eq!(baseline.used(), 0); + assert!(baseline.attempts() > 0); + for index in 0..baseline.attempts() { + let budget = RefusingBudget::new(index); + let error = run(&budget).unwrap_err(); + assert_eq!( + error.kind(), + crate::ErrorKind::Cancelled, + "allocation {index}: {error}" + ); + assert_eq!(budget.used(), 0, "allocation {index}"); + } + } + + #[test] + fn reader_session_workspace_refusals_release_rar5_packed_output_cipher_and_checkpoints() { + let data = b"RAR5 charged member payload!\n".repeat(3); + for encrypted in crate::test_features::encryption_cases() { + for filtered in [false, true] { + let input = if encrypted { + entry(b"charged.bin", &data).with_password(b"password".to_vec()) + } else { + entry(b"charged.bin", &data) + }; + let mut writer = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + ) + .entry(input); + if filtered { + writer = writer.filter_policy(FilterPolicy::Explicit( + crate::FilterSpec::whole(FilterKind::Delta { channels: 2 }), + )); + } + let bytes = writer.finish().unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let file = archive.files().next().unwrap(); + assert!(!file.is_stored()); + for limit in if filtered { + vec![u64::MAX] + } else { + vec![u64::MAX, 1] + } { + refuse_each_reader_session_allocation(|budget| { + let mut session = + DecoderSession::with_allowance(Some(b"password"), limit, budget); + let mut out = Buffer::new(budget); + session.write_file_to(&archive, file, &mut out)?; + assert_eq!(&out[..], &data); + let mut second = Buffer::new(budget); + session.write_file_to(&archive, file, &mut second)?; + assert_eq!(&second[..], &data); + Ok(()) + }); + } + } + } + } + + use super::super::{ + ArchiveEntry, ArchiveSource, Block, BlockHeader, FileEncryption, FileHash, FilterKind, + FilterPolicy, MainHeader, Rar50Writer, WriterOptions, HEAD_FILE, HFL_SPLIT_AFTER, + HFL_SPLIT_BEFORE, + }; + use super::*; + use crate::{ArchiveVersion, FeatureSet}; + use std::cell::RefCell; + #[cfg(feature = "encryption")] + use std::io::Cursor; + use std::rc::Rc; + use std::sync::Arc; + + /// Builds a member from bytes the test already holds. + fn entry(name: &[u8], data: &[u8]) -> ArchiveEntry { + ArchiveEntry::new( + name.to_vec(), + crate::EntrySource::from_bytes(Arc::<[u8]>::from(data.to_vec())), + ) + } + + #[test] + fn streaming_checkpoint_rolls_back_filter_sink_cancellation_and_integrity_failures() { + let mut word = 0x1234_5678u32; + // Avoid a period aligned with the 64 KiB flush: appending or resetting + // to the first failed flush must change the successor's match bytes. + let seed: Vec<_> = (0..32749) + .map(|_| { + word ^= word << 13; + word ^= word >> 17; + word ^= word << 5; + word as u8 + }) + .collect(); + let first_data = seed.repeat(4); + let next_data = first_data[127..4224].to_vec(); + let mut builder = crate::Builder::new(ArchiveVersion::Rar50) + .store(false) + .solid(true) + .compression_level(Some(1)); + builder + .add_bytes(b"first".to_vec(), first_data.clone(), None, None) + .unwrap(); + builder + .add_bytes(b"next".to_vec(), next_data.clone(), None, None) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert!(files.iter().all(|file| !file.is_stored())); + assert!(files[1].decoded_compression_info().unwrap().solid); + assert!( + files[1].packed_size() < 128, + "next member must use old history" + ); + + let filtered_data = b"late filter checkpoint bytes\n".repeat(8192); + let filtered_bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + ) + .entry(entry(b"filtered", &filtered_data)) + .filter_policy(FilterPolicy::Explicit(crate::FilterSpec::range( + FilterKind::E8, + 128 * 1024..filtered_data.len(), + ))) + .finish() + .unwrap(); + let filtered_archive = Archive::parse_owned(filtered_bytes).unwrap(); + let mut filtered = filtered_archive.files().next().unwrap().clone(); + filtered.compression_info |= 0x40; + + struct CancelOnWrite<'a> { + token: &'a crate::ReadCancellation, + written: usize, + } + impl Write for CancelOnWrite<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.written += bytes.len(); + self.token.cancel(); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + struct RejectWrite; + impl Write for RejectWrite { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("checkpoint sink failed")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + for failure in ["filter", "sink", "cancellation", "integrity"] { + let mut session = DecoderSession::with_allowance(None, 0, &Allowance::default()); + session + .write_file_to(&archive, files[0], &mut std::io::sink()) + .unwrap(); + match failure { + "filter" => { + let mut partial = Vec::new(); + let error = session + .write_file_to(&filtered_archive, &filtered, &mut partial) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert!(partial.len() >= 64 * 1024, "filter follows a flush"); + } + "sink" => { + let error = session + .write_file_to(&archive, files[0], &mut RejectWrite) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + } + "cancellation" => { + let token = crate::ReadCancellation::new(); + session.decoder.read_control = + crate::read_control::ReadControl::new(Some(&token)); + let mut writer = CancelOnWrite { + token: &token, + written: 0, + }; + let error = session + .write_file_to(&archive, files[0], &mut writer) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Cancelled); + assert!(writer.written >= 64 * 1024); + session.decoder.read_control = Default::default(); + } + "integrity" => { + let mut corrupt = files[1].clone(); + corrupt.hash = None; + corrupt.data_crc32 = Some(crc32(&next_data) ^ 1); + let error = session + .write_file_to(&archive, &corrupt, &mut std::io::sink()) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ChecksumMismatch); + } + _ => unreachable!(), + } + // The successor references the old, nonperiodic raw dictionary. + // A failed checkpoint must not have reset or appended to it. + let mut decoded = Vec::new(); + session + .write_file_to(&archive, files[1], &mut decoded) + .unwrap(); + assert_eq!(decoded, next_data, "after {failure}"); + } + + // Prove the successor detects the two erroneous history states that + // the failed first flush could publish, rather than accepting a fixture + // whose repeating phase accidentally masks a broken checkpoint. + let prefix = &first_data[..64 * 1024]; + let packed_prefix = crate::codec::rar50::encode_literal_only(prefix, 0).unwrap(); + for appended in [false, true] { + let mut session = DecoderSession::with_allowance(None, 0, &Allowance::default()); + session + .write_file_to(&archive, files[0], &mut std::io::sink()) + .unwrap(); + session + .decoder + .decode_member_from_reader_with_dictionary_to_sink( + &mut packed_prefix.as_slice(), + 0, + prefix.len(), + 4 * 1024 * 1024, + appended, + |_| Ok::<(), std::convert::Infallible>(()), + ) + .unwrap(); + let error = session + .write_file_to(&archive, files[1], &mut std::io::sink()) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ChecksumMismatch); + } + } + + #[test] + fn parallel_extraction_emits_a_bounded_batch_before_decoding_the_next() { + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ) + .entries([entry(b"first", &[1; 64]), entry(b"second", &[2; 64])]) + .finish() + .unwrap(); + let mut archive = Archive::parse_owned(bytes).unwrap(); + for block in &mut archive.blocks { + if let super::super::Block::File(file) = block { + if file.name == b"second" { + file.hash = None; + file.data_crc32 = Some(0); + } + } + } + let mut opened = Vec::new(); + let error = archive + .extract_to_parallel_buffered( + crate::ArchiveReadOptions::default().with_rar50_buffered_decode_limit(64), + |meta| { + opened.push(meta.name.clone()); + Ok(Box::new(std::io::sink())) + }, + ) + .unwrap_err(); + assert!(matches!(error, Error::AtEntry { source, .. } + if matches!(*source, Error::Crc32Mismatch { .. }))); + assert_eq!(opened, vec![b"first".to_vec()]); + } + + #[test] + fn parallel_extraction_handles_empty_entries_and_small_limits() { + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ) + .entries([entry(b"empty", &[]), entry(b"data", &[1; 64])]) + .finish() + .unwrap(); + let archive = Archive::parse_owned(bytes).unwrap(); + for limit in [0, 32, 64, 128] { + let mut opened = Vec::new(); + archive + .extract_to_parallel_buffered( + crate::ArchiveReadOptions::default().with_rar50_buffered_decode_limit(limit), + |meta| { + opened.push(meta.name.clone()); + Ok(Box::new(std::io::sink())) + }, + ) + .unwrap(); + assert_eq!(opened, vec![b"empty".to_vec(), b"data".to_vec()]); + } + } + + fn plain_file(name: &[u8], data: &[u8], hash: Option) -> FileHeader { + FileHeader { + block: empty_block(HEAD_FILE, 0, 0..0), + file_flags: 0, + rewrite_metadata_complete: true, + unpacked_size: data.len() as u64, + attributes: 0x20, + mtime: None, + htime_mtime: None, + htime_mtime_refinement: None, + file_times: None, + data_crc32: None, + compression_info: 0, + host_os: 2, + name: name.to_vec(), + hash, + redirection: None, + service_data: None, + encrypted: false, + encryption: None, + crypto: None, + } + } + + #[test] + fn limited_parallel_extraction_accounts_for_directory_link_and_stored_file() { + struct Capture(Rc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let mut directory = plain_file(b"dir", b"", None); + directory.file_flags |= super::super::FHFL_DIRECTORY; + let mut link = plain_file(b"link", b"", None); + link.redirection = Some(super::super::FileRedirection { + redirection_type: 1, + flags: 0, + target_name: b"file".to_vec(), + }); + let mut file = plain_file(b"file", b"payload", None); + file.block = empty_block(HEAD_FILE, 0, 0..7); + let archive = archive_with_blocks( + vec![Block::File(directory), Block::File(link), Block::File(file)], + b"payload".to_vec(), + ); + let output = Rc::new(RefCell::new(Vec::new())); + let mut seen = Vec::new(); + archive + .extract_to_parallel_buffered( + crate::ArchiveReadOptions::new().with_max_reader_workspace_bytes(1 << 20), + |meta| { + seen.push((meta.name.clone(), meta.is_directory)); + Ok(Box::new(Capture(output.clone()))) + }, + ) + .unwrap(); + assert_eq!(seen, [(b"dir".to_vec(), true), (b"file".to_vec(), false)]); + assert_eq!(&*output.borrow(), b"payload"); + } + + #[test] + fn direct_streaming_extracts_compressed_zero_run() { + struct Capture(Rc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let data = vec![0; 16 * 1024]; + let bytes = Rar50Writer::new(WriterOptions::new( + ArchiveVersion::Rar50, + FeatureSet::store_only(), + )) + .entries([entry(b"zeroes", &data)]) + .finish() + .unwrap(); + let archive = Archive::parse_owned(bytes).unwrap(); + assert!(!archive.files().next().unwrap().is_stored()); + let output = Rc::new(RefCell::new(Vec::new())); + archive + .extract_to( + crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(0), + |_| Ok(Box::new(Capture(output.clone()))), + ) + .unwrap(); + assert_eq!(*output.borrow(), data); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_streams_rar50_blocks() { + let key = [3u8; 32]; + let iv = [4u8; 16]; + let plain = *b"0123456789abcdefRAR5 block two!!"; + let mut encrypted = plain; + Rar50Cipher::new(key, iv) + .encrypt_in_place(&mut encrypted) + .unwrap(); + let mut reader = Rar50DecryptingReader::new(Cursor::new(encrypted), key, iv); + let mut out = Vec::new(); + let mut buf = [0u8; 5]; + + loop { + let count = reader.read(&mut buf).unwrap(); + if count == 0 { + break; + } + out.extend_from_slice(&buf[..count]); + } + + assert_eq!(out, plain); + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_preserves_partial_blocks_across_source_errors() { + let key = [3; 32]; + let iv = [4; 16]; + let plain = *b"0123456789abcdefRAR5 block two!!"; + let mut encrypted = plain; + Rar50Cipher::new(key, iv) + .encrypt_in_place(&mut encrypted) + .unwrap(); + for kind in [ + std::io::ErrorKind::Interrupted, + std::io::ErrorKind::PermissionDenied, + ] { + for fail_at in [0, 5, 16, 21] { + let inner = + crate::read_errors::ErrorOnceReader::new(encrypted.to_vec(), fail_at, kind); + let mut reader = Rar50DecryptingReader::new(inner, key, iv); + let mut out = Vec::new(); + let result = reader.read_to_end(&mut out); + if kind == std::io::ErrorKind::Interrupted { + result.unwrap(); + } else { + let error = result.unwrap_err(); + assert_eq!(error.kind(), kind); + assert_eq!(error.to_string(), "source read failed"); + assert_eq!(out, plain[..fail_at as usize / 16 * 16]); + reader.read_to_end(&mut out).unwrap(); + } + assert_eq!(out, plain, "error {kind:?} at {fail_at}"); + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn decrypting_reader_handles_short_reads_and_truncated_final_blocks() { + let key = [3u8; 32]; + let iv = [4u8; 16]; + let plain = *b"0123456789abcdefRAR5 block two!!"; + let mut encrypted = plain; + Rar50Cipher::new(key, iv) + .encrypt_in_place(&mut encrypted) + .unwrap(); + + struct ShortReads(Cursor>); + impl Read for ShortReads { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let count = out.len().min(3); + self.0.read(&mut out[..count]) + } + } + + for length in 0..=encrypted.len() { + let mut reader = Rar50DecryptingReader::new( + ShortReads(Cursor::new(encrypted[..length].to_vec())), + key, + iv, + ); + // An empty destination must neither consume nor validate input. + assert_eq!(reader.read(&mut []).unwrap(), 0); + assert_eq!(reader.inner.0.position(), 0); + let mut out = Vec::new(); + let result = reader.read_to_end(&mut out); + let complete_length = length / 16 * 16; + assert_eq!(out, plain[..complete_length], "ciphertext length {length}"); + if length % 16 == 0 { + result.unwrap(); + assert_eq!(reader.read(&mut [0]).unwrap(), 0); + } else { + assert_eq!( + result.unwrap_err().kind(), + std::io::ErrorKind::UnexpectedEof, + "ciphertext length {length}" + ); + } + } + } + + #[test] + fn split_checksum_diagnostics_continue_past_uncheckable_fragments() { + let data = b"fragment"; + let scratch = crate::scratch::case("rar5-split-checksum-source"); + let missing = scratch.join("removed.part"); + std::fs::write(&missing, data).unwrap(); + for first_kind in 0..5 { + let mut first = stored_split_archive(data, data, crc32(data), HFL_SPLIT_AFTER); + match first_kind { + 0 => { + let Block::File(file) = &mut first.blocks[0] else { + unreachable!(); + }; + file.data_crc32 = None; + } + 1 => {} + 2 => first.source = ArchiveSource::File(Arc::new(missing.clone())), + 3 | 4 => { + let Block::File(file) = &mut first.blocks[0] else { + unreachable!(); + }; + file.block.data_range = if first_kind == 3 { + data.len()..0 + } else { + 0..data.len() + 1 + }; + } + _ => unreachable!(), + } + let mut volumes = vec![ + first, + stored_split_archive( + data, + data, + crc32(data) ^ 1, + HFL_SPLIT_BEFORE | HFL_SPLIT_AFTER, + ), + stored_split_archive(data, data, crc32(data) ^ 2, HFL_SPLIT_BEFORE), + ]; + let mut pending = PendingSplitRefs::new(volumes[0].files().next().unwrap(), 0, 0); + pending.append(1, 0).unwrap(); + pending.append(2, 0).unwrap(); + if first_kind == 2 { + std::fs::remove_file(&missing).unwrap(); + } + let error = pending.checksum_error(&volumes).unwrap(); + assert!(matches!(error, Error::InVolume { number: 2, source } + if matches!(*source, Error::Crc32Mismatch { expected, actual } + if expected == crc32(data) ^ 1 && actual == crc32(data)))); + let Block::File(file) = &mut volumes[1].blocks[0] else { + unreachable!(); + }; + file.data_crc32 = Some(crc32(data)); + // The final fragment carries the member checksum, not a checksum + // of its packed bytes, and must be excluded from this diagnostic. + assert!(pending.checksum_error(&volumes).is_none()); + } + } + + #[test] + fn stored_split_entries_stream_fragments_to_writer() { + struct SharedWriter(Rc>>); + + impl Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let first = b"stored "; + let second = b"split payload"; + let full = [first.as_slice(), second.as_slice()].concat(); + let expected_crc = crc32(&full); + let volumes = vec![ + stored_split_archive(first, &full, expected_crc, HFL_SPLIT_AFTER), + stored_split_archive(second, &full, expected_crc, HFL_SPLIT_BEFORE), + ]; + let captured = Rc::new(RefCell::new(Vec::new())); + let sink = captured.clone(); + + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default(), + move |_meta| Ok(Box::new(SharedWriter(sink.clone()))), + ) + .unwrap(); + + assert_eq!(&*captured.borrow(), &full); + } + + #[test] + fn filtered_checksum_failure_cannot_publish_unfiltered_payload() { + let data = [ + b"prefix".as_slice(), + b"\xe8\0\0\0\0code".repeat(20).as_slice(), + ] + .concat(); + let mut bytes = Rar50Writer::new(WriterOptions::new( + ArchiveVersion::Rar50, + FeatureSet::store_only(), + )) + .entry(entry(b"filtered.bin", &data)) + .filter_policy(FilterPolicy::explicit(FilterKind::E8)) + .finish() + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + let raw = ReaderState::new(&Allowance::default()) + .decode_member_with_dictionary( + &file.packed_data(&archive).unwrap(), + info.algorithm_version, + data.len(), + info.dictionary_size as usize, + false, + DecodeMode::LzNoFilters, + ) + .unwrap(); + assert_ne!(raw, data); + let mut fields = super::super::HeaderReader::new(&bytes, file.block.header_range.clone()); + let flags = fields.read_vint().unwrap(); + fields.read_vint().unwrap(); + fields.read_vint().unwrap(); + if flags & super::super::FHFL_MTIME != 0 { + fields.read_u32().unwrap(); + } + let crc_pos = fields.pos; + assert_ne!(flags & super::super::FHFL_CRC32, 0); + let header_end = file.block.header_range.end + file.block.extra_area_size.unwrap() as usize; + let mut hash_range = None; + super::super::parse_extra_records( + &bytes, + file.block.header_range.end..header_end, + false, + &crate::read_control::ReadControl::default(), + |kind, range| { + if kind == super::super::FHEXTRA_HASH { + assert_eq!(bytes[range.start], 0); + hash_range = Some(range.start + 1..range.end); + } + Ok(()) + }, + ) + .unwrap(); + bytes[crc_pos..crc_pos + 4].copy_from_slice(&crc32(&raw).to_le_bytes()); + bytes[hash_range.unwrap()].copy_from_slice(&blake2sp::hash(&raw)); + let start = file.block.offset; + let header_crc = crc32(&bytes[start + 4..header_end]); + bytes[start..start + 4].copy_from_slice(&header_crc.to_le_bytes()); + let archive = Archive::parse(&bytes).unwrap(); + let captured = Rc::new(RefCell::new(Vec::new())); + struct Capture(Rc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let error = archive + .extract_to(crate::ArchiveReadOptions::new(), |_| { + Ok(Box::new(Capture(captured.clone()))) + }) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::HashMismatch { hash_type: 0 } + )); + assert!(captured.borrow().is_empty()); + } + + #[test] + fn bounded_filtered_members_use_buffered_decode() { + let mut data = Vec::new(); + while data.len() + 29 <= BUFFERED_DECODE_LIMIT as usize { + data.extend_from_slice(b"\xe8\0\0\0\0filtered payload block\n"); + } + assert!(data.len() as u64 <= BUFFERED_DECODE_LIMIT); + + let archive = Rar50Writer::new(WriterOptions { + target: crate::ArchiveVersion::Rar50, + features: crate::FeatureSet::store_only(), + compression_level: None, + dictionary_size: None, + }) + .entries( + [entry(b"filtered.bin", &data) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .filter_policy(FilterPolicy::explicit(FilterKind::E8)) + .finish() + .unwrap(); + let archive = Archive::parse(&archive).unwrap(); + let file = archive.files().next().unwrap(); + assert!(!file.should_stream_decode(BUFFERED_DECODE_LIMIT)); + + let mut out = Vec::new(); + file.write_to(&archive, None, &mut out).unwrap(); + + assert_eq!(out, data); + } + + #[test] + fn streaming_filtered_members_return_typed_error_without_preflight_decode() { + for prefix_len in [0, 64 * 1024] { + let mut data = vec![b'P'; prefix_len]; + while (data.len() - prefix_len) as u64 <= BUFFERED_DECODE_LIMIT { + data.extend_from_slice(b"\xe8\0\0\0\0filtered payload block\n"); + } + + let archive = Rar50Writer::new(WriterOptions { + target: crate::ArchiveVersion::Rar50, + features: crate::FeatureSet::store_only(), + compression_level: None, + dictionary_size: None, + }) + .entries( + [entry(b"filtered.bin", &data) + .with_attributes(0x20) + .with_host_os(3)] + .to_vec(), + ) + .filter_policy(FilterPolicy::Explicit(crate::FilterSpec::range( + FilterKind::E8, + prefix_len..data.len(), + ))) + .finish() + .unwrap(); + let archive = Archive::parse(&archive).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.should_stream_decode(BUFFERED_DECODE_LIMIT)); + + // The member is valid and extracts in full when buffering is allowed. + let captured = Rc::new(RefCell::new(Vec::new())); + struct Capture(Rc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + archive + .extract_to( + crate::ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(data.len() as u64), + |_| Ok(Box::new(Capture(captured.clone()))), + ) + .unwrap(); + assert_eq!(*captured.borrow(), data); + + let mut out = Vec::new(); + let error = file.write_to(&archive, None, &mut out).unwrap_err(); + + assert!(matches!( + error, + Error::AtEntry { + operation: "decoding", + source, + .. + } if matches!(*source, Error::Rar50BufferedDecodeLimitExceeded { .. }) + )); + assert_eq!(out, data[..prefix_len]); + } + } + + #[test] + fn streaming_crc32_zero_advance_matches_byte_update() { + let mut bytewise = Crc32::new(); + bytewise.update(&vec![0; 100_000]); + + let mut skipped = Crc32::new(); + skipped.update_zeroes(100_000); + + assert_eq!(skipped.finish(), bytewise.finish()); + } + + #[test] + fn repeated_chunk_does_not_advance_crc_after_sink_error() { + struct FailingWriter; + + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("sink failed")) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let mut writer = FailingWriter; + let mut crc = Crc32::new(); + let expected = Crc32::new().finish(); + + assert!(write_repeated_chunk(&mut writer, &mut crc, &mut None, 0, 1024).is_err()); + assert_eq!(crc.finish(), expected); + } + + #[test] + fn encrypted_stored_decode_rejects_nonzero_discarded_padding() { + let mut file = plain_file(b"secret.txt", b"secret", None); + file.encrypted = true; + file.unpacked_size = 6; + let mut decoder = ReaderState::new(&Allowance::default()); + + assert_eq!( + file.decode_packed_with_decoder(b"secret\0\0", &mut decoder) + .unwrap(), + b"secret" + ); + assert!(matches!( + file.decode_packed_with_decoder(b"secret\0\x01", &mut decoder), + Err(Error::InvalidHeader( + "RAR 5 encrypted stored file has non-zero padding" + )) + )); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn unrepresentable_dictionary_is_refused_by_buffered_streaming_and_split_decoders() { + struct NeverRead; + impl Read for NeverRead { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + panic!("an unrepresentable dictionary must be rejected before reading") + } + } + let mut file = plain_file(b"large-dictionary", &[0], None); + file.compression_info = (5 << 7) | (15 << 10); + assert_eq!( + file.decoded_compression_info().unwrap().dictionary_size, + 1u64 << 32 + ); + let expected = Error::InvalidHeader("RAR 5 dictionary size overflows host address size"); + let mut decoder = ReaderState::new(&Allowance::default()); + assert_eq!( + file.decode_packed_owned(&[0], &mut decoder).unwrap_err(), + expected + ); + assert_eq!( + file.stream_packed_with_decoder( + &mut NeverRead, + None, + &mut decoder, + u64::MAX, + None, + &mut std::io::sink() + ) + .unwrap_err(), + expected + ); + + let mut first = file.clone(); + first.block.flags = HFL_SPLIT_AFTER; + first.block.data_range = 0..1; + first.block.data_size = Some(1); + let mut last = first.clone(); + last.block.flags = HFL_SPLIT_BEFORE; + let volumes = [ + archive_with_blocks(vec![Block::File(first.clone())], vec![0]), + archive_with_blocks(vec![Block::File(last.clone())], vec![0]), + ]; + let mut split = PendingSplitRefs::new(&first, 0, 0); + split.append(1, 0).unwrap(); + assert_eq!( + last.decode_split_with_decoder(&volumes, &split, &mut decoder, None) + .unwrap_err(), + expected + ); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn encrypted_stored_size_above_the_host_limit_is_refused() { + let mut file = plain_file(b"large-stored", &[], None); + file.encrypted = true; + file.encryption = Some(FileEncryption { + version: 0, + flags: 0, + kdf_count: 0, + salt: [0; 16], + iv: [0; 16], + check_value: None, + }); + file.unpacked_size = 1u64 << 32; + let mut decoder = ReaderState::new(&Allowance::default()); + assert_eq!( + file.decode_packed_owned(&[0; 16], &mut decoder) + .unwrap_err(), + Error::InvalidHeader("RAR 5 unpacked size overflows host address size") + ); + } + + #[test] + fn checked_unpacked_size_rejects_values_above_host_usize() { + assert_eq!(checked_unpacked_size(123).unwrap(), 123usize); + + let overflowing = usize::MAX as u128 + 1; + if overflowing <= u64::MAX as u128 { + assert!(checked_unpacked_size(overflowing as u64).is_err()); + } + } + + #[test] + fn constant_time_hash_comparison_keeps_hash_validation_behaviour() { + let data = b"hash me"; + let file = FileHeader { + block: empty_block(HEAD_FILE, 0, 0..0), + file_flags: 0, + rewrite_metadata_complete: true, + unpacked_size: data.len() as u64, + attributes: 0x20, + mtime: None, + htime_mtime: None, + htime_mtime_refinement: None, + file_times: None, + data_crc32: None, + compression_info: 0, + host_os: 2, + name: b"hash.txt".to_vec(), + hash: Some(FileHash { + hash_type: 0, + data: blake2sp::hash(data).to_vec(), + }), + redirection: None, + service_data: None, + encrypted: false, + encryption: None, + crypto: None, + }; + + file.verify_integrity_with_keys(data, None).unwrap(); + + let mut wrong = file; + wrong.hash.as_mut().unwrap().data[31] ^= 0x01; + assert!(matches!( + wrong.verify_integrity_with_keys(data, None), + Err(Error::HashMismatch { hash_type: 0 }) + )); + } + + #[test] + fn verify_integrity_rejects_bad_blake2sp_length_and_ignores_unknown_hash_type() { + let data = b"hash me"; + let mut bad_length = plain_file( + b"a.txt", + data, + Some(FileHash { + hash_type: 0, + data: vec![0u8; 16], + }), + ); + bad_length.data_crc32 = Some(crc32(data)); + assert!(matches!( + bad_length.verify_integrity_with_keys(data, None), + Err(Error::InvalidHeader(_)) + )); + + bad_length.hash.as_mut().unwrap().hash_type = 99; + bad_length.hash.as_mut().unwrap().data = vec![0u8; 32]; + bad_length.verify_integrity_with_keys(data, None).unwrap(); + bad_length.data_crc32 = Some(0); + assert!(matches!( + bad_length.verify_integrity_with_keys(data, None), + Err(Error::Crc32Mismatch { .. }) + )); + } + + #[test] + fn streaming_hash_verifier_rejects_bad_blake2sp_length_and_ignores_unknown_hash_type() { + let mut file = plain_file( + b"a.txt", + b"", + Some(FileHash { + hash_type: 0, + data: vec![0u8; 16], + }), + ); + assert!(matches!( + streaming_hash_verifier(&file), + Err(Error::InvalidHeader(_)) + )); + + file.hash.as_mut().unwrap().hash_type = 7; + file.hash.as_mut().unwrap().data = vec![0u8; 32]; + assert!(matches!(streaming_hash_verifier(&file), Ok(None))); + + let nohash = plain_file(b"a.txt", b"", None); + assert!(matches!(streaming_hash_verifier(&nohash), Ok(None))); + } + + #[test] + #[cfg(feature = "encryption")] + fn encryption_keys_reject_missing_password_record_and_unsupported_versions() { + let mut missing = plain_file(b"a.txt", b"", None); + missing.encrypted = true; + assert!(matches!( + missing.encryption_keys(None), + Err(Error::NeedPassword) + )); + assert!(matches!( + missing.encryption_keys(Some(b"pw")), + Err(Error::InvalidHeader(_)) + )); + + let mut bad_version = plain_file(b"a.txt", b"", None); + bad_version.encrypted = true; + bad_version.encryption = Some(FileEncryption { + version: 1, + flags: 0, + kdf_count: 0, + salt: [0u8; 16], + iv: [0u8; 16], + check_value: None, + }); + assert!(matches!( + bad_version.encryption_keys(Some(b"pw")), + Err(Error::UnsupportedFeature { .. }) + )); + } + + #[test] + #[cfg(feature = "encryption")] + fn encryption_keys_handles_missing_check_value() { + let mut file = plain_file(b"a.txt", b"", None); + file.encrypted = true; + file.encryption = Some(FileEncryption { + version: 0, + flags: 0, + kdf_count: 0, + salt: [0u8; 16], + iv: [0u8; 16], + check_value: None, + }); + file.encryption_keys(Some(b"pw")).unwrap(); + } + + #[test] + fn decode_packed_rejects_stored_size_mismatch() { + let mut decoder = ReaderState::new(&Allowance::default()); + + let mut file = plain_file(b"a.txt", &[0u8; 32], None); + file.unpacked_size = 32; + let short = vec![0u8; 16]; + assert!(matches!( + file.decode_packed_with_decoder(&short, &mut decoder), + Err(Error::InvalidHeader(_)) + )); + + let mut encrypted = plain_file(b"b.txt", &[0u8; 32], None); + encrypted.encrypted = true; + encrypted.unpacked_size = 32; + let too_short = vec![0u8; 16]; + assert!(matches!( + encrypted.decode_packed_with_decoder(&too_short, &mut decoder), + Err(Error::InvalidHeader(_)) + )); + + let exact = vec![0u8; 64]; + let trimmed = encrypted + .decode_packed_with_decoder(&exact, &mut decoder) + .unwrap(); + assert_eq!(trimmed.len(), encrypted.unpacked_size as usize); + } + + #[test] + fn extraction_rejects_stored_members_with_mismatched_packed_size() { + let data = b"payload"; + for unpacked_size in [data.len() as u64 - 1, data.len() as u64 + 1] { + let mut file = plain_file(b"stored.txt", data, None); + file.unpacked_size = unpacked_size; + file.block.data_range = 0..data.len(); + let archive = archive_with_blocks(vec![Block::File(file)], data.to_vec()); + + for parallel in [false, true] { + let open = |_: &ExtractedEntryMeta| Ok(Box::new(std::io::sink()) as Box); + let result = if parallel { + archive.extract_to_parallel_buffered(crate::ArchiveReadOptions::default(), open) + } else { + archive.extract_to(crate::ArchiveReadOptions::default(), open) + }; + let error = result.expect_err("stored member size mismatch must fail"); + assert!(matches!(error.root_cause(), Error::InvalidHeader(_))); + } + } + } + + #[test] + fn stored_extraction_rejects_file_source_truncated_after_header_read() { + let data = b"stored source can change"; + let mut file = plain_file(b"stored.txt", data, None); + file.block.data_range = 0..data.len(); + file.block.data_size = Some(data.len() as u64); + let scratch = crate::scratch::case("rar5-stored-truncated-source"); + let path = scratch.join("member.part"); + std::fs::write(&path, data).unwrap(); + let mut archive = archive_with_blocks(vec![Block::File(file)], Vec::new()); + archive.source = ArchiveSource::File(Arc::new(path.clone())); + std::fs::write(&path, &data[..data.len() - 3]).unwrap(); + + let captured = Rc::new(RefCell::new(Vec::new())); + struct Capture(Rc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let error = archive + .extract_to(crate::ArchiveReadOptions::default(), |_| { + Ok(Box::new(Capture(captured.clone()))) + }) + .unwrap_err(); + assert_eq!(*captured.borrow(), data[..data.len() - 3]); + assert!(matches!( + error.root_cause(), + Error::InvalidHeader("RAR 5 stored file has mismatched packed and unpacked sizes") + )); + } + + #[test] + fn verify_streaming_integrity_validates_crc_and_hash() { + let payload = b"streaming"; + let crc_value = crc32(payload); + let hash_value = blake2sp::hash(payload); + + let mut file = plain_file(b"s.txt", payload, None); + file.data_crc32 = Some(crc_value); + file.hash = Some(FileHash { + hash_type: 0, + data: hash_value.to_vec(), + }); + + let make_state = || { + let mut crc = Crc32::new(); + crc.update(payload); + let mut hasher = blake2sp::Hasher::new(); + hasher.update(payload); + (crc, Some((hash_value, hasher))) + }; + + let (crc, hash) = make_state(); + file.verify_streaming_integrity(crc, hash, None).unwrap(); + + // A wrong CRC32 alongside a valid BLAKE2sp is ignored, because the + // reference readers never evaluate it there. Drop the hash record and + // the same wrong CRC32 has to be caught. + let (crc, hash) = make_state(); + let mut bad = file.clone(); + bad.data_crc32 = Some(crc_value ^ 0x1); + bad.verify_streaming_integrity(crc, hash, None).unwrap(); + + let (crc, _) = make_state(); + let mut crc_only = bad.clone(); + crc_only.hash = None; + assert!(matches!( + crc_only.verify_streaming_integrity(crc, None, None), + Err(Error::Crc32Mismatch { .. }) + )); + + let (crc, _) = make_state(); + let mut wrong_expected = hash_value; + wrong_expected[0] ^= 0xff; + let mut hasher = blake2sp::Hasher::new(); + hasher.update(payload); + let mut bad_hash = file.clone(); + bad_hash.data_crc32 = None; + assert!(matches!( + bad_hash.verify_streaming_integrity(crc, Some((wrong_expected, hasher)), None), + Err(Error::HashMismatch { hash_type: 0 }) + )); + + let empty = plain_file(b"e.txt", b"", None); + empty + .verify_streaming_integrity(Crc32::new(), None, None) + .unwrap(); + } + + #[test] + fn write_repeated_chunk_updates_crc_hash_and_writer() { + let mut writer = Vec::new(); + let mut crc_zero = Crc32::new(); + let mut hash = Some(([0u8; 32], blake2sp::Hasher::new())); + write_repeated_chunk(&mut writer, &mut crc_zero, &mut hash, 0, 70_000).unwrap(); + assert_eq!(writer.len(), 70_000); + let zero_crc = crc_zero.finish(); + + let mut bytewise = Crc32::new(); + bytewise.update(&vec![0u8; 70_000]); + assert_eq!(zero_crc, bytewise.finish()); + + let mut writer = Vec::new(); + let mut crc_ff = Crc32::new(); + let mut hash_none: Option<([u8; 32], blake2sp::Hasher)> = None; + write_repeated_chunk(&mut writer, &mut crc_ff, &mut hash_none, 0xff, 1024).unwrap(); + assert_eq!(writer, vec![0xffu8; 1024]); + } + + #[test] + #[cfg(feature = "encryption")] + fn map_rar50_crypto_error_translates_kdf_count() { + assert!(matches!( + super::super::map_rar50_crypto_error(crate::crypto::rar50::Error::KdfCountTooLarge), + Error::UnsupportedFeature { .. } + )); + assert!(matches!( + super::super::map_rar50_crypto_error(crate::crypto::rar50::Error::BadPassword), + Error::WrongPasswordOrCorruptData + )); + } + + #[test] + fn constant_time_eq_returns_false_for_length_mismatch() { + assert!(!constant_time_eq(b"abc", b"abcd")); + assert!(constant_time_eq(b"abc", b"abc")); + assert!(!constant_time_eq(b"abc", b"abd")); + } + + fn stored_split_archive(data: &[u8], full: &[u8], crc: u32, flags: u64) -> Archive { + let source: Arc<[u8]> = Arc::from(data.to_vec().into_boxed_slice()); + Archive { + sfx_offset: 0, + main: MainHeader { + block: empty_block(1, 0, 0..0), + archive_flags: 0, + volume_number: None, + extras: Vec::new(), + encrypted_headers: false, + rewrite_metadata_complete: true, + }, + blocks: vec![Block::File(FileHeader { + block: empty_block(HEAD_FILE, flags, 0..data.len()), + file_flags: 0, + rewrite_metadata_complete: true, + unpacked_size: full.len() as u64, + attributes: 0x20, + mtime: None, + htime_mtime: None, + htime_mtime_refinement: None, + file_times: None, + data_crc32: Some(crc), + compression_info: 0, + host_os: 2, + name: b"split.txt".to_vec(), + hash: Some(FileHash { + hash_type: 0, + data: blake2sp::hash(full).to_vec(), + }), + redirection: None, + service_data: None, + encrypted: false, + encryption: None, + crypto: None, + })], + source: ArchiveSource::Memory(source), + } + } + + fn empty_block( + header_type: u64, + flags: u64, + data_range: std::ops::Range, + ) -> BlockHeader { + BlockHeader { + header_crc: 0, + header_size: 0, + header_type, + flags, + extra_area_size: None, + data_size: Some(data_range.len() as u64), + offset: 0, + header_range: 0..0, + data_range, + } + } + + fn split_fragment_file(name: &[u8], hfl_flags: u64) -> FileHeader { + FileHeader { + block: empty_block(HEAD_FILE, hfl_flags, 0..0), + file_flags: 0, + rewrite_metadata_complete: true, + unpacked_size: 0, + attributes: 0x20, + mtime: None, + htime_mtime: None, + htime_mtime_refinement: None, + file_times: None, + data_crc32: None, + compression_info: 0, + host_os: 2, + name: name.to_vec(), + hash: None, + redirection: None, + service_data: None, + encrypted: false, + encryption: None, + crypto: None, + } + } + + fn archive_with_blocks(blocks: Vec, source: Vec) -> Archive { + let bytes: Arc<[u8]> = Arc::from(source.into_boxed_slice()); + Archive { + sfx_offset: 0, + main: MainHeader { + block: empty_block(1, 0, 0..0), + archive_flags: 0, + volume_number: None, + extras: Vec::new(), + encrypted_headers: false, + rewrite_metadata_complete: true, + }, + blocks, + source: ArchiveSource::Memory(bytes), + } + } + + fn never_open(_meta: &ExtractedEntryMeta) -> Result> { + panic!("open should not be invoked for this test"); + } + + #[test] + fn extract_volumes_to_rejects_volume_state_violations() { + let empty: Vec = Vec::new(); + assert!(matches!( + extract_volumes_to(&empty, crate::ArchiveReadOptions::default(), never_open), + Err(Error::InvalidHeader(_)) + )); + + let only_continuation = vec![archive_with_blocks( + vec![Block::File(split_fragment_file(b"a.txt", HFL_SPLIT_BEFORE))], + Vec::new(), + )]; + assert!(matches!( + extract_volumes_to( + &only_continuation, + crate::ArchiveReadOptions::default(), + never_open, + ), + Err(Error::InvalidHeader(_)) + )); + + let interrupted = vec![archive_with_blocks( + vec![ + Block::File(split_fragment_file(b"a.txt", HFL_SPLIT_AFTER)), + Block::File(plain_file(b"other.txt", b"", None)), + ], + Vec::new(), + )]; + assert!(matches!( + extract_volumes_to( + &interrupted, + crate::ArchiveReadOptions::default(), + never_open, + ), + Err(Error::InvalidHeader(_)) + )); + + let incomplete = vec![archive_with_blocks( + vec![Block::File(split_fragment_file(b"a.txt", HFL_SPLIT_AFTER))], + Vec::new(), + )]; + assert!(matches!( + extract_volumes_to( + &incomplete, + crate::ArchiveReadOptions::default(), + never_open, + ), + Err(Error::InvalidHeader(_)) + )); + } + + #[test] + fn validate_split_fragment_rejects_directories_and_demands_password_for_encrypted() { + let mut dir = split_fragment_file(b"d", HFL_SPLIT_AFTER); + dir.file_flags = 0x0001; + assert!(matches!( + validate_split_fragment(&dir, None), + Err(Error::InvalidHeader(_)) + )); + + let mut encrypted = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + encrypted.encrypted = true; + #[cfg(feature = "encryption")] + { + assert!(matches!( + validate_split_fragment(&encrypted, None), + Err(Error::NeedPassword) + )); + validate_split_fragment(&encrypted, Some(b"pw")).unwrap(); + } + #[cfg(not(feature = "encryption"))] + for password in [None, Some(b"pw".as_slice())] { + assert!(matches!( + validate_split_fragment(&encrypted, password), + Err(Error::FeatureDisabled { + feature: "encryption" + }) + )); + } + + let plain = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + validate_split_fragment(&plain, None).unwrap(); + } + + #[test] + fn validate_split_continuation_refs_rejects_property_drift_between_fragments() { + let first = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + let pending = PendingSplitRefs::new(&first, 0, 0); + + let renamed = split_fragment_file(b"b.txt", HFL_SPLIT_BEFORE); + assert!(matches!( + validate_split_continuation_refs(&pending, &renamed, None), + Err(Error::InvalidHeader(_)) + )); + + let mut new_compression = split_fragment_file(b"a.txt", HFL_SPLIT_BEFORE); + new_compression.compression_info = 0x123; + assert!(matches!( + validate_split_continuation_refs(&pending, &new_compression, None), + Err(Error::InvalidHeader(_)) + )); + + let mut new_encryption = split_fragment_file(b"a.txt", HFL_SPLIT_BEFORE); + new_encryption.encrypted = true; + #[cfg(feature = "encryption")] + assert!(matches!( + validate_split_continuation_refs(&pending, &new_encryption, Some(b"pw")), + Err(Error::InvalidHeader(_)) + )); + #[cfg(not(feature = "encryption"))] + assert!(matches!( + validate_split_continuation_refs(&pending, &new_encryption, Some(b"pw")), + Err(Error::FeatureDisabled { + feature: "encryption" + }) + )); + + let same = split_fragment_file(b"a.txt", HFL_SPLIT_BEFORE); + validate_split_continuation_refs(&pending, &same, None).unwrap(); + } + + #[test] + fn extraction_modes_handle_directories_links_and_single_volume_splits() { + let mut directory = plain_file(b"dir", b"", None); + directory.file_flags = 1; + directory.compression_info = 5 << 7; + let mut link = plain_file(b"link", b"", None); + link.compression_info = 5 << 7; + link.redirection = Some(super::super::FileRedirection { + redirection_type: 1, + flags: 0, + target_name: b"dir".to_vec(), + }); + let archive = archive_with_blocks(vec![Block::File(directory), Block::File(link)], vec![]); + for parallel in [false, true] { + let mut opened = vec![]; + let options = crate::ArchiveReadOptions::new() + .with_rar50_dictionary_size_limit(0) + .with_max_member_output_bytes(0); + let mut open = |meta: &ExtractedEntryMeta| { + opened.push(meta.name.clone()); + assert!(meta.is_directory); + Ok(Box::new(std::io::sink()) as Box) + }; + if parallel { + archive.extract_to_parallel_buffered(options, &mut open) + } else { + extract_volumes_to(std::slice::from_ref(&archive), options, &mut open) + } + .unwrap(); + assert_eq!(opened, [b"dir".to_vec()]); + } + let mut select = |_: &crate::ArchiveMember| { + Ok(crate::ExtractionDecision::Extract( + Box::new(std::io::sink()), + )) + }; + let error = archive + .extract_controlled(crate::ArchiveReadOptions::new(), &mut select, None) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::UnsupportedFeature); + assert_eq!(error.entry_context().unwrap().0, b"link"); + + for flags in [HFL_SPLIT_BEFORE, HFL_SPLIT_AFTER] { + let archive = archive_with_blocks( + vec![Block::File(split_fragment_file(b"split", flags))], + vec![], + ); + let error = archive + .extract_to_parallel_buffered(crate::ArchiveReadOptions::new(), never_open) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::InvalidHeader("RAR 5 split entry requires multivolume extraction") + )); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_stored_split_checks_the_final_crc_mac() { + let data = b"Hello, RAR 5.0 fixture world.\n"; + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ) + .entry(entry(b"hello.txt", data).with_password(b"password".to_vec())) + .finish() + .unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let mut original = archive.files().next().unwrap().clone(); + assert!(original.uses_hash_mac()); + assert!(original.is_stored()); + // Exercise the CRC-only layout as well as the writer's default hash + // layout. The CRC is already MACed independently of its BLAKE2sp field. + original.hash = None; + let split = original.block.data_range.start + 13; + let mut first = original.clone(); + first.block.flags |= HFL_SPLIT_AFTER; + first.block.data_range.end = split; + first.block.data_size = Some(13); + first.data_crc32 = Some(crc32(&first.packed_data(&archive).unwrap())); + let mut last = original; + last.block.flags |= HFL_SPLIT_BEFORE; + last.block.data_range.start = split; + last.block.data_size = Some(last.block.data_range.len() as u64); + let mut volume1 = archive.clone(); + volume1.blocks = vec![Block::File(first)]; + let mut volume2 = archive; + volume2.blocks = vec![Block::File(last)]; + let captured = Rc::new(RefCell::new(Vec::new())); + struct Capture(Rc>>); + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut volumes = [volume1, volume2]; + let options = crate::ArchiveReadOptions::with_password(b"password"); + extract_volumes_to(&volumes, options, |_| { + Ok(Box::new(Capture(captured.clone()))) + }) + .unwrap(); + assert_eq!( + captured.borrow().as_slice(), + b"Hello, RAR 5.0 fixture world.\n" + ); + let Block::File(last) = &mut volumes[1].blocks[0] else { + unreachable!() + }; + last.data_crc32 = Some(last.data_crc32.unwrap() ^ 1); + let error = + extract_volumes_to(&volumes, options, |_| Ok(Box::new(std::io::sink()))).unwrap_err(); + assert!(matches!(error.root_cause(), Error::Crc32Mismatch { .. })); + } + + #[test] + fn split_decode_failures_use_fragment_diagnostics_only_when_available() { + for crc in [None, Some(0), Some(1)] { + let mut first = split_fragment_file(b"truncated", HFL_SPLIT_AFTER); + first.compression_info = 5 << 7; + first.unpacked_size = 16; + first.data_crc32 = crc; + let mut last = first.clone(); + last.block.flags = HFL_SPLIT_BEFORE; + last.data_crc32 = None; + let volumes = [ + archive_with_blocks(vec![Block::File(first)], vec![]), + archive_with_blocks(vec![Block::File(last)], vec![]), + ]; + for limit in [0, 1024] { + let error = extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(limit), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap_err(); + if crc == Some(1) { + assert!(matches!( + error.root_cause(), + Error::Crc32Mismatch { + expected: 1, + actual: 0 + } + )); + } else { + assert!(matches!( + error.root_cause(), + Error::Codec(crate::codec::Error::NeedMoreInput) + )); + } + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_buffered_crc_verification_requires_keys_and_checks_mac() { + let data = b"MAC checked payload"; + let keys = Rar50Keys::derive(b"pw", [3; 16], 0).unwrap(); + let mut file = plain_file(b"encrypted", data, None); + file.encrypted = true; + file.encryption = Some(FileEncryption { + version: 0, + flags: 2, + kdf_count: 0, + salt: [3; 16], + iv: [4; 16], + check_value: None, + }); + file.data_crc32 = Some(keys.mac_crc32(crc32(data))); + assert!(matches!( + file.verify_integrity_with_keys(data, None), + Err(Error::InvalidHeader(_)) + )); + file.verify_integrity_with_keys(data, Some(&keys)).unwrap(); + assert!(matches!( + file.verify_integrity_with_keys(b"damaged", Some(&keys)), + Err(Error::Crc32Mismatch { .. }) + )); + file.block.data_range = 0..data.len(); + file.block.data_size = Some(data.len() as u64); + let archive = archive_with_blocks(vec![], data.to_vec()); + assert!(matches!( + file.packed_reader_with_password(&archive, Some(b"pw")), + Err(Error::InvalidHeader( + "RAR 5 encrypted file payload is not block aligned" + )) + )); + } + + #[test] + fn archive_extract_to_rejects_split_entries_in_single_volume_archive() { + let split = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + let archive = archive_with_blocks(vec![Block::File(split)], Vec::new()); + let err = archive + .extract_to(crate::ArchiveReadOptions::default(), never_open) + .unwrap_err(); + assert!( + matches!(err, Error::InvalidHeader(msg) if msg.contains("requires multivolume")), + "expected multivolume error, got {err:?}" + ); + } + + #[test] + fn archive_extract_to_skips_redirection_entries_without_opening_writer() { + let mut redirect = plain_file(b"link", b"", None); + redirect.redirection = Some(super::super::FileRedirection { + redirection_type: 1, + flags: 0, + target_name: b"target".to_vec(), + }); + let archive = archive_with_blocks(vec![Block::File(redirect)], Vec::new()); + archive + .extract_to( + crate::ArchiveReadOptions::default().with_rar50_dictionary_size_limit(0), + never_open, + ) + .unwrap(); + } + + #[test] + fn archive_extract_to_with_redirections_reports_redirection_entries() { + let mut redirect = plain_file(b"link", b"", None); + redirect.redirection = Some(super::super::FileRedirection { + redirection_type: 1, + flags: 0, + target_name: b"target".to_vec(), + }); + let archive = archive_with_blocks(vec![Block::File(redirect)], Vec::new()); + let mut seen = Vec::new(); + archive + .extract_to_with_redirections( + crate::ArchiveReadOptions::default(), + never_open, + |meta, redirection| { + seen.push((meta.name.clone(), redirection.target_name.clone())); + Ok(()) + }, + ) + .unwrap(); + + assert_eq!(seen, vec![(b"link".to_vec(), b"target".to_vec())]); + } + + #[test] + fn zero_total_allows_redirection_callbacks_in_regular_and_volume_extraction() { + let mut redirect = plain_file(b"link", b"", None); + // A metadata-only redirection's size placeholder must not consume quota + // or trigger the unknown logical file-size policy. + redirect.unpacked_size = u64::MAX; + redirect.file_flags |= 0x8; + redirect.redirection = Some(super::super::FileRedirection { + redirection_type: 1, + flags: 0, + target_name: b"target".to_vec(), + }); + let archive = archive_with_blocks(vec![Block::File(redirect)], Vec::new()); + let options = crate::ArchiveReadOptions::new().with_max_total_output_bytes(0); + for volumes in [false, true] { + let mut seen = 0; + let mut redirect = |_: &ExtractedEntryMeta, _: &FileRedirection| { + seen += 1; + Ok(()) + }; + if volumes { + extract_volumes_to_with_redirections( + std::slice::from_ref(&archive), + options, + never_open, + &mut redirect, + ) + .unwrap(); + } else { + archive + .extract_to_with_redirections(options, never_open, &mut redirect) + .unwrap(); + } + assert_eq!(seen, 1); + } + } + + #[test] + fn extract_volumes_to_skips_redirection_entries_without_opening_writer() { + let mut redirect = plain_file(b"link", b"", None); + redirect.redirection = Some(super::super::FileRedirection { + redirection_type: 1, + flags: 0, + target_name: b"target".to_vec(), + }); + let volumes = vec![archive_with_blocks(vec![Block::File(redirect)], Vec::new())]; + extract_volumes_to( + &volumes, + crate::ArchiveReadOptions::default().with_rar50_dictionary_size_limit(0), + never_open, + ) + .unwrap(); + } + + #[test] + fn extract_volumes_to_with_redirections_reports_redirection_entries() { + let mut redirect = plain_file(b"link", b"", None); + redirect.redirection = Some(super::super::FileRedirection { + redirection_type: 5, + flags: 0, + target_name: b"target".to_vec(), + }); + let volumes = vec![archive_with_blocks(vec![Block::File(redirect)], Vec::new())]; + let mut seen = Vec::new(); + extract_volumes_to_with_redirections( + &volumes, + crate::ArchiveReadOptions::default(), + never_open, + |meta, redirection| { + seen.push((meta.name.clone(), redirection.target_name.clone())); + Ok(()) + }, + ) + .unwrap(); + + assert_eq!(seen, vec![(b"link".to_vec(), b"target".to_vec())]); + } + + #[test] + fn pending_split_refs_write_stored_to_rejects_unpacked_size_mismatch() { + let payload: &[u8] = b"unmatched-size payload"; + let mut first = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + first.block.data_range = 0..payload.len(); + first.block.data_size = Some(payload.len() as u64); + first.unpacked_size = (payload.len() + 5) as u64; // mismatch + let final_file = first.clone(); + let pending = PendingSplitRefs::new(&first, 0, 0); + let volumes = vec![archive_with_blocks( + vec![Block::File(first)], + payload.to_vec(), + )]; + + let mut out: Vec = Vec::new(); + let err = pending + .write_stored_to(&volumes, &final_file, None, &mut out) + .unwrap_err(); + assert!( + matches!(err, Error::InvalidHeader(msg) if msg.contains("mismatched packed and unpacked")), + "expected size mismatch error, got {err:?}" + ); + } + + #[test] + fn pending_split_refs_write_stored_to_rejects_crc_mismatch_on_unencrypted() { + let payload: &[u8] = b"crc-mismatch payload"; + let mut first = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + first.block.data_range = 0..payload.len(); + first.block.data_size = Some(payload.len() as u64); + first.unpacked_size = payload.len() as u64; + first.data_crc32 = Some(crc32(payload).wrapping_add(1)); + let final_file = first.clone(); + let pending = PendingSplitRefs::new(&first, 0, 0); + let volumes = vec![archive_with_blocks( + vec![Block::File(first)], + payload.to_vec(), + )]; + + let mut out: Vec = Vec::new(); + let err = pending + .write_stored_to(&volumes, &final_file, None, &mut out) + .unwrap_err(); + assert!( + matches!(err, Error::Crc32Mismatch { .. }), + "expected CRC mismatch, got {err:?}" + ); + } + + #[test] + fn pending_split_refs_write_stored_to_rejects_hash_mismatch_on_unencrypted() { + let payload: &[u8] = b"hash-mismatch payload"; + let mut wrong_hash = blake2sp::hash(payload); + wrong_hash[0] ^= 0xff; + + let mut first = split_fragment_file(b"a.txt", HFL_SPLIT_AFTER); + first.block.data_range = 0..payload.len(); + first.block.data_size = Some(payload.len() as u64); + first.unpacked_size = payload.len() as u64; + first.data_crc32 = Some(crc32(payload)); + first.hash = Some(FileHash { + hash_type: 0, + data: wrong_hash.to_vec(), + }); + let final_file = first.clone(); + let pending = PendingSplitRefs::new(&first, 0, 0); + let volumes = vec![archive_with_blocks( + vec![Block::File(first)], + payload.to_vec(), + )]; + + let mut out: Vec = Vec::new(); + let err = pending + .write_stored_to(&volumes, &final_file, None, &mut out) + .unwrap_err(); + assert!( + matches!(err, Error::HashMismatch { hash_type: 0 }), + "expected hash mismatch, got {err:?}" + ); + } + + #[test] + fn decoded_data_unverified_returns_stored_payload_without_crc_check() { + let payload = b"decoded_data_unverified stored payload"; + let mut file = plain_file(b"a.txt", payload, None); + file.block.data_range = 0..payload.len(); + file.block.data_size = Some(payload.len() as u64); + file.unpacked_size = payload.len() as u64; + // Set wrong CRC — unverified path must not check it. + file.data_crc32 = Some(crc32(payload).wrapping_add(1)); + + let archive = archive_with_blocks(vec![Block::File(file.clone())], payload.to_vec()); + let decoded = file.decoded_data_unverified(&archive, None).unwrap(); + assert_eq!(decoded, payload); + } + + #[test] + fn decoded_data_unverified_accepts_empty_compressed_member() { + for crc in [None, Some(0)] { + let mut file = plain_file(b"empty.txt", b"", None); + file.compression_info = 5 << 7; + file.data_crc32 = crc; + let archive = archive_with_blocks(vec![Block::File(file.clone())], Vec::new()); + assert!(file + .decoded_data_unverified(&archive, None) + .unwrap() + .is_empty()); + for parallel in [false, true] { + let open = |_: &ExtractedEntryMeta| Ok(Box::new(std::io::sink()) as Box); + let result = if parallel { + archive.extract_to_parallel_buffered(crate::ArchiveReadOptions::default(), open) + } else { + archive.extract_to(crate::ArchiveReadOptions::default(), open) + }; + result.unwrap(); + } + } + // An empty logical member with a present stream must still validate + // that stream rather than taking the omitted-stream shortcut. + let mut file = plain_file(b"empty-with-stream.txt", b"", None); + file.compression_info = 5 << 7; + file.block.data_range = 0..1; + file.block.data_size = Some(1); + let archive = archive_with_blocks(vec![Block::File(file.clone())], vec![0]); + assert!(matches!( + file.decoded_data_unverified(&archive, None), + Err(Error::Codec(_)) + )); + } + + #[test] + fn dictionary_limit_checks_declared_sizes_before_opening_output() { + for info in [5 << 7, (5 << 7) | 1 | (3 << 15), (5 << 7) | (15 << 10)] { + let mut file = plain_file(b"limited", b"", None); + file.compression_info = info; + let required = file.decoded_compression_info().unwrap().dictionary_size; + for solid in [false, true] { + file.compression_info = info | if solid { 0x40 } else { 0 }; + let archive = archive_with_blocks(vec![Block::File(file.clone())], Vec::new()); + for limit in [None, Some(required), Some(required - 1), Some(0)] { + for parallel in [false, true] { + let options = crate::ArchiveReadOptions { + rar50_dictionary_size_limit: limit, + ..Default::default() + }; + let mut opened = false; + let open = |_: &ExtractedEntryMeta| { + opened = true; + Ok(Box::new(std::io::sink()) as Box) + }; + let result = if parallel { + archive.extract_to_parallel_buffered(options, open) + } else { + archive.extract_to(options, open) + }; + if let Some(limit) = limit.filter(|limit| *limit < required) { + let error = result.unwrap_err(); + assert!(!opened); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert!( + matches!(error.root_cause(), Error::Rar50DictionaryLimitExceeded { + limit: actual, required: size + } if *actual == limit && *size == required) + ); + assert_eq!(error.entry_context().unwrap().0, b"limited"); + } else { + result.unwrap(); + assert!(opened); + } + } + } + } + } + } + + #[test] + fn truncated_checksumless_members_fail_in_buffered_and_streaming_extraction() { + // A compressed member advertises output, but its packed stream is + // truncated to nothing. Without integrity records, decoding itself + // must still establish success; an empty result is not a substitute. + let mut file = plain_file(b"truncated", b"", None); + file.compression_info = 5 << 7; + file.unpacked_size = 16; + file.data_crc32 = None; + file.hash = None; + let archive = archive_with_blocks(vec![Block::File(file)], Vec::new()); + for limit in [0, 1024] { + let options = + crate::ArchiveReadOptions::default().with_rar50_buffered_decode_limit(limit); + for parallel in [false, true] { + let open = |_: &ExtractedEntryMeta| Ok(Box::new(std::io::sink()) as Box); + let result = if parallel { + archive.extract_to_parallel_buffered(options, open) + } else { + archive.extract_to(options, open) + }; + let error = result.expect_err("truncated member must not succeed"); + assert!(matches!( + error.root_cause(), + Error::Codec(crate::codec::Error::NeedMoreInput) + )); + } + } + } + + #[test] + fn encryption_iv_falls_back_to_encryption_record_and_errors_when_missing() { + let mut with_record = plain_file(b"a.txt", b"", None); + with_record.encrypted = true; + with_record.encryption = Some(FileEncryption { + version: 0, + flags: 0, + kdf_count: 0, + salt: [0u8; 16], + iv: [5u8; 16], + check_value: None, + }); + assert_eq!(with_record.encryption_iv().unwrap(), [5u8; 16]); + + let missing = plain_file(b"a.txt", b"", None); + assert!(matches!( + missing.encryption_iv(), + Err(Error::InvalidHeader(_)) + )); + } +} + +#[cfg(all(test, feature = "write"))] +#[path = "independent_history_tests.rs"] +mod independent_history_tests; diff --git a/arca-rar/vendor/rars/src/rar50/framing.rs b/arca-rar/vendor/rars/src/rar50/framing.rs new file mode 100644 index 00000000..a91ce9b2 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/framing.rs @@ -0,0 +1,173 @@ +//! Header framing shared by writers and recovery repair. + +#[cfg(feature = "encryption")] +use super::map_rar50_crypto_error; +use super::HFL_EXTRA; +#[cfg(feature = "encryption")] +use crate::crypto::rar50::Rar50Cipher; +use crate::crypto::rar50::Rar50Keys; +use crate::{crc32::crc32, Error, Result}; + +/// Bounded framing scratch; callers choose capacities from the on-disk fields. +pub(crate) struct HeaderScratch { + bytes: [u8; N], + len: usize, +} + +impl HeaderScratch { + pub(crate) fn new() -> Self { + Self { + bytes: [0; N], + len: 0, + } + } + pub(crate) fn extend_from_slice(&mut self, bytes: &[u8]) { + self.bytes[self.len..self.len + bytes.len()].copy_from_slice(bytes); + self.len += bytes.len(); + } + pub(crate) fn vint(&mut self, mut value: u64) { + loop { + self.extend_from_slice(&[(value as u8 & 0x7f) | if value >= 0x80 { 0x80 } else { 0 }]); + value >>= 7; + if value == 0 { + break; + } + } + } + pub(crate) fn as_slice(&self) -> &[u8] { + &self.bytes[..self.len] + } + pub(crate) fn len(&self) -> usize { + self.len + } +} + +impl std::ops::Deref for HeaderScratch { + type Target = [u8]; + fn deref(&self) -> &[u8] { + self.as_slice() + } +} + +pub(crate) fn image_size_error() -> Error { + Error::InvalidArgument("RAR 5 header size overflows") +} + +pub(crate) fn checked_image_len(parts: &[usize]) -> Result { + let len = parts + .iter() + .try_fold(0usize, |total, &size| total.checked_add(size)) + .ok_or_else(image_size_error)?; + if len > isize::MAX as usize { + return Err(image_size_error()); + } + Ok(len) +} + +/// Computes framing without copying variable-length fields. All header paths +/// render into their final allocation, including IV, padding and trailing data. +pub(crate) struct HeaderImage<'a> { + prefix: HeaderScratch<40>, + size: HeaderScratch<10>, + specific: &'a [u8], + extra: &'a [u8], + plain_len: usize, +} + +impl<'a> HeaderImage<'a> { + pub(crate) fn new( + kind: u64, + flags: u64, + data_size: Option, + specific: &'a [u8], + extra: &'a [u8], + ) -> Result { + let mut prefix = HeaderScratch::new(); + prefix.vint(kind); + prefix.vint(flags); + if flags & HFL_EXTRA != 0 { + prefix.vint(extra.len() as u64); + } + if let Some(data_size) = data_size { + prefix.vint(data_size); + } + let body_len = checked_image_len(&[prefix.len(), specific.len(), extra.len()])?; + let mut size = HeaderScratch::new(); + size.vint(body_len as u64); + let plain_len = checked_image_len(&[4, size.len(), body_len])?; + Ok(Self { + prefix, + size, + specific, + extra, + plain_len, + }) + } + + pub(crate) fn header_len(&self, encrypted: bool) -> Result { + if encrypted { + let padded = self + .plain_len + .checked_add(15) + .ok_or_else(image_size_error)? + & !15; + checked_image_len(&[16, padded]) + } else { + Ok(self.plain_len) + } + } + + pub(crate) fn image_len(&self, encrypted: bool, data_len: usize) -> Result { + checked_image_len(&[self.header_len(encrypted)?, data_len]) + } + + /// Render into a zeroed buffer of exactly `image_len` bytes. Allocation and + /// admission belong to the caller; salts, framing and encryption are shared. + pub(crate) fn render_into( + &self, + keys: Option<&Rar50Keys>, + data: &[u8], + out: &mut [u8], + ) -> Result<()> { + if keys.is_some() { + crate::crypto::require_encryption()?; + } + let header_len = self.header_len(keys.is_some())?; + let start = if keys.is_some() { 16 } else { 0 }; + let mut offset = start + 4; + for part in [ + self.size.as_slice(), + self.prefix.as_slice(), + self.specific, + self.extra, + ] { + out[offset..offset + part.len()].copy_from_slice(part); + offset += part.len(); + } + let crc = crc32(&out[start + 4..start + self.plain_len]); + out[start..start + 4].copy_from_slice(&crc.to_le_bytes()); + if let Some(keys) = keys { + #[cfg(not(feature = "encryption"))] + { + let _ = keys; + return Err(Error::FeatureDisabled { + feature: "encryption", + }); + } + #[cfg(feature = "encryption")] + { + let mut iv = [0; 16]; + crate::entropy::fill_entropy( + &mut iv, + "RAR 5 writer could not generate encryption IV", + )?; + out[..16].copy_from_slice(&iv); + Rar50Cipher::new(keys.key, iv) + .encrypt_in_place(&mut out[16..header_len]) + .map_err(map_rar50_crypto_error)?; + } + } + out[header_len..].copy_from_slice(data); + Ok(()) + } +} diff --git a/arca-rar/vendor/rars/src/rar50/independent_history_tests.rs b/arca-rar/vendor/rars/src/rar50/independent_history_tests.rs new file mode 100644 index 00000000..8a9e3735 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/independent_history_tests.rs @@ -0,0 +1,374 @@ +use super::super::{ArchiveEntry, Block, FilterKind, FilterPolicy, Rar50Writer, WriterOptions}; +use super::*; +use crate::{ArchiveVersion, FeatureSet}; +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; + +type ExtractedFiles = Vec<(Vec, Vec)>; + +fn entry(name: &[u8], data: &[u8]) -> ArchiveEntry { + ArchiveEntry::new( + name.to_vec(), + crate::EntrySource::from_bytes(Arc::<[u8]>::from(data.to_vec())), + ) +} +#[derive(Clone)] +struct Sink(Rc>>); +impl Write for Sink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(data); + Ok(data.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} +fn archive(version: ArchiveVersion, filter: Option) -> (Archive, Vec) { + let data = b"independent buffered dictionary filter member repeated bytes\n".repeat(64); + let mut writer = Rar50Writer::new( + WriterOptions::new(version, FeatureSet::store_only()) + .with_compression_level(1) + .with_dictionary_size(if version == ArchiveVersion::Rar70 { + 192 * 1024 + } else { + 128 * 1024 + }), + ) + .entries([entry(b"first", &data), entry(b"second", &data)]); + if let Some(filter) = filter { + writer = writer.filter_policy(FilterPolicy::Explicit(crate::FilterSpec::whole(filter))); + } + let bytes = writer.finish().unwrap(); + let archive = Archive::parse_owned(bytes).unwrap(); + assert!(!archive.main.is_solid()); + assert!(archive.files().all(|file| !file.is_stored() + && !file.is_split_before() + && !file.is_split_after() + && !file.decoded_compression_info().unwrap().solid)); + assert!(archive.files().all(|file| { + file.decoded_compression_info().unwrap().algorithm_version + == u8::from(version == ArchiveVersion::Rar70) + })); + assert!(archive.files().all(|file| file + .hash + .as_ref() + .is_some_and(|hash| hash.hash_type == 0 && hash.data.len() == 32))); + (archive, data) +} +fn extract( + archive: &Archive, + parallel: bool, + options: crate::ArchiveReadOptions<'_>, +) -> (std::result::Result<(), String>, ExtractedFiles) { + let mut writers = Vec::new(); + let mut open = |meta: &ExtractedEntryMeta| { + let data = Rc::new(RefCell::new(Vec::new())); + writers.push((meta.name.clone(), data.clone())); + Ok(Box::new(Sink(data)) as Box) + }; + let result = if parallel { + archive.extract_to_parallel_buffered(options, &mut open) + } else { + archive.extract_to(options, &mut open) + }; + ( + result.map_err(|error| format!("{error:?}")), + writers + .into_iter() + .map(|(name, data)| (name, data.borrow().clone())) + .collect(), + ) +} +#[test] +fn independent_history_public_extraction_keeps_all_filter_bytes_and_bounded_routes() { + for version in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for filter in [ + None, + Some(FilterKind::Delta { channels: 2 }), + Some(FilterKind::E8), + Some(FilterKind::E8E9), + Some(FilterKind::Arm), + ] { + let (archive, data) = archive(version, filter); + for limit in [None, Some(64 * 1024 * 1024)] { + let mut options = + crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(u64::MAX); + options.max_reader_workspace_bytes = limit; + let expected = extract(&archive, false, options); + let actual = extract(&archive, true, options); + assert_eq!(actual, expected); + assert!(actual.0.is_ok()); + assert_eq!( + actual.1, + [ + (b"first".to_vec(), data.clone()), + (b"second".to_vec(), data.clone()) + ] + ); + } + } + } +} +#[test] +fn independent_history_verification_keeps_only_the_ordered_verified_prefix() { + for filter in [None, Some(FilterKind::Delta { channels: 2 })] { + for hash_failure in [false, true] { + let (mut archive, data) = archive(ArchiveVersion::Rar50, filter); + for block in &mut archive.blocks { + if let Block::File(file) = block { + if file.name == b"second" { + if hash_failure { + file.hash.as_mut().unwrap().data[0] ^= 1; + } else { + file.hash = None; + file.data_crc32 = Some(0); + } + } + } + } + let options = + crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(u64::MAX); + let expected = extract(&archive, false, options); + let actual = extract(&archive, true, options); + assert_eq!(actual.0, expected.0); + assert!(actual.0.as_ref().unwrap_err().contains(if hash_failure { + "HashMismatch" + } else { + "Crc32Mismatch" + })); + assert_eq!(actual.1, [(b"first".to_vec(), data.clone())]); + // Existing sequential direct sinks open before verification; + // independent publication opens only the verified prefix. + assert_eq!( + expected.1, + [(b"first".to_vec(), data), (b"second".to_vec(), Vec::new())] + ); + } + } +} + +#[cfg(feature = "encryption")] +#[test] +fn independent_history_encrypted_digest_and_password_errors_match_stateful_decode() { + let data = b"encrypted independent Delta and BLAKE2sp MAC bytes\n".repeat(64); + for filter in [None, Some(FilterKind::Delta { channels: 2 })] { + let mut writer = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + ) + .entry(entry(b"encrypted", &data).with_password(b"password".to_vec())); + if let Some(filter) = filter { + writer = writer.filter_policy(FilterPolicy::Explicit(crate::FilterSpec::whole(filter))); + } + let bytes = writer.finish().unwrap(); + let archive = Archive::parse_owned(bytes).unwrap(); + assert!(archive + .files() + .all(|file| file.encrypted && !file.is_stored() && file.hash.is_some())); + for password in [ + None, + Some(b"wrong".as_slice()), + Some(b"password".as_slice()), + ] { + let mut options = + crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(u64::MAX); + options.password = password; + let expected = extract(&archive, false, options); + let actual = extract(&archive, true, options); + assert_eq!(actual.0, expected.0); + if password == Some(b"password".as_slice()) { + assert!(actual.0.is_ok()); + assert_eq!(actual.1, expected.1); + assert_eq!(actual.1, [(b"encrypted".to_vec(), data.clone())]); + } else { + assert!(actual.0.is_err()); + assert!(actual.1.is_empty()); + } + } + } +} + +fn entry_result( + value: Result>, +) -> std::result::Result<(String, Vec), String> { + value + .map(|entry| match entry { + ParallelExtractedEntry::File { meta, data } => (format!("{meta:?}"), data.to_vec()), + ParallelExtractedEntry::Directory(meta) | ParallelExtractedEntry::Verified(meta) => { + (format!("{meta:?}"), Vec::new()) + } + ParallelExtractedEntry::Redirection { meta, redirection } => { + (format!("{meta:?} {redirection:?}"), Vec::new()) + } + }) + .map_err(|error| format!("{error:?}")) +} +#[test] +fn independent_history_worker_limited_keeps_original_attempts_callbacks_and_errors() { + use crate::codec::workspace::RefusingBudget; + for filter in [None, Some(FilterKind::Delta { channels: 2 })] { + let (archive, _) = archive(ArchiveVersion::Rar50, filter); + let file = archive.files().next().unwrap(); + let options = crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(u64::MAX); + let baseline = RefusingBudget::new(usize::MAX); + drop( + reference_decode_parallel_entry(&archive, file, None, u64::MAX, options, &baseline) + .unwrap(), + ); + assert_eq!(baseline.used(), 0); + for fail_at in 0..=baseline.attempts() { + for cancelled in [false, true] { + let old = RefusingBudget::new(fail_at); + let actual = RefusingBudget::new(fail_at); + let old_token = crate::ReadCancellation::new(); + let actual_token = crate::ReadCancellation::new(); + // Use identical upfront cancellation, plus the decoder-level + // checkpoint controls in codec/history_retention_tests.rs. + if cancelled { + old_token.cancel(); + actual_token.cancel(); + } + let mut old_options = options; + old_options.cancellation = Some(&old_token); + let mut actual_options = options; + actual_options.cancellation = Some(&actual_token); + let expected = entry_result(reference_decode_parallel_entry( + &archive, + file, + None, + u64::MAX, + old_options, + &old, + )); + let output = entry_result(decode_parallel_entry_with_allowance( + &archive, + file, + None, + u64::MAX, + actual_options, + false, + &actual, + )); + assert_eq!(output, expected, "filter={filter:?}, fail_at={fail_at}"); + assert_eq!(actual.attempts(), old.attempts()); + assert_eq!(actual.used(), 0); + assert_eq!(old.used(), 0); + assert_eq!(actual_token.is_cancelled(), old_token.is_cancelled()); + } + } + } +} + +#[test] +fn independent_history_public_stream_and_scratch_fallbacks_keep_existing_results() { + let dir = crate::scratch::case("independent-history-fallbacks"); + let scratch = crate::Rar50Scratch::new(&*dir, 64 * 1024); + for filter in [None, Some(FilterKind::Delta { channels: 2 })] { + let (archive, data) = archive(ArchiveVersion::Rar50, filter); + for use_scratch in [false, true] { + let mut options = crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(1); + if use_scratch { + options.rar50_scratch = Some(&scratch); + } + let expected = extract(&archive, false, options); + let actual = extract(&archive, true, options); + assert_eq!(actual, expected); + if filter.is_some() && !use_scratch { + assert!(actual + .0 + .as_ref() + .unwrap_err() + .contains("Rar50BufferedDecodeLimitExceeded")); + } else { + assert!(actual.0.is_ok()); + assert_eq!( + actual.1, + [ + (b"first".to_vec(), data.clone()), + (b"second".to_vec(), data.clone()) + ] + ); + } + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } + } +} + +#[test] +fn independent_history_late_publication_cancellation_keeps_the_verified_prefix() { + let (archive, data) = archive( + ArchiveVersion::Rar50, + Some(FilterKind::Delta { channels: 2 }), + ); + for parallel in [false, true] { + let token = crate::ReadCancellation::new(); + let mut options = + crate::ArchiveReadOptions::new().with_rar50_buffered_decode_limit(u64::MAX); + options.cancellation = Some(&token); + let mut writers = Vec::new(); + let mut open = |meta: &ExtractedEntryMeta| { + let bytes = Rc::new(RefCell::new(Vec::new())); + writers.push((meta.name.clone(), bytes.clone())); + if meta.name == b"second" { + token.cancel(); + } + Ok(Box::new(Sink(bytes)) as Box) + }; + let error = if parallel { + archive.extract_to_parallel_buffered(options, &mut open) + } else { + archive.extract_to(options, &mut open) + } + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Cancelled); + let output: Vec<_> = writers + .into_iter() + .map(|(name, bytes)| { + let data = bytes.borrow().clone(); + (name, data) + }) + .collect(); + assert_eq!( + output, + [ + (b"first".to_vec(), data.clone()), + (b"second".to_vec(), Vec::new()) + ] + ); + } +} + +// Original d2 private-worker pipeline, independent of the history selection. +fn reference_decode_parallel_entry( + archive: &Archive, + file: &FileHeader, + password: Option<&[u8]>, + buffered_decode_limit: u64, + options: crate::ArchiveReadOptions<'_>, + allowance: &B, +) -> Result> { + options.check_cancelled()?; + let mut budget = crate::output_limit::OutputBudget::new(options); + file.check_dictionary_limit(options.rar50_dictionary_size_limit)?; + file.check_output_limit(&budget)?; + if let Some(redirection) = &file.redirection { + return Ok(ParallelExtractedEntry::Redirection { + meta: file.metadata(), + redirection: redirection.clone(), + }); + } + // The only caller dispatches workers after excluding split members. + let meta = file.metadata(); + if meta.is_directory { + return Ok(ParallelExtractedEntry::Directory(meta)); + } + let mut data = Buffer::new(allowance); + let mut session = DecoderSession::with_allowance(password, buffered_decode_limit, allowance); + session.decoder.read_control = budget.control.clone(); + session.scratch = options.rar50_scratch; + budget.run(&file.name, &mut data, |writer| { + session.write_file_to(archive, file, writer) + })?; + Ok(ParallelExtractedEntry::File { meta, data }) +} diff --git a/arca-rar/vendor/rars/src/rar50/scratch.rs b/arca-rar/vendor/rars/src/rar50/scratch.rs new file mode 100644 index 00000000..5fd6181c --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/scratch.rs @@ -0,0 +1,387 @@ +use super::*; +use crate::codec::rar50::{apply_filter_data_with_allowance, FilterType, PendingFilter}; +use crate::temp_file::TemporaryFile; +use std::cell::RefCell; +use std::io::{Seek, SeekFrom}; +use std::rc::Rc; + +struct DiskBudget { + used: u64, + limit: u64, +} +struct ScratchFile { + spool: TemporaryFile, + budget: Rc>, + pos: u64, + len: u64, +} + +impl ScratchFile { + fn create(policy: &crate::Rar50Scratch, budget: &Rc>) -> Result { + Ok(Self { + spool: TemporaryFile::create(&policy.directory)?, + budget: budget.clone(), + pos: 0, + len: 0, + }) + } + fn rewind(&mut self) -> Result<()> { + self.seek(SeekFrom::Start(0))?; + Ok(()) + } +} +impl Read for ScratchFile { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + let count = self.spool.read(bytes)?; + self.pos += count as u64; + Ok(count) + } +} +impl Write for ScratchFile { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + // Decoder output and admitted record writes are bounded before dispatch; + // filter overwrites stay within their validated payload range. + let end = self.pos + bytes.len() as u64; + let mut budget = self.budget.borrow_mut(); + let additional = end.saturating_sub(self.len); + if additional > budget.limit - budget.used { + return Err(std::io::Error::other(Error::Rar50ScratchLimitExceeded { + limit: budget.limit, + required: budget.used.saturating_add(additional), + })); + } + let count = self.spool.write(bytes)?; + self.pos += count as u64; + budget.used += self.pos.saturating_sub(self.len); + self.len = self.len.max(self.pos); + Ok(count) + } + fn flush(&mut self) -> std::io::Result<()> { + self.spool.flush() + } +} +impl Seek for ScratchFile { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.pos = self.spool.seek(from)?; + Ok(self.pos) + } +} + +fn copy( + source: &mut ScratchFile, + mut destination: &mut dyn Write, + control: &crate::read_control::ReadControl, +) -> Result<()> { + source.rewind()?; + let mut buffer = [0; 64 * 1024]; + loop { + control.check()?; + let count = source.read(&mut buffer)?; + if count == 0 { + break; + } + control.write_all(&mut destination, &buffer[..count])?; + } + control.check() +} + +fn verify( + file: &FileHeader, + data: &mut ScratchFile, + keys: Option<&Rar50Keys>, + control: &crate::read_control::ReadControl, +) -> Result<()> { + data.rewind()?; + let mut crc = Crc32::new(); + let mut hash = streaming_hash_verifier(file)?; + let mut buffer = [0; 64 * 1024]; + loop { + control.check()?; + let count = data.read(&mut buffer)?; + if count == 0 { + break; + } + crc.update(&buffer[..count]); + if let Some((_, hasher)) = &mut hash { + hasher.update(&buffer[..count]); + } + } + file.verify_streaming_integrity(crc, hash, keys) +} + +fn encode_filter(filter: PendingFilter) -> [u8; 24] { + let mut bytes = [0; 24]; + bytes[..8].copy_from_slice(&(filter.start as u64).to_le_bytes()); + bytes[8..16].copy_from_slice(&(filter.length as u64).to_le_bytes()); + bytes[16] = match filter.filter_type { + FilterType::Delta => 0, + FilterType::E8 => 1, + FilterType::E8E9 => 2, + FilterType::Arm => 3, + }; + bytes[17] = filter.channels as u8; + bytes +} + +fn decode_filter( + bytes: [u8; 24], + output_size: usize, + filter_memory_limit: u64, +) -> Result { + let start = usize::try_from(u64::from_le_bytes(bytes[..8].try_into().unwrap())) + .map_err(|_| Error::InvalidHeader("scratch filter offset overflows"))?; + let length = usize::try_from(u64::from_le_bytes(bytes[8..16].try_into().unwrap())) + .map_err(|_| Error::InvalidHeader("scratch filter length overflows"))?; + let filter_type = match bytes[16] { + 0 => FilterType::Delta, + 1 => FilterType::E8, + 2 => FilterType::E8E9, + 3 => FilterType::Arm, + _ => return Err(Error::InvalidHeader("scratch filter type changed")), + }; + let required = (length as u64).saturating_mul(2); + if required > filter_memory_limit { + return Err(Error::Rar50FilterMemoryLimitExceeded { + limit: filter_memory_limit, + required, + }); + } + if start + .checked_add(length) + .is_none_or(|end| end > output_size) + { + return Err(Error::InvalidHeader("scratch filter range exceeds output")); + } + Ok(PendingFilter { + start, + length, + filter_type, + channels: bytes[17] as usize, + }) +} + +pub(super) fn decode( + file: &FileHeader, + packed: &mut R, + keys: Option<&Rar50Keys>, + decoder: &mut ReaderState, + policy: &crate::Rar50Scratch, + writer: &mut dyn Write, +) -> Result<()> { + if cfg!(all(target_arch = "wasm32", target_os = "unknown")) { + return Err(Error::UnsupportedFamilyFeature { + family: crate::ArchiveFamily::Rar50Plus, + feature: "disk-backed reader scratch on bare WebAssembly", + }); + } + let control = decoder.read_control.clone(); + control.check()?; + // Reserve the known payload storage before creating any temporary files. + let required = file + .unpacked_size + .checked_mul(2) + .ok_or(Error::Rar50ScratchLimitExceeded { + limit: policy.max_bytes, + required: u64::MAX, + })?; + if required > policy.max_bytes { + return Err(Error::Rar50ScratchLimitExceeded { + limit: policy.max_bytes, + required, + }); + } + let payload_reservation = required; + let budget = Rc::new(RefCell::new(DiskBudget { + used: 0, + limit: policy.max_bytes, + })); + let mut raw = ScratchFile::create(policy, &budget)?; + let mut records = ScratchFile::create(policy, &budget)?; + let info = file.decoded_compression_info()?; + let output_size = checked_unpacked_size(file.unpacked_size)?; + let dictionary_size = usize::try_from(info.dictionary_size) + .map_err(|_| Error::InvalidHeader("RAR 5 dictionary size overflows host address size"))?; + decoder + .decode_to_sink_with_filters( + packed, + info.algorithm_version, + output_size, + dictionary_size, + info.solid, + |chunk| -> Result<()> { + control.check()?; + match chunk { + DecodedChunk::Bytes(bytes) => raw.write_all(bytes)?, + DecodedChunk::Repeated { byte, len } => { + let buffer = [byte; 64 * 1024]; + let mut remaining = len; + while remaining != 0 { + control.check()?; + let count = remaining.min(buffer.len()); + raw.write_all(&buffer[..count])?; + remaining -= count; + } + } + } + Ok(()) + }, + Some(&mut |filter: PendingFilter| -> Result<()> { + control.check()?; + let required = (filter.length as u64).saturating_mul(2); + if required > policy.filter_memory_limit { + return Err(Error::Rar50FilterMemoryLimitExceeded { + limit: policy.filter_memory_limit, + required, + }); + } + // Records may arrive before payload bytes: retain the payload + // reservation rather than letting records spend that space. + let disk_required = payload_reservation + .checked_add(records.len) + .and_then(|bytes| bytes.checked_add(24)) + .ok_or(Error::Rar50ScratchLimitExceeded { + limit: policy.max_bytes, + required: u64::MAX, + })?; + if disk_required > policy.max_bytes { + return Err(Error::Rar50ScratchLimitExceeded { + limit: policy.max_bytes, + required: disk_required, + }); + } + records.write_all(&encode_filter(filter))?; + Ok(()) + }), + ) + .map_err(|error| match error { + StreamDecodeError::Decode(error) => Error::from(error), + StreamDecodeError::Sink(error) => error, + StreamDecodeError::FilteredMember => unreachable!("scratch supplies a filter handler"), + })?; + if records.len == 0 { + verify(file, &mut raw, keys, &control)?; + return copy(&mut raw, writer, &control); + } + let mut transformed = ScratchFile::create(policy, &budget)?; + copy(&mut raw, &mut transformed, &control)?; + records.rewind()?; + while records.pos < records.len { + control.check()?; + let mut bytes = [0; 24]; + records.read_exact(&mut bytes)?; + // These bytes came back from disk. Revalidate them before allocating + // or seeking, even though the codec admitted the original record. + let filter = decode_filter(bytes, output_size, policy.filter_memory_limit)?; + transformed.seek(SeekFrom::Start(filter.start as u64))?; + let mut data = Buffer::filled(filter.length, 0, &decoder.allowance())?; + control.reader(&mut transformed).read_exact(&mut data)?; + apply_filter_data_with_allowance(&mut data, &filter, &control, &decoder.allowance())?; + transformed.seek(SeekFrom::Start(filter.start as u64))?; + control.write_all(&mut transformed, &data)?; + } + verify(file, &mut transformed, keys, &control)?; + copy(&mut transformed, writer, &control) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scratch_files_share_disk_quota_and_overwrites_do_not_spend_it_twice() { + let dir = crate::scratch::case("reader-scratch-shared-quota"); + let policy = crate::Rar50Scratch::new(&*dir, 8); + let budget = Rc::new(RefCell::new(DiskBudget { used: 0, limit: 8 })); + { + let mut first = ScratchFile::create(&policy, &budget).unwrap(); + let mut second = ScratchFile::create(&policy, &budget).unwrap(); + first.write_all(b"abcd").unwrap(); + second.write_all(b"efgh").unwrap(); + first.rewind().unwrap(); + first.write_all(b"ABCD").unwrap(); + first.flush().unwrap(); + assert_eq!(budget.borrow().used, 8); + let error = second.write_all(b"i").unwrap_err(); + assert!(matches!( + error.get_ref().unwrap().downcast_ref::(), + Some(Error::Rar50ScratchLimitExceeded { + limit: 8, + required: 9 + }) + )); + first.rewind().unwrap(); + let mut data = Vec::new(); + first.read_to_end(&mut data).unwrap(); + assert_eq!(data, b"ABCD"); + } + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } + + #[test] + fn scratch_filter_records_reject_corrupt_tags_and_preserve_offsets() { + for filter_type in [ + FilterType::Delta, + FilterType::E8, + FilterType::E8E9, + FilterType::Arm, + ] { + let filter = PendingFilter { + start: usize::MAX - 17, + length: 17, + filter_type, + channels: 3, + }; + let mut bytes = encode_filter(filter); + let decoded = decode_filter(bytes, usize::MAX, 34).unwrap(); + assert_eq!(decoded.start, filter.start); + assert_eq!(decoded.length, filter.length); + assert_eq!(decoded.filter_type, filter.filter_type); + assert_eq!(decoded.channels, filter.channels); + bytes[16] = 4; + assert!(matches!( + decode_filter(bytes, usize::MAX, 34), + Err(Error::InvalidHeader("scratch filter type changed")) + )); + bytes[16] = 0; + bytes[8..16].copy_from_slice(&18u64.to_le_bytes()); + assert!(matches!( + decode_filter(bytes, usize::MAX, 34), + Err(Error::Rar50FilterMemoryLimitExceeded { + limit: 34, + required: 36 + }) + )); + bytes[8..16].copy_from_slice(&17u64.to_le_bytes()); + bytes[..8].copy_from_slice(&(usize::MAX as u64).to_le_bytes()); + assert!(matches!( + decode_filter(bytes, usize::MAX, 34), + Err(Error::InvalidHeader("scratch filter range exceeds output")) + )); + bytes[..8].copy_from_slice(&1u64.to_le_bytes()); + assert!(matches!( + decode_filter(bytes, 17, 34), + Err(Error::InvalidHeader("scratch filter range exceeds output")) + )); + } + + if usize::BITS < 64 { + let mut bytes = encode_filter(PendingFilter { + start: 0, + length: 1, + filter_type: FilterType::Delta, + channels: 1, + }); + bytes[..8].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(matches!( + decode_filter(bytes, usize::MAX, u64::MAX), + Err(Error::InvalidHeader("scratch filter offset overflows")) + )); + bytes[..8].copy_from_slice(&0u64.to_le_bytes()); + bytes[8..16].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(matches!( + decode_filter(bytes, usize::MAX, u64::MAX), + Err(Error::InvalidHeader("scratch filter length overflows")) + )); + } + } +} diff --git a/arca-rar/vendor/rars/src/rar50/streaming_checks_tests.rs b/arca-rar/vendor/rars/src/rar50/streaming_checks_tests.rs new file mode 100644 index 00000000..52caf3d4 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/streaming_checks_tests.rs @@ -0,0 +1,531 @@ +use super::*; +use crate::rar50::{ + ArchiveEntry, Block, FileHash, FilterPolicy, Rar50Writer, WriterOptions, HFL_SPLIT_AFTER, + HFL_SPLIT_BEFORE, +}; +use crate::{ArchiveVersion, EntrySource, FeatureSet}; +use std::cell::Cell; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Work { + bytes: usize, + zeroes: usize, + repeated_bytes: usize, + repeated_zeroes: usize, +} + +std::thread_local! { + static WORK: Cell = const { Cell::new(Work { + bytes: 0, zeroes: 0, repeated_bytes: 0, repeated_zeroes: 0, + }) }; +} + +pub(super) fn record_crc_work(len: usize, zeroes: bool) { + WORK.with(|work| { + let mut value = work.get(); + if zeroes { + value.zeroes += len; + } else { + value.bytes += len; + } + work.set(value); + }); +} + +pub(super) fn record_repeated_output(len: usize, zeroes: bool) { + WORK.with(|work| { + let mut value = work.get(); + value.repeated_bytes += len; + if zeroes { + value.repeated_zeroes += len; + } + work.set(value); + }); +} + +fn measured(run: impl FnOnce() -> T) -> (T, Work) { + WORK.with(|work| work.set(Work::default())); + let value = run(); + (value, WORK.with(Cell::get)) +} + +fn fixture(data: &[u8], stored: bool, encrypted: bool) -> (Archive, FileHeader) { + let mut input = ArchiveEntry::new(b"payload".to_vec(), EntrySource::from_bytes(data.to_vec())); + if encrypted { + input = input.with_password(b"password".to_vec()); + } + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(if stored { 0 } else { 1 }) + .with_dictionary_size(128 * 1024), + ) + .entry(input) + .filter_policy(FilterPolicy::None) + .finish() + .unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let file = archive.files().next().unwrap().clone(); + assert_eq!(file.is_stored(), stored); + (archive, file) +} + +fn ordinary(archive: &Archive, file: &FileHeader) -> (Result>, Work) { + measured(|| { + let mut session = + DecoderSession::with_allowance(Some(b"password"), 0, &Allowance::default()); + let mut output = Vec::new(); + session.write_file_to(archive, file, &mut output)?; + Ok(output) + }) +} + +#[test] +fn ordinary_streaming_skips_crc_work_only_when_integrity_does_not_use_it() { + let data = b"ordinary streaming integrity bytes\n".repeat(4096); + for stored in [false, true] { + let (archive, original) = fixture(&data, stored, false); + for layout in ["hash", "none", "crc", "unknown"] { + let mut file = original.clone(); + match layout { + "hash" => file.data_crc32 = Some(crc32(&data) ^ 1), + "none" => { + file.data_crc32 = None; + file.hash = None; + } + "crc" => file.hash = None, + "unknown" => { + file.hash = Some(FileHash { + hash_type: 7, + data: vec![1], + }); + } + _ => unreachable!(), + } + let (result, work) = ordinary(&archive, &file); + assert_eq!(result.unwrap(), data); + assert_eq!( + work.bytes + work.zeroes, + if matches!(layout, "crc" | "unknown") { + data.len() + } else { + 0 + }, + "stored={stored}, layout={layout}, work={work:?}", + ); + } + } +} + +#[test] +fn optional_repeated_output_keeps_hash_and_real_zero_run_delivery() { + let data = vec![0; 200 * 1024]; + let (archive, mut file) = fixture(&data, false, false); + file.data_crc32 = Some(1); + let (result, work) = ordinary(&archive, &file); + assert_eq!(result.unwrap(), data); + assert!( + work.repeated_zeroes > 64 * 1024, + "must use actual codec zero runs" + ); + assert_eq!(work.bytes + work.zeroes, 0, "{work:?}"); +} + +#[test] +fn optional_repeated_output_preserves_nonzero_hash_and_sink_failure_order() { + let data = vec![0xff; 140_000]; + let mut output = Vec::new(); + let expected = blake2sp::hash(&data); + let mut hash = Some((expected, blake2sp::Hasher::new())); + let (result, work) = measured(|| { + write_repeated_chunk_with_optional_crc(&mut output, None, &mut hash, 0xff, data.len()) + }); + result.unwrap(); + assert_eq!(output, data); + assert_eq!(hash.unwrap().1.finalize(), expected); + assert_eq!(work.bytes + work.zeroes, 0); + assert_eq!(work.repeated_bytes, data.len()); + + let mut crc = Crc32::new(); + let mut hash = Some((expected, blake2sp::Hasher::new())); + let (result, work) = + measured(|| write_repeated_chunk(&mut RejectWrite, &mut crc, &mut hash, 0, 100_000)); + assert!(result.is_err()); + assert_eq!(crc.finish(), Crc32::new().finish()); + assert_eq!(hash.unwrap().1.finalize(), blake2sp::hash(b"")); + assert_eq!(work, Work::default()); +} + +#[test] +fn ordinary_streaming_keeps_checksum_authority_and_malformed_hash_precedence() { + let data = b"checksum authority\n".repeat(8192); + for stored in [false, true] { + let (archive, original) = fixture(&data, stored, false); + for layout in ["bad hash", "bad crc", "unknown hash", "malformed hash"] { + let mut file = original.clone(); + match layout { + "bad hash" => file.hash.as_mut().unwrap().data[0] ^= 1, + "bad crc" => { + file.hash = None; + file.data_crc32 = Some(crc32(&data) ^ 1); + } + "unknown hash" => { + file.hash = Some(FileHash { + hash_type: 99, + data: vec![1], + }); + file.data_crc32 = Some(crc32(&data) ^ 1); + } + "malformed hash" => { + file.hash.as_mut().unwrap().data.pop(); + file.data_crc32 = Some(crc32(&data) ^ 1); + } + _ => unreachable!(), + } + let (result, work) = ordinary(&archive, &file); + let error = result.unwrap_err(); + match layout { + "bad hash" => { + assert!(matches!( + error.root_cause(), + Error::HashMismatch { hash_type: 0 } + )); + assert_eq!(work.bytes + work.zeroes, 0); + } + "bad crc" | "unknown hash" => { + assert!(matches!(error.root_cause(), Error::Crc32Mismatch { .. })); + assert_eq!(work.bytes + work.zeroes, data.len()); + } + "malformed hash" => { + assert!(matches!( + error.root_cause(), + Error::InvalidHeader("RAR 5 BLAKE2sp hash record has invalid length") + )); + assert_eq!(work, Work::default()); + } + _ => unreachable!(), + } + } + } +} + +struct RejectWrite; +impl Write for RejectWrite { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("ordinary streaming sink failed")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn ordinary_streaming_preserves_sink_error_context_before_hash_failure() { + let data = b"sink failure must precede final checksum\n".repeat(4096); + for stored in [false, true] { + let (archive, mut file) = fixture(&data, stored, false); + file.hash.as_mut().unwrap().data[0] ^= 1; + let mut session = DecoderSession::with_allowance(None, 0, &Allowance::default()); + let (result, work) = measured(|| session.write_file_to(&archive, &file, &mut RejectWrite)); + let error = result.unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.entry_context().unwrap().1, + if stored { "writing" } else { "decoding" }, + ); + assert_eq!(work.bytes + work.zeroes, 0); + } +} + +#[test] +#[cfg(feature = "encryption")] +fn ordinary_encrypted_streaming_preserves_crc_mac_hash_mac_padding_and_truncation() { + let data = b"encrypted MAC payload!".repeat(4097); + for stored in [false, true] { + let (archive, original) = fixture(&data, stored, true); + assert!(original.uses_hash_mac()); + let (result, work) = ordinary(&archive, &original); + assert_eq!(result.unwrap(), data); + assert_eq!(work.bytes + work.zeroes, 0); + let mut crc_only = original.clone(); + crc_only.hash = None; + let (result, work) = ordinary(&archive, &crc_only); + assert_eq!(result.unwrap(), data); + assert_eq!(work.bytes + work.zeroes, data.len()); + crc_only.data_crc32 = Some(crc_only.data_crc32.unwrap() ^ 1); + assert!(matches!( + ordinary(&archive, &crc_only).0.unwrap_err().root_cause(), + Error::Crc32Mismatch { .. }, + )); + let mut bad_hash = original.clone(); + bad_hash.hash.as_mut().unwrap().data[0] ^= 1; + assert!(matches!( + ordinary(&archive, &bad_hash).0.unwrap_err().root_cause(), + Error::HashMismatch { hash_type: 0 }, + )); + let mut truncated = original.clone(); + truncated.block.data_range.end -= 1; + truncated.block.data_size = Some(truncated.block.data_range.len() as u64); + let error = ordinary(&archive, &truncated).0.unwrap_err(); + // Compare the complete legacy path, including reader admission. + // Shortening the declared cipher payload fails its alignment check + // before either stored or compressed decoding can read it. + let mut decoder = ReaderState::new(&Allowance::default()); + let expected = (|| { + let (mut packed, keys) = truncated.packed_reader_with_allowance( + &archive, + Some(b"password"), + &Allowance::default(), + )?; + truncated.stream_packed_with_decoder( + &mut packed, + keys.as_ref(), + &mut decoder, + 0, + None, + &mut std::io::sink(), + ) + })() + .unwrap_err(); + assert_eq!(error.root_cause(), expected.root_cause()); + assert!(matches!( + expected.root_cause(), + Error::InvalidHeader("RAR 5 encrypted file payload is not block aligned") + )); + if stored { + let mut padding = bad_hash; + padding.unpacked_size -= 1; + let error = ordinary(&archive, &padding).0.unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::InvalidHeader("RAR 5 encrypted stored file has non-zero padding") + )); + } + } +} + +#[test] +fn ordinary_streaming_admission_and_source_open_precede_malformed_hash() { + let dir = crate::scratch::case("ordinary-streaming-precedence"); + let path = dir.join("removed.rar"); + for stored in [false, true] { + let data = b"header resource admission".repeat(4096); + let (original, _) = fixture(&data, stored, false); + let crate::source::ArchiveSource::Memory(bytes) = &original.source else { + unreachable!() + }; + std::fs::write(&path, bytes).unwrap(); + let mut archive = Archive::parse_path(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + let Block::File(file) = archive + .blocks + .iter_mut() + .find(|b| matches!(b, Block::File(_))) + .unwrap() + else { + unreachable!() + }; + file.hash.as_mut().unwrap().data.pop(); + let mut opened = 0; + let limit = data.len() as u64 - 1; + for options in [ + crate::ArchiveReadOptions::new().with_max_member_output_bytes(limit), + crate::ArchiveReadOptions::new().with_max_total_output_bytes(limit), + ] { + let error = archive + .extract_to(options.with_rar50_buffered_decode_limit(0), |_| { + opened += 1; + Ok(Box::new(std::io::sink()) as Box) + }) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + } + assert_eq!(opened, 0); + let file = archive.files().next().unwrap(); + if !stored { + let error = archive + .extract_to( + crate::ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(0) + .with_rar50_dictionary_size_limit(0), + |_| { + opened += 1; + Ok(Box::new(std::io::sink()) as Box) + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(opened, 0); + } + let (result, work) = ordinary(&archive, file); + let error = result.unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!(work, Work::default()); + } +} + +#[test] +fn split_compressed_members_retain_always_crc_work() { + let data = b"split compressed hash and crc\n".repeat(8192); + let (archive, original) = fixture(&data, false, false); + let middle = original.block.data_range.start + original.packed_size() as usize / 2; + let mut first = original.clone(); + first.block.flags |= HFL_SPLIT_AFTER; + first.block.data_range.end = middle; + first.block.data_size = Some(first.block.data_range.len() as u64); + let mut last = original; + last.block.flags |= HFL_SPLIT_BEFORE; + last.block.data_range.start = middle; + last.block.data_size = Some(last.block.data_range.len() as u64); + last.data_crc32 = Some(1); // Its valid BLAKE remains authoritative. + let mut volume1 = archive.clone(); + volume1.blocks = vec![Block::File(first.clone())]; + let mut volume2 = archive; + volume2.blocks = vec![Block::File(last.clone())]; + let volumes = [volume1, volume2]; + let mut split = PendingSplitRefs::with_allowance(&first, 0, 0, &Allowance::default()).unwrap(); + split.append(1, 0).unwrap(); + let mut session = DecoderSession::with_allowance(None, 0, &Allowance::default()); + let mut output = Vec::new(); + let (result, work) = + measured(|| session.stream_split_to(&volumes, &split, &last, None, &mut output)); + result.unwrap(); + assert_eq!(output, data); + assert_eq!(work.bytes + work.zeroes, data.len()); +} + +#[test] +fn optional_crc_policy_keeps_scratch_dispatch_before_hash_initialization() { + let dir = crate::scratch::case("ordinary-streaming-scratch-policy"); + let policy = crate::Rar50Scratch::new(&*dir, 1024 * 1024); + let data = b"scratch route bytes\n".repeat(4096); + let (archive, mut file) = fixture(&data, false, false); + let mut session = DecoderSession::with_allowance(None, 0, &Allowance::default()); + session.scratch = Some(&policy); + let mut output = Vec::new(); + session.write_file_to(&archive, &file, &mut output).unwrap(); + assert_eq!(output, data); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + file.hash.as_mut().unwrap().data.pop(); + file.block.data_range.end = file.block.data_range.start; + file.block.data_size = Some(0); + let error = session + .write_file_to(&archive, &file, &mut std::io::sink()) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::Codec(crate::codec::Error::NeedMoreInput) + )); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); +} + +#[test] +fn optional_crc_keeps_limited_growth_and_refusals_identical_to_legacy_streaming() { + use crate::codec::workspace::RefusingBudget; + let data = b"limited CRC policy bytes\n".repeat(4096); + let (archive, file) = fixture(&data, false, false); + let run = |budget: &RefusingBudget, optional| { + let mut session = DecoderSession::with_allowance(None, 0, budget); + let mut output = Buffer::new(budget); + if optional { + session.write_file_to(&archive, &file, &mut output)?; + } else { + let mut decoder = session.decoder.try_clone()?; + let (mut packed, keys) = file.packed_reader_with_allowance(&archive, None, budget)?; + file.stream_packed_with_decoder( + &mut packed, + keys.as_ref(), + &mut decoder, + 0, + None, + &mut output, + )?; + } + assert_eq!(&output[..], &data); + Ok::<(), Error>(()) + }; + let baseline = RefusingBudget::new(usize::MAX); + run(&baseline, false).unwrap(); + let attempts = baseline.attempts(); + assert!(attempts > 0); + assert_eq!(baseline.used(), 0); + for fail_at in 0..=attempts { + let old = RefusingBudget::new(fail_at); + let new = RefusingBudget::new(fail_at); + let expected = run(&old, false); + let actual = run(&new, true); + assert_eq!( + actual.as_ref().err().map(Error::kind), + expected.as_ref().err().map(Error::kind), + "allocation {fail_at}" + ); + assert_eq!(new.attempts(), old.attempts(), "allocation {fail_at}"); + assert_eq!(new.used(), 0); + assert_eq!(old.used(), 0); + } +} + +#[test] +fn optional_crc_preserves_bad_hash_checkpoint_for_a_history_dependent_successor() { + let mut random = 0x1234_5678u32; + let seed: Vec<_> = (0..32749) + .map(|_| { + random ^= random << 13; + random ^= random >> 17; + random ^= random << 5; + random as u8 + }) + .collect(); + let first = seed.repeat(4); + let next = first[127..4224].to_vec(); + let mut builder = crate::Builder::new(ArchiveVersion::Rar50) + .store(false) + .solid(true) + .compression_level(Some(1)); + builder + .add_bytes(b"first".to_vec(), first, None, None) + .unwrap(); + builder + .add_bytes(b"next".to_vec(), next.clone(), None, None) + .unwrap(); + let archive = Archive::parse_owned(builder.to_bytes().unwrap()).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert!(files[1].decoded_compression_info().unwrap().solid); + assert!( + files[1].packed_size() < 128, + "successor must use predecessor history" + ); + + let different = b"failed member replaces the old raw dictionary\n".repeat(4096); + let (failed_archive, valid) = fixture(&different, false, false); + let mut corrupt = valid.clone(); + corrupt.hash.as_mut().unwrap().data[0] ^= 1; + let mut session = DecoderSession::with_allowance(None, 0, &Allowance::default()); + session + .write_file_to(&archive, files[0], &mut std::io::sink()) + .unwrap(); + let (result, work) = + measured(|| session.write_file_to(&failed_archive, &corrupt, &mut std::io::sink())); + assert!(matches!( + result.unwrap_err().root_cause(), + Error::HashMismatch { hash_type: 0 } + )); + assert_eq!( + work.bytes + work.zeroes, + 0, + "bad digest does not change hash authority" + ); + let mut output = Vec::new(); + session + .write_file_to(&archive, files[1], &mut output) + .unwrap(); + assert_eq!(output, next); + + // Prove a mistakenly committed failed dictionary cannot pass the fixture. + session + .write_file_to(&failed_archive, &valid, &mut std::io::sink()) + .unwrap(); + let error = session + .write_file_to(&archive, files[1], &mut std::io::sink()) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ChecksumMismatch); +} diff --git a/arca-rar/vendor/rars/src/rar50/write/compress.rs b/arca-rar/vendor/rars/src/rar50/write/compress.rs new file mode 100644 index 00000000..8d68f48a --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/compress.rs @@ -0,0 +1,3235 @@ +//! Turning member sources into packed payloads with workspace admission. +//! +//! RAR 5 compresses in independent blocks: a block depends only on its own +//! bytes and on up to a dictionary's worth of the raw input that precedes it. +//! Since that preceding input is just the file being read, the history a block +//! needs is known before any compression happens, and blocks can be compressed +//! in parallel while their packed output is written back in order. +//! +//! Non-solid members each carry their own history and are interleaved so that +//! several small members keep every core busy. Solid members share one history +//! chain that runs across member boundaries, so their blocks are produced by a +//! single walk through the members in order — the walk is just reading, which +//! is cheap, so waves of blocks still compress in parallel. + +#[cfg(test)] +use super::filter_policy::encode_member_with_filter_policy_candidates_and_progress; +use super::filter_policy::{ + candidates_with_allowance, compression_info, should_store_compressed_payload, +}; +use super::FilterPolicy; +#[cfg(test)] +use crate::codec::rar50::EncodeOptions; +use crate::codec::rar50::{streaming_blocks_with_allowance, BlockSplitter}; +use crate::codec::workspace::{Allowance, Budget, Buffer}; +use crate::crc32::Crc32; +use crate::rar50::blake2sp; +use crate::streaming::preparation::Records; +use crate::streaming::Spool; +use crate::{EntrySource, Error, Result, WriterResources}; +use std::io::{Read, Write}; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Compression work and member lifecycle, independent of presentation. +pub(super) trait CompressionProgress: Sync { + fn advance(&self, bytes: u64) -> bool; + fn is_cancelled(&self) -> bool { + false + } + fn started(&self, _member: usize, _size: u64) {} + fn finished(&self, _member: usize, _size: u64) {} +} +impl bool + Sync> CompressionProgress for F { + fn advance(&self, bytes: u64) -> bool { + self(bytes) + } +} + +/// Batch-local cancellation joins already admitted workers and prevents queued +/// work from opening a source after a sibling has failed. The caller's token is +/// never changed, so the same resources can be reused after an ordinary error. +struct BatchProgress<'a> { + progress: &'a dyn CompressionProgress, + resources: &'a WriterResources, + stopped: AtomicBool, +} +impl CompressionProgress for BatchProgress<'_> { + fn advance(&self, bytes: u64) -> bool { + if self.is_cancelled() { + return false; + } + if !self.progress.advance(bytes) { + self.stopped.store(true, Ordering::Release); + return false; + } + !self.is_cancelled() + } + fn is_cancelled(&self) -> bool { + self.stopped.load(Ordering::Acquire) + || self.resources.is_cancelled() + || self.progress.is_cancelled() + } + fn started(&self, member: usize, size: u64) { + self.progress.started(member, size); + } + fn finished(&self, member: usize, size: u64) { + self.progress.finished(member, size); + } +} + +/// Slots are admitted before dispatch. Both input and result descriptor storage +/// stay owned until all callbacks join, even if one callback fails. The caller +/// retains its workspace reservation through consumption of returned results. +fn run_jobs( + jobs: Records, + resources: &WriterResources, + progress: &dyn CompressionProgress, + map: impl Fn(T, &dyn CompressionProgress) -> Result + Sync + Send, +) -> Result> { + let output = Records::new(jobs.len(), resources)?; + let slots = Records::collect(jobs.into_iter().map(|job| Ok((Some(job), None))), resources)?; + complete_jobs(slots, output, resources, progress, |_, job, progress| { + map(job, progress) + }) +} + +type JobSlot = (Option, Option>); + +fn complete_jobs( + mut slots: Records>, + mut output: Records, + resources: &WriterResources, + progress: &dyn CompressionProgress, + map: impl Fn(usize, T, &dyn CompressionProgress) -> Result + Sync + Send, +) -> Result> { + let batch = BatchProgress { + progress, + resources, + stopped: AtomicBool::new(false), + }; + crate::parallel::for_each_mut(&mut slots, |index, (job, result)| { + if batch.is_cancelled() { + return; + } + let mapped = map(index, job.take().expect("one callback per slot"), &batch); + if mapped.is_err() { + batch.stopped.store(true, Ordering::Release); + } + *result = Some(mapped); + }); + // Prefer the source/codec failure over cancellation caused by that failure. + // Scan in job order, so scheduling does not choose which error we report. + let mut failure = None; + for (_, result) in slots.iter_mut() { + if result.as_ref().is_some_and(Result::is_err) { + let error = result.take().unwrap().err().unwrap(); + if failure + .as_ref() + .is_none_or(|error: &Error| error.kind() == crate::ErrorKind::Cancelled) + { + failure = Some(error); + } + } + } + if let Some(error) = failure { + return Err(error); + } + if batch.is_cancelled() { + return Err(Error::Cancelled); + } + for (_, result) in slots { + output.push(result.expect("all jobs completed")?)?; + } + Ok(output) +} + +/// Admit fixed worker scopes only after coordinator slots have capacity. Keep +/// every reservation until all callbacks join, then retire unused capacity; +/// output owners remain charged after return. No worker can extend its scope. +fn run_jobs_admitted( + jobs: Records, + resources: &WriterResources, + progress: &dyn CompressionProgress, + required: impl Fn(&T) -> u64, + map: impl Fn( + T, + &WriterResources, + &dyn CompressionProgress, + &crate::codec::workspace::Limited, + ) -> Result + + Sync + + Send, +) -> Result> { + let ledger = resources + .execution + .as_ref() + .expect("admitted execution ledger"); + let output = Records::new(jobs.len(), resources)?; + let slots = Records::collect(jobs.into_iter().map(|job| Ok((Some(job), None))), resources)?; + let mut reservations = Records::new(slots.len(), resources)?; + for (job, _) in slots.iter() { + if progress.is_cancelled() || resources.is_cancelled() { + return Err(Error::Cancelled); + } + reservations.push(ledger.reserve(required(job.as_ref().unwrap()))?)?; + } + for reservation in reservations.iter_mut() { + reservation.start(); + } + let result = complete_jobs( + slots, + output, + resources, + progress, + |index, job, progress| { + let allowance = reservations[index].allowance(); + let worker = resources + .clone() + .with_execution_allowance(allowance.clone()); + map(job, &worker, progress, &allowance) + }, + ); + for reservation in reservations { + reservation.retire(); + } + result +} + +/// A member that has been compressed and is waiting to be framed. +pub(super) struct CompressedMember { + pub(super) input_size: u64, + pub(super) crc32: u32, + pub(super) hash: [u8; 32], + pub(super) packed: Spool, + /// True when the payload should be written as-is from the source because + /// compressing it did not pay. + pub(super) store: bool, + /// True when this member continues the previous member's dictionary. + pub(super) solid_continuation: bool, +} + +#[cfg(test)] +use super::plan::whole_member_workspace; +pub(super) use super::plan::CompressPlan; +use super::plan::{Execution, ExecutionPlan, MemberPlan}; + +/// A bounded run of adjacent blocks, sharing one copy of the preceding input. +struct BlockJob { + data: Buffer, + history: Buffer, + /// Member index, end within `data`, and final-block flag. + blocks: Records<(usize, usize, bool)>, +} + +impl BlockJob { + fn append(&mut self, data: Buffer) -> Result<()> { + if self.data.is_empty() { + self.data = data; + } else { + self.data + .extend_from_slice(&data) + .map_err(Into::::into)?; + } + Ok(()) + } +} + +fn run_size(plan: &CompressPlan) -> usize { + plan.encode_options + .max_match_distance + .max(crate::codec::rar50::MAX_LZ_BLOCK_SIZE) +} + +/// A member being read, and the packed bytes it has produced so far. +struct MemberStream { + member: usize, + input_size: u64, + started: bool, + source: EntrySource, + reader: Option>, + remaining: u64, + packed: Spool, + /// A chunk read to decide a block boundary and not used by that block. + pushback: Buffer, + crc: Crc32, + hash: blake2sp::Hasher, +} + +impl MemberStream { + fn new( + member: usize, + source: &EntrySource, + size: u64, + resources: &WriterResources, + progress: &dyn CompressionProgress, + allowance: &B, + ) -> Result { + if size == 0 { + progress.started(member, size); + if progress.is_cancelled() { + return Err(Error::Cancelled); + } + check_source_end(&mut *source.open()?)?; + progress.finished(member, size); + } + Ok(Self { + member, + input_size: size, + started: size == 0, + source: source.clone(), + reader: None, + remaining: size, + packed: Spool::create_parked(resources)?, + pushback: Buffer::new(allowance), + crc: Crc32::new(), + hash: blake2sp::Hasher::new(), + }) + } + + /// Whether this member has anything left, read or unread. + fn has_more(&self) -> bool { + self.remaining != 0 || !self.pushback.is_empty() + } +} + +/// `advance` is called with each newly completed chunk of work and returns +/// false when the caller wants to stop. +pub(super) fn compress_members_with_context( + sources: &[EntrySource], + plan: &CompressPlan, + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), +) -> Result> { + if advance.is_cancelled() || resources.is_cancelled() { + return Err(Error::Cancelled); + } + let mut integrity = Records::new(sources.len(), resources)?; + for (index, source) in sources.iter().enumerate() { + let input_size = source.len().map_err(|error| error_context(index, error))?; + // Compression fills these checksums while consuming the source. + integrity.push((input_size, 0, [0; 32]))?; + } + + let execution = + ExecutionPlan::with_resources(plan, integrity.iter().map(|entry| entry.0), resources)?; + let packed = match execution { + ExecutionPlan::IndependentMembers(members) => { + return compress_members_whole( + sources, + &integrity, + plan, + &members, + resources, + advance, + error_context, + ); + } + // Storing is not "compress and hope it does not help": the header + // records method zero, so the payload must be the source bytes. + ExecutionPlan::Stored => { + for (index, (source, (input_size, crc, hash))) in + sources.iter().zip(&mut integrity).enumerate() + { + advance.started(index, *input_size); + (*crc, *hash) = super::source_integrity( + source, + *input_size, + plan.block_size, + advance, + resources, + ) + .map_err(|error| error_context(index, error))?; + if !advance.advance(*input_size) { + return Err(Error::Cancelled); + } + advance.finished(index, *input_size); + } + Records::collect( + integrity.iter().map(|_| Spool::create_parked(resources)), + resources, + )? + } + ExecutionPlan::Blocks { workspace } => compress_streaming_members( + sources, + &mut integrity, + plan, + workspace, + resources, + advance, + error_context, + )?, + }; + + Records::collect( + packed.into_iter().zip(&integrity).enumerate().map( + |(member, (packed, &(input_size, crc32, hash)))| { + Ok(CompressedMember { + input_size, + crc32, + hash, + // One rule, shared with the whole-member path and the legacy + // writers, plus the two cases that are not really fallbacks: + // storing was asked for, and an empty member has nothing to + // pack. `StoreFallback` refuses to store a solid member, whose + // successors decode against the dictionary it fills. + store: plan.method == 0 + || input_size == 0 + || should_store_compressed_payload( + input_size, + packed.len(), + plan.solid, + &plan.filter_policy, + ), + packed, + solid_continuation: plan.solid && member > 0, + }) + }, + ), + resources, + ) +} + +#[allow(clippy::too_many_arguments)] +fn compress_streaming_members( + sources: &[EntrySource], + integrity: &mut [(u64, u32, [u8; 32])], + plan: &CompressPlan, + required: u64, + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), +) -> Result> { + let max_jobs_by_memory = resources.memory_limit() / required; + if max_jobs_by_memory == 0 { + resources + .acquire_cancellable(required, plan.dictionary_size, &|| advance.is_cancelled())?; + unreachable!("oversized workspace acquisition must fail"); + } + let batch_capacity = usize::try_from(max_jobs_by_memory) + .unwrap_or(usize::MAX) + .min(crate::parallel::threads()) + .max(1); + + if let Some(allowance) = &resources.execution { + let encode = if plan.solid { + compress_solid_chain:: + } else { + compress_independent_members:: + }; + return encode( + sources, + integrity, + plan, + batch_capacity, + required, + resources, + advance, + error_context, + allowance, + ); + } + + if plan.solid { + compress_solid_chain( + sources, + integrity, + plan, + batch_capacity, + required, + resources, + advance, + error_context, + &Allowance::default(), + ) + } else { + compress_independent_members( + sources, + integrity, + plan, + batch_capacity, + required, + resources, + advance, + error_context, + &Allowance::default(), + ) + } +} + +/// Keep estimated codec admission separate from actual coordinator capacity. +/// Assembly runs on the coordinator; workers receive only their fixed scopes. +/// Retained history and spool capacity reduce the next wave's available space. +fn streaming_wave_capacity( + maximum: usize, + _required: u64, + _plan: &CompressPlan, + _resources: &WriterResources, +) -> usize { + if let Some(ledger) = &_resources.execution { + // Run growth/replacement, block assembly and history copies coexist. + // Descriptor capacity is still checked by its owners before dispatch. + let assembly = (run_size(_plan) as u64) + .saturating_mul(8) + .saturating_add((_plan.block_size as u64).saturating_mul(4)); + let per_job = _required + .saturating_add(assembly) + .saturating_add(crate::codec::workspace::RESERVATION_BYTES); + return maximum + .min(usize::try_from(ledger.available() / per_job.max(1)).unwrap_or(usize::MAX)) + .max(1); + } + maximum +} + +/// Compresses independent whole members concurrently, with each workspace +/// admitted against the shared budget before its input is loaded. +/// +/// The resolved plan schedules automatic-filter fallback outside whole-member +/// batches. Explicit filters retain their whole-member requirement and fail +/// admission when their workspace exceeds the budget. +#[allow(clippy::too_many_arguments)] +fn compress_members_whole( + sources: &[EntrySource], + integrity: &[(u64, u32, [u8; 32])], + plan: &CompressPlan, + members: &[MemberPlan], + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), +) -> Result> { + let mut results = Records::new(sources.len(), resources)?; + let mut start = 0; + while start < sources.len() { + if advance.is_cancelled() || resources.is_cancelled() { + return Err(Error::Cancelled); + } + if members[start].execution != Execution::WholeMember { + results + .push( + compress_fallback_member( + start, + &sources[start], + integrity[start], + plan, + members[start].workspace, + resources, + advance, + error_context, + ) + .map_err(|error| error_context(start, error))?, + ) + .expect("one compression result per source was admitted"); + start += 1; + continue; + } + let (end, reserved) = whole_member_wave( + members, + start, + crate::parallel::threads(), + resources.memory_limit(), + ); + let (end, reserved) = if let Some(ledger) = &resources.execution { + let slot_bytes = std::mem::size_of::() + + std::mem::size_of::() + + std::mem::size_of::>() + + std::mem::size_of::(); + whole_member_wave_with_slots( + members, + start, + crate::parallel::threads(), + resources.memory_limit(), + slot_bytes as u64 + crate::codec::workspace::RESERVATION_BYTES, + ledger.available(), + ) + } else { + (end, reserved) + }; + // Acquire the entire wave on the coordinator. No dispatched worker + // waits for workspace held by another worker in the same pool. + let _permit = resources + .acquire_cancellable(reserved, plan.dictionary_size, &|| advance.is_cancelled()) + .map_err(|error| error_context(start, error))?; + let jobs = Records::collect((start..end).map(Ok), resources)?; + let unlimited = |jobs| { + run_jobs(jobs, resources, advance, |index, progress| { + compress_whole_member( + index, + &sources[index], + integrity[index], + plan, + resources, + progress, + &Allowance::default(), + ) + .map_err(|error| error_context(index, error)) + }) + }; + let completed = if resources.execution.is_some() { + run_jobs_admitted( + jobs, + resources, + advance, + |index| members[*index].workspace, + |index, worker, progress, allowance| { + compress_whole_member( + index, + &sources[index], + integrity[index], + plan, + worker, + progress, + allowance, + ) + .map_err(|error| error_context(index, error)) + }, + )? + } else { + unlimited(jobs)? + }; + for member in completed { + results.push(member)?; + } + start = end; + } + Ok(results) +} + +/// A contiguous wave fitting both the configured estimate and the worker count. +/// Include an oversized first job so admission reports its real requirement. +fn whole_member_wave( + members: &[MemberPlan], + start: usize, + threads: usize, + limit: u64, +) -> (usize, u64) { + whole_member_wave_with_slots(members, start, threads, limit, 0, u64::MAX) +} + +fn whole_member_wave_with_slots( + members: &[MemberPlan], + start: usize, + max_jobs: usize, + limit: u64, + slot_bytes: u64, + capacity: u64, +) -> (usize, u64) { + let mut reserved = members[start].workspace; + let mut end = start + 1; + while end < members.len() && end - start < max_jobs.max(1) { + if members[end].execution != Execution::WholeMember { + break; + } + let Some(total) = reserved.checked_add(members[end].workspace) else { + break; + }; + if total > limit + || total.saturating_add(slot_bytes.saturating_mul((end - start + 1) as u64)) > capacity + { + break; + } + reserved = total; + end += 1; + } + (end, reserved) +} + +fn compress_whole_member( + index: usize, + source: &EntrySource, + integrity: (u64, u32, [u8; 32]), + plan: &CompressPlan, + resources: &WriterResources, + advance: &dyn CompressionProgress, + allowance: &B, +) -> Result { + let (input_size, _, _) = integrity; + let mut crc = Crc32::new(); + let mut hasher = blake2sp::Hasher::new(); + + let mut packed_spool = Spool::create(resources)?; + let mut stored = input_size == 0; + if !stored { + advance.started(index, input_size); + if advance.is_cancelled() { + return Err(Error::Cancelled); + } + let size = usize::try_from(input_size) + .map_err(|_| Error::InvalidArgument("entry size overflows usize"))?; + let mut data = Buffer::filled(size, 0, allowance)?; + let mut reader = source.open()?; + for chunk in data.chunks_mut(plan.block_size.max(1)) { + if advance.is_cancelled() { + return Err(Error::Cancelled); + } + reader.read_exact(chunk)?; + crc.update(chunk); + hasher.update(chunk); + } + check_source_end(&mut *reader)?; + // The filter search walks the member many times over, so + // encoder positions are scaled down to the member's share + // of that total: many passes, one member's worth of + // progress. + let walk = super::filter_policy_walk_bytes( + &data, + &plan.filter_policy, + plan.algorithm_version, + plan.candidates.len(), + ) + .max(input_size) + .max(1); + let share = + |bytes: u64| (u128::from(bytes) * u128::from(input_size) / u128::from(walk)) as u64; + let mut reported = 0u64; + let mut charged = 0u64; + let mut report = |event| { + let position = match event { + crate::filter_search::EncodeProgress::PassStarted => { + reported = 0; + return !advance.is_cancelled(); + } + crate::filter_search::EncodeProgress::Advanced(position) => position as u64, + }; + let delta = position.saturating_sub(reported); + reported = position; + let target = (charged + delta).min(walk); + let scaled = share(target) - share(charged); + charged = target; + advance.advance(scaled) + }; + let packed = candidates_with_allowance( + &data, + plan.algorithm_version, + &plan.filter_policy, + &plan.candidates, + Some(&mut report), + allowance, + )?; + // An explicitly requested filter is not discarded just + // because the result did not shrink. + stored = should_store_compressed_payload( + data.len() as u64, + packed.len() as u64, + plan.solid, + &plan.filter_policy, + ); + if !stored { + packed_spool.write_all(&packed)?; + } + } + + if input_size == 0 { + advance.started(index, input_size); + if advance.is_cancelled() { + return Err(Error::Cancelled); + } + check_source_end(&mut *source.open()?)?; + } + packed_spool.park(); + if !stored { + source.release(); + } + advance.finished(index, input_size); + Ok(CompressedMember { + input_size, + crc32: crc.finish(), + hash: hasher.finalize(), + store: stored, + packed: packed_spool, + solid_continuation: false, + }) +} + +/// Execute the planned automatic-filter fallback without re-entering planning. +/// Block execution uses only `encode_options`; filter search and the candidate +/// list belong to whole-member execution. Store fallback follows that unfiltered +/// base encoding, while the shared codec settings remain intact. +#[allow(clippy::too_many_arguments)] +fn compress_fallback_member( + index: usize, + source: &EntrySource, + integrity: (u64, u32, [u8; 32]), + plan: &CompressPlan, + required: u64, + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), +) -> Result { + struct Remapped<'a> { + index: usize, + progress: &'a dyn CompressionProgress, + } + impl CompressionProgress for Remapped<'_> { + fn is_cancelled(&self) -> bool { + self.progress.is_cancelled() + } + fn advance(&self, bytes: u64) -> bool { + self.progress.advance(bytes) + } + fn started(&self, _: usize, size: u64) { + self.progress.started(self.index, size); + } + fn finished(&self, _: usize, size: u64) { + self.progress.finished(self.index, size); + } + } + if advance.is_cancelled() || resources.is_cancelled() { + return Err(Error::Cancelled); + } + let mut integrity = [integrity]; + let mut packed = compress_streaming_members( + std::slice::from_ref(source), + &mut integrity, + plan, + required, + resources, + &Remapped { + index, + progress: advance, + }, + &|_, error| error_context(index, error), + )?; + let packed = packed.pop().expect("one fallback member"); + let (input_size, crc32, hash) = integrity[0]; + Ok(CompressedMember { + input_size, + crc32, + hash, + store: should_store_compressed_payload( + input_size, + packed.len(), + false, + &FilterPolicy::None, + ), + packed, + solid_continuation: false, + }) +} + +/// Members with independent dictionaries, interleaved so a batch of small +/// members can still saturate the machine. +#[allow(clippy::too_many_arguments)] +fn compress_independent_members( + sources: &[EntrySource], + integrity: &mut [(u64, u32, [u8; 32])], + plan: &CompressPlan, + batch_capacity: usize, + required: u64, + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), + allowance: &B, +) -> Result> +where + B::Charge: Send, +{ + let mut packed = Records::new(sources.len(), resources)?; + let mut group_start = 0; + while group_start < sources.len() { + let group_capacity = streaming_wave_capacity(batch_capacity, required, plan, resources); + let group_end = group_start + .saturating_add(group_capacity) + .min(sources.len()); + let group = &sources[group_start..group_end]; + let mut streams = Records::collect( + group.iter().enumerate().map(|(offset, source)| { + MemberStream::new( + group_start + offset, + source, + integrity[group_start + offset].0, + resources, + advance, + allowance, + ) + .map_err(|error| error_context(group_start + offset, error)) + }), + resources, + )?; + + let mut histories = Records::collect( + (0..streams.len()).map(|_| Ok(Buffer::new(allowance))), + resources, + )?; + let mut cursor = 0usize; + while streams.iter().any(MemberStream::has_more) { + let wave_capacity = streaming_wave_capacity(batch_capacity, required, plan, resources); + let reserved = required.saturating_mul(wave_capacity as u64); + let _permit = resources + .acquire_cancellable(reserved, plan.dictionary_size, &|| advance.is_cancelled())?; + + let mut jobs = Records::new(wave_capacity, resources)?; + let mut misses = 0usize; + while jobs.len() < wave_capacity && misses < streams.len() { + let stream_count = streams.len(); + let member = cursor; + let stream = &mut streams[member]; + cursor = (cursor + 1) % stream_count; + if !stream.has_more() { + misses += 1; + continue; + } + misses = 0; + + let mut job = BlockJob { + data: Buffer::new(allowance), + history: Buffer::copied(&histories[member], allowance)?, + blocks: Records::new(0, resources)?, + }; + while stream.has_more() && job.data.len() < run_size(plan) { + job.append( + read_block(stream, plan.block_size, advance) + .map_err(|error| error_context(stream.member, error))?, + )?; + job.blocks + .push_growing((member, job.data.len(), !stream.has_more()))?; + } + histories[member].remember(&job.data, plan.encode_options.max_match_distance)?; + jobs.push(job)?; + } + + compress_wave( + jobs, + plan, + required, + &mut streams, + resources, + advance, + error_context, + )?; + } + + for stream in streams { + let slot = &mut integrity[stream.member]; + slot.1 = stream.crc.finish(); + slot.2 = stream.hash.finalize(); + packed.push(stream.packed)?; + } + group_start = group_end; + } + Ok(packed) +} + +/// One dictionary running through every member in order. +#[allow(clippy::too_many_arguments)] +fn compress_solid_chain( + sources: &[EntrySource], + integrity: &mut [(u64, u32, [u8; 32])], + plan: &CompressPlan, + batch_capacity: usize, + required: u64, + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), + allowance: &B, +) -> Result> +where + B::Charge: Send, +{ + let mut streams = Records::collect( + sources.iter().enumerate().map(|(member, source)| { + MemberStream::new( + member, + source, + integrity[member].0, + resources, + advance, + allowance, + ) + .map_err(|error| error_context(member, error)) + }), + resources, + )?; + + let mut history = Buffer::new(allowance); + let mut next = 0usize; + loop { + let wave_capacity = streaming_wave_capacity(batch_capacity, required, plan, resources); + let reserved = required.saturating_mul(wave_capacity as u64); + let _permit = resources + .acquire_cancellable(reserved, plan.dictionary_size, &|| advance.is_cancelled())?; + + // Run boundaries depend on input and dictionary size, never on the + // worker count. Adjacent blocks amortize history copies and seeding. + let mut jobs = Records::new(wave_capacity, resources)?; + while jobs.len() < wave_capacity { + while next < streams.len() && !streams[next].has_more() { + next += 1; + } + if next == streams.len() { + break; + } + let mut job = BlockJob { + data: Buffer::new(allowance), + history: Buffer::copied(&history, allowance)?, + blocks: Records::new(0, resources)?, + }; + while job.data.len() < run_size(plan) { + while next < streams.len() && !streams[next].has_more() { + next += 1; + } + let Some(stream) = streams.get_mut(next) else { + break; + }; + job.append( + read_block(stream, plan.block_size, advance) + .map_err(|error| error_context(stream.member, error))?, + )?; + job.blocks + .push_growing((stream.member, job.data.len(), !stream.has_more()))?; + } + // The outer loop found a stream with data, so this job contains + // at least that stream's next block. + history.remember(&job.data, plan.encode_options.max_match_distance)?; + jobs.push(job)?; + } + + if jobs.is_empty() { + break; + } + compress_wave( + jobs, + plan, + required, + &mut streams, + resources, + advance, + error_context, + )?; + } + + Records::collect( + streams.into_iter().map(|stream| { + let slot = &mut integrity[stream.member]; + slot.1 = stream.crc.finish(); + slot.2 = stream.hash.finalize(); + Ok(stream.packed) + }), + resources, + ) +} + +/// Reads the next block from `stream`, checking the source has not grown. +/// +/// One chunk, then further chunks while the data is not moving, which is the +/// same question [`BlockSplitter`] answers for the buffered writer. Both have +/// to reach the same answer or the same input packs to two different archives. +fn read_block( + stream: &mut MemberStream, + block_size: usize, + progress: &dyn CompressionProgress, +) -> Result> { + if !stream.started { + progress.started(stream.member, stream.input_size); + stream.started = true; + } + let mut data = read_chunk(stream, block_size, progress)?; + let mut splitter = BlockSplitter::new(); + splitter.accept(&data); + while stream.has_more() { + let next = read_chunk(stream, block_size, progress)?; + if !splitter.try_extend(&next) { + // Deciding needs the chunk in hand, so this reads one further than + // it keeps. Hand it back for the next block rather than seeking + // backwards, which an `EntryReader` cannot always do. + stream.pushback = next; + break; + } + data.extend_from_slice(&next) + .map_err(Into::::into)?; + } + Ok(data) +} + +/// Reads one chunk, preferring anything a previous read put back. +fn read_chunk( + stream: &mut MemberStream, + block_size: usize, + progress: &dyn CompressionProgress, +) -> Result> { + if progress.is_cancelled() { + return Err(Error::Cancelled); + } + if !stream.pushback.is_empty() { + return Ok({ + let empty = Buffer::new(&stream.pushback.allowance()); + std::mem::replace(&mut stream.pushback, empty) + }); + } + let wanted = usize::try_from(stream.remaining.min(block_size as u64)) + .map_err(|_| Error::InvalidArgument("RAR 5 block size overflows usize"))?; + let mut data = Buffer::filled(wanted, 0u8, &stream.pushback.allowance())?; + // Solid planning can retain every member, but only the current input + // needs a reader. Release it at EOF, even when a block has pushback. + if stream.reader.is_none() { + stream.reader = Some(stream.source.open()?); + } + let reader = stream.reader.as_mut().expect("opened above"); + reader.read_exact(&mut data)?; + stream.crc.update(&data); + stream.hash.update(&data); + stream.remaining -= wanted as u64; + if stream.remaining == 0 { + check_source_end(&mut **reader)?; + stream.reader = None; + } + Ok(data) +} + +fn check_source_end(reader: &mut dyn Read) -> Result<()> { + let mut trailing = [0u8; 1]; + if reader.read(&mut trailing)? != 0 { + return Err(Error::SourceChanged( + "entry source size changed while compressing", + )); + } + Ok(()) +} + +/// Compresses a wave of blocks in parallel, then appends the results to their +/// members in job order so output does not depend on scheduling. +fn compress_wave( + jobs: Records>, + plan: &CompressPlan, + _required: u64, + streams: &mut [MemberStream], + resources: &WriterResources, + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), +) -> Result<()> +where + B::Charge: Send, +{ + if resources.execution.is_some() { + let jobs = prepare_block_jobs::(jobs, resources)?; + let packed_runs = run_jobs_admitted( + jobs, + resources, + advance, + |_| _required, + |job, _, progress, allowance| encode_block_job(job, plan, progress, allowance), + )?; + return append_packed_runs(packed_runs, plan, streams, advance, error_context); + } + let jobs = prepare_block_jobs::(jobs, resources)?; + let packed_runs = run_jobs(jobs, resources, advance, |job, progress| { + let allowance = job.0.data.allowance(); + encode_block_job(job, plan, progress, &allowance) + })?; + append_packed_runs(packed_runs, plan, streams, advance, error_context) +} + +type PackedBlocks = Records<(usize, Buffer, bool)>; +type PreparedBlockJob = (BlockJob, Records<(usize, bool)>, PackedBlocks); + +fn prepare_block_jobs( + jobs: Records>, + resources: &WriterResources, +) -> Result>> { + // Allocate boundary and result descriptors before dispatch. Worker outputs + // carry their scoped codec charge; descriptor arrays keep the root charge. + Records::collect( + jobs.into_iter().map(|job| { + let boundaries = Records::collect( + job.blocks.iter().map(|&(_, end, last)| Ok((end, last))), + resources, + )?; + let output = Records::new(job.blocks.len(), resources)?; + Ok((job, boundaries, output)) + }), + resources, + ) +} + +fn encode_block_job( + (job, boundaries, mut output): PreparedBlockJob, + plan: &CompressPlan, + progress: &dyn CompressionProgress, + allowance: &C, +) -> Result> { + let mut block_done = |bytes: usize| progress.advance(bytes as u64); + let packed = streaming_blocks_with_allowance( + &job.data, + &job.history, + &boundaries, + plan.algorithm_version, + plan.encode_options, + Some(&mut block_done), + allowance, + )?; + for ((member, _, last), packed) in job.blocks.into_iter().zip(packed) { + output.push((member, packed, last))?; + } + Ok(output) +} + +fn append_packed_runs( + packed_runs: Records>, + plan: &CompressPlan, + streams: &mut [MemberStream], + advance: &dyn CompressionProgress, + error_context: &(dyn Fn(usize, Error) -> Error + Sync), +) -> Result<()> { + // A solid wave can cover thousands of tiny members. Keep only the spool + // currently being appended open, rather than one descriptor per member. + let mut previous: Option = None; + for (member, packed, last) in packed_runs.into_iter().flatten() { + if previous != Some(member) { + if let Some(previous) = previous { + streams[previous].packed.park(); + } + previous = Some(member); + } + streams[member] + .packed + .write_all(&packed) + .map_err(|error| error_context(streams[member].member, error.into()))?; + if last { + let stream = &streams[member]; + // A completed block run always came from `has_more`, so an empty + // member cannot reach this branch. + if !should_store_compressed_payload( + stream.input_size, + stream.packed.len(), + plan.solid, + &plan.filter_policy, + ) { + stream.source.release(); + } + advance.finished(stream.member, stream.input_size); + } + } + // Both wave builders dispatch only non-empty jobs, and every job emits + // one packed block for each recorded boundary. + let previous = previous.expect("compression wave contains a packed block"); + streams[previous].packed.park(); + Ok(()) +} + +/// The compression-info vint for a member, including its solid flag. +pub(super) fn member_compression_info( + plan: &CompressPlan, + member: &CompressedMember, +) -> Result { + compression_info( + plan.algorithm_version, + if member.store { 0 } else { plan.method }, + plan.dictionary_size, + member.solid_continuation, + ) +} + +#[cfg(test)] +pub(super) fn compress_members_reporting( + sources: &[EntrySource], + plan: CompressPlan, + resources: &WriterResources, + advance: &dyn CompressionProgress, +) -> Result> { + compress_members_with_context(sources, &plan, resources, advance, &|_, error| error) + .map(|members| members.into_iter().collect()) +} + +#[cfg(test)] +mod tests { + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn stream_creation_failure_keeps_member_context_before_opening_source() { + let scratch = crate::scratch::case("stream-creation-context"); + for solid in [false, true] { + let resources = WriterResources::new(128 * 1024 * 1024) + .with_max_memory_bytes(128 * 1024 * 1024) + .with_temp_dir(scratch.join("missing-directory")); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let source = EntrySource::from_opener(1, || { + panic!("spool creation must fail before opening the source") + }); + let error = compress_members_with_context( + &[source], + &plan, + &resources, + &|_| true, + &|index, error| { + assert_eq!(index, 0); + error.at_entry(b"unopened.bin".to_vec(), "compressing") + }, + ) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((b"unopened.bin".as_slice(), "compressing")) + ); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(resources.managed_memory_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn cancellation_after_wave_check_keeps_admission_error_context() { + struct CancelAfterObservation(crate::WriteCancellation); + impl CompressionProgress for CancelAfterObservation { + fn advance(&self, _: u64) -> bool { + !self.0.is_cancelled() + } + fn is_cancelled(&self) -> bool { + // Model cancellation arriving just after the observation. + let observed = self.0.is_cancelled(); + self.0.cancel(); + observed + } + } + let scratch = crate::scratch::case("wave-admission-context"); + let resources = WriterResources::new(128 * 1024 * 1024).with_temp_dir(&*scratch); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![options].into(), + }; + let execution = ExecutionPlan::with_resources(&plan, [32].into_iter(), &resources).unwrap(); + let ExecutionPlan::IndependentMembers(members) = execution else { + panic!("expected whole-member plan"); + }; + assert_eq!(members[0].execution, Execution::WholeMember); + let source = EntrySource::from_opener(32, || { + panic!("cancelled admission must precede source opening") + }); + let progress = CancelAfterObservation(crate::WriteCancellation::new()); + let error = compress_members_whole( + &[source], + &[(32, 0, [0; 32])], + &plan, + &members, + &resources, + &progress, + &|index, error| { + assert_eq!(index, 0); + error.at_entry(b"waiting.bin".to_vec(), "compressing") + }, + ) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::Cancelled); + assert_eq!( + error.entry_context(), + Some((b"waiting.bin".as_slice(), "compressing")) + ); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[test] + fn streaming_compression_releases_each_refused_preparation_allocation() { + use std::sync::atomic::Ordering; + + let scratch = crate::scratch::case("streaming-preparation-refusals"); + let sources = [EntrySource::from_bytes(vec![b'A'; 2048])]; + let options = EncodeOptions::new(8); + for solid in [false, true] { + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 1024, + solid, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let run = |resources: &WriterResources| -> Result<()> { + let packed = + compress_members_reporting(&sources, plan.clone(), resources, &|_| true)?; + assert_eq!(packed.len(), 1); + Ok(()) + }; + let (resources, attempts) = WriterResources::default() + .with_temp_dir(&*scratch) + .refuse_preparation_growth_at(usize::MAX); + run(&resources).unwrap(); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 5); + assert!(count < 100); + assert_eq!(resources.preparation_in_use(), 0); + + for index in 0..count { + let (resources, _) = WriterResources::default() + .with_temp_dir(&*scratch) + .refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "solid mode {solid}, admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + } + + #[test] + fn admitted_whole_member_worker_propagates_source_failure() { + let options = EncodeOptions::new(8); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 1024, + solid: false, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options, options].into(), + }; + let source = EntrySource::from_opener(1024, || { + Err(std::io::Error::other("injected worker source failure").into()) + }); + let resources = + WriterResources::new(70 * 1024 * 1024).with_max_memory_bytes(70 * 1024 * 1024); + let error = compress_members_reporting(&[source], plan, &resources, &|_| true) + .err() + .expect("the admitted worker must report its source failure"); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(resources.execution.as_ref().unwrap().used(), 0); + } + + #[test] + fn stored_member_descriptor_refusal_releases_preparation_charge() { + let options = EncodeOptions::new(8); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 128 * 1024, + block_size: 4096, + solid: false, + method: 0, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let source = EntrySource::from_bytes(b"payload".to_vec()); + let limit = std::mem::size_of::<(u64, u32, [u8; 32])>() as u64; + let resources = WriterResources::default().with_max_preparation_bytes(limit); + let error = compress_members_reporting(&[source], plan, &resources, &|_| true) + .err() + .expect("the stored spool descriptors must exceed the integrity record"); + assert!(matches!( + error, + Error::WriterPreparationLimitExceeded { + limit: actual_limit, + used, + .. + } if actual_limit == limit && used == limit + )); + drop(Records::::new(limit as usize, &resources).unwrap()); + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn whole_member_cancellation_after_start_does_not_open_source() { + use std::sync::atomic::{AtomicBool, Ordering}; + + struct StopAfterStart(AtomicBool); + impl CompressionProgress for StopAfterStart { + fn advance(&self, _: u64) -> bool { + !self.is_cancelled() + } + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::SeqCst) + } + fn started(&self, _: usize, _: u64) { + self.0.store(true, Ordering::SeqCst); + } + } + + let scratch = crate::scratch::case("whole-member-cancel-after-start"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let options = EncodeOptions::new(8); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 128 * 1024, + block_size: 4096, + solid: false, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + for size in [0, 16] { + let source = EntrySource::from_opener(size, || { + panic!("cancelled compression must not open the source") + }); + let progress = StopAfterStart(AtomicBool::new(false)); + assert!(matches!( + compress_whole_member( + 0, + &source, + (size, 0, [0; 32]), + &plan, + &resources, + &progress, + &Allowance::default(), + ), + Err(Error::Cancelled) + )); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn whole_member_cancellation_between_waves_does_not_open_next_source() { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + struct StopAfterFirst { + finished: AtomicBool, + checks_after_finish: AtomicUsize, + resource_cancellation: Option, + } + impl CompressionProgress for StopAfterFirst { + fn advance(&self, _: u64) -> bool { + !self.is_cancelled() + } + fn is_cancelled(&self) -> bool { + // Let the completed wave return its result, then cancel at + // the coordinator's check before admitting the next wave. + if !self.finished.load(Ordering::SeqCst) { + return false; + } + if self.checks_after_finish.fetch_add(1, Ordering::SeqCst) == 0 { + return false; + } + if let Some(token) = &self.resource_cancellation { + token.cancel(); + false + } else { + true + } + } + fn finished(&self, index: usize, _: u64) { + if index == 0 { + self.finished.store(true, Ordering::SeqCst); + } + } + } + + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![options].into(), + }; + let scratch = crate::scratch::case("whole-member-cancel-between-waves"); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .unwrap(); + for resource_cancellation in [false, true] { + let first = EntrySource::from_bytes(b"first".to_vec()); + let second = EntrySource::from_opener(6, || { + panic!("cancellation between waves must not open the next source") + }); + let token = resource_cancellation.then(crate::WriteCancellation::new); + let mut resources = WriterResources::new(80 * 1024 * 1024).with_temp_dir(&*scratch); + if let Some(token) = &token { + resources = resources.with_cancellation(token.clone()); + } + let progress = StopAfterFirst { + finished: AtomicBool::new(false), + checks_after_finish: AtomicUsize::new(0), + resource_cancellation: token.clone(), + }; + let result = pool.install(|| { + compress_members_reporting(&[first, second], plan.clone(), &resources, &progress) + }); + assert!( + matches!(result, Err(Error::Cancelled)), + "{:?}", + result.err() + ); + if let Some(token) = &token { + assert!(token.is_cancelled()); + } else { + assert!(progress.is_cancelled()); + } + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + struct Cancelled; + + impl CompressionProgress for Cancelled { + fn advance(&self, _: u64) -> bool { + false + } + + fn is_cancelled(&self) -> bool { + true + } + } + + #[test] + fn cancellation_prevents_worker_dispatch_and_empty_source_open() { + let resources = WriterResources::default(); + let jobs = Records::collect([1u8].into_iter().map(Ok), &resources).unwrap(); + assert_eq!( + run_jobs(jobs, &resources, &Cancelled, |_, _| -> Result { + panic!("cancelled job was dispatched") + }) + .err() + .unwrap(), + Error::Cancelled + ); + + let ledger = crate::codec::workspace::Allowance::limited(1024 * 1024); + let admitted = resources.with_execution_allowance(ledger.clone()); + let jobs = Records::collect([1u8].into_iter().map(Ok), &admitted).unwrap(); + assert_eq!( + run_jobs_admitted( + jobs, + &admitted, + &Cancelled, + |_| 1, + |_, _, _, _| -> Result { panic!("cancelled job was admitted") }, + ) + .err() + .unwrap(), + Error::Cancelled + ); + assert_eq!(ledger.used(), 0); + + let cancellation = crate::WriteCancellation::new(); + let admitted = WriterResources::default() + .with_cancellation(cancellation.clone()) + .with_execution_allowance(ledger.clone()); + let jobs = Records::collect([1u8].into_iter().map(Ok), &admitted).unwrap(); + cancellation.cancel(); + assert_eq!( + run_jobs_admitted( + jobs, + &admitted, + &|_| true, + |_| 1, + |_, _, _, _| -> Result { panic!("cancelled resource dispatched a job") }, + ) + .err() + .unwrap(), + Error::Cancelled + ); + assert_eq!(ledger.used(), 0); + + let source = EntrySource::from_opener(0, || panic!("empty source was opened")); + assert_eq!( + MemberStream::new( + 0, + &source, + 0, + &WriterResources::default(), + &Cancelled, + &crate::codec::workspace::Allowance::default(), + ) + .err() + .unwrap(), + Error::Cancelled + ); + } + + #[test] + fn a_codec_failure_takes_precedence_over_sibling_cancellation() { + let resources = WriterResources::default(); + let jobs = Records::collect( + [ + (None::, Some(Err::(Error::Cancelled))), + (None, Some(Err(Error::InvalidArgument("codec failure")))), + ] + .into_iter() + .map(Ok), + &resources, + ) + .unwrap(); + let output = Records::new(2, &resources).unwrap(); + assert_eq!( + complete_jobs( + jobs, + output, + &resources, + &Cancelled, + |_, _, _| -> Result { + panic!("cancelled worker overwrote its recorded result") + } + ) + .err() + .unwrap(), + Error::InvalidArgument("codec failure") + ); + + let jobs = Records::collect( + [ + ( + None::, + Some(Err::(Error::InvalidArgument("first"))), + ), + (None, Some(Err(Error::InvalidArgument("second")))), + ] + .into_iter() + .map(Ok), + &resources, + ) + .unwrap(); + let output = Records::new(2, &resources).unwrap(); + assert_eq!( + complete_jobs( + jobs, + output, + &resources, + &Cancelled, + |_, _, _| -> Result { + panic!("cancelled worker overwrote its recorded result") + } + ) + .err() + .unwrap(), + Error::InvalidArgument("first") + ); + } + + #[test] + fn compression_guards_sources_before_start_and_fallback() { + struct Untouched; + impl crate::streaming::SourceFactory for Untouched { + fn len(&self) -> Result { + panic!("cancelled compression measured a source") + } + fn open(&self) -> Result> { + panic!("cancelled compression opened a source") + } + } + let options = EncodeOptions::new(8); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 128 * 1024, + block_size: 4096, + solid: false, + method: 0, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let resources = WriterResources::default(); + let source = EntrySource::from_factory(Untouched); + assert_eq!( + compress_members_with_context(&[source], &plan, &resources, &Cancelled, &|_, error| { + error + }) + .err() + .unwrap(), + Error::Cancelled + ); + assert_eq!( + compress_members_with_context( + &[EntrySource::from_bytes(b"payload".to_vec())], + &plan, + &resources, + &|_| false, + &|_, error| error, + ) + .err() + .unwrap(), + Error::Cancelled + ); + let token = crate::WriteCancellation::new(); + token.cancel(); + let resources = WriterResources::default().with_cancellation(token); + assert_eq!( + compress_members_with_context( + &[EntrySource::from_factory(Untouched)], + &plan, + &resources, + &|_| true, + &|_, error| error, + ) + .err() + .unwrap(), + Error::Cancelled + ); + let running = |_: u64| true; + for (resources, progress) in [ + ( + WriterResources::default(), + &Cancelled as &dyn CompressionProgress, + ), + (resources, &running as &dyn CompressionProgress), + ] { + assert_eq!( + compress_fallback_member( + 0, + &EntrySource::from_factory(Untouched), + (7, 0, [0; 32]), + &plan, + 0, + &resources, + progress, + &|_, error| error, + ) + .err() + .unwrap(), + Error::Cancelled + ); + } + } + + #[test] + fn whole_waves_admit_the_sum_without_crossing_fallbacks_or_overflowing() { + let whole = |workspace| MemberPlan { + execution: Execution::WholeMember, + workspace, + }; + let members = [whole(60), whole(40), whole(1), whole(u64::MAX)]; + assert_eq!(whole_member_wave(&members, 0, 4, 100), (2, 100)); + assert_eq!(whole_member_wave(&members, 0, 1, 100), (1, 60)); + assert_eq!(whole_member_wave(&members, 0, 4, 59), (1, 60)); + assert_eq!(whole_member_wave(&members, 2, 4, u64::MAX), (3, 1)); + let fallback = MemberPlan { + execution: Execution::Blocks { + fallback: super::super::plan::FallbackReason::AutomaticFilterWorkspace { + required: 100, + limit: 99, + }, + }, + workspace: 10, + }; + assert_eq!( + whole_member_wave(&[whole(0), fallback, whole(0)], 0, 4, 0), + (1, 0) + ); + } + + #[test] + fn stream_read_admission_precedes_open_and_pushback_keeps_its_charge() { + use std::sync::{atomic::AtomicUsize, Arc}; + let scratch = crate::scratch::case("stream-read-allowance"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let opens = Arc::new(AtomicUsize::new(0)); + let source = EntrySource::from_opener(8, { + let opens = opens.clone(); + move || { + opens.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(std::io::Cursor::new(*b"abcdefgh"))) + } + }); + let allowance = Allowance::limited(3); + let mut stream = + MemberStream::new(0, &source, 8, &resources, &|_| true, &allowance).unwrap(); + assert_eq!( + read_chunk(&mut stream, 4, &|_| true).unwrap_err().kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(opens.load(Ordering::Relaxed), 0); + assert_eq!(stream.remaining, 8); + assert_eq!(allowance.used(), 0); + drop(stream); + + let allowance = Allowance::limited(8); + let mut stream = + MemberStream::new(0, &source, 8, &resources, &|_| true, &allowance).unwrap(); + stream.pushback = read_chunk(&mut stream, 4, &|_| true).unwrap(); + assert_eq!(allowance.used(), 4); + let first = read_chunk(&mut stream, 4, &|_| true).unwrap(); + assert_eq!(&*first, b"abcd"); + assert_eq!(allowance.used(), 4); + assert_eq!(stream.remaining, 4); + let last = read_chunk(&mut stream, 4, &|_| true).unwrap(); + assert_eq!(&*last, b"efgh"); + assert_eq!(allowance.used(), 8); + assert!(stream.reader.is_none()); + drop(stream); + assert_eq!(allowance.used(), 8); + drop((first, last)); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn streaming_writer_allowance_covers_assembly_encoding_and_failure_cleanup() { + let scratch = crate::scratch::case("stream-writer-allowance"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let data: Vec = (0..8192).map(|i| ((i * 71) % 251) as u8).collect(); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + for solid in [false, true] { + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 1024, + solid, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let sources = [ + EntrySource::from_bytes(data.clone()), + EntrySource::from_bytes(data[..2048].to_vec()), + EntrySource::from_bytes(Vec::new()), + ]; + let initial = [(8192, 0, [0; 32]), (2048, 0, [0; 32]), (0, 0, [0; 32])]; + let mut expected_integrity = initial; + let encode = if solid { + compress_solid_chain:: + } else { + compress_independent_members:: + }; + let expected = encode( + &sources, + &mut expected_integrity, + &plan, + 2, + 1, + &resources, + &|_| true, + &|_, error| error, + &Allowance::default(), + ) + .unwrap(); + let payloads = |spools: Records| -> Vec> { + spools + .into_iter() + .map(|mut spool| { + let mut bytes = Vec::new(); + spool.copy_to(&mut bytes).unwrap(); + bytes + }) + .collect() + }; + let expected = payloads(expected); + let mut refusals = 0; + let mut successes = 0; + for limit in [0, 1023, 2048, 8192, 32768, 131072, 524288, 4 * 1024 * 1024] { + let allowance = Allowance::limited(limit); + let mut integrity = initial; + let encode = if solid { + compress_solid_chain:: + } else { + compress_independent_members:: + }; + let result = encode( + &sources, + &mut integrity, + &plan, + 2, + 1, + &resources, + &|_| { + assert!(allowance.used() > 0); + true + }, + &|_, error| error, + &allowance, + ); + match result { + Ok(spools) => { + successes += 1; + assert_eq!(payloads(spools), expected); + assert_eq!(integrity, expected_integrity); + } + Err(error) => { + refusals += 1; + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit, "{error}"); + } + } + assert_eq!(allowance.used(), 0, "solid={solid}, limit={limit}"); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + assert!(refusals >= 3 && successes > 0); + + let allowance = Allowance::limited(4 * 1024 * 1024); + let encode = if solid { + compress_solid_chain:: + } else { + compress_independent_members:: + }; + let mut integrity = initial; + let result = encode( + &sources, + &mut integrity, + &plan, + 2, + 1, + &resources, + &|_| false, + &|_, error| error, + &allowance, + ); + assert!(matches!(result, Err(Error::Cancelled))); + assert_eq!(allowance.used(), 0); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn bounded_streaming_history_survives_multiple_waves_and_member_boundaries() { + let scratch = crate::scratch::case("stream-history-allowance"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let size = crate::codec::rar50::MAX_LZ_BLOCK_SIZE + 4096; + let data: Vec = (0..size).map(|i| ((i * 71) % 251) as u8).collect(); + let sources = [ + EntrySource::from_bytes(data.clone()), + EntrySource::from_bytes(data), + ]; + let options = EncodeOptions::new(8).with_max_match_distance(65536); + for solid in [false, true] { + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let mut expected_integrity = [(size as u64, 0, [0; 32]); 2]; + let mut actual_integrity = expected_integrity; + let encode = if solid { + compress_solid_chain:: + } else { + compress_independent_members:: + }; + let expected = encode( + &sources, + &mut expected_integrity, + &plan, + 1, + 1, + &resources, + &|_| true, + &|_, error| error, + &Allowance::default(), + ) + .unwrap(); + let allowance = Allowance::limited(64 * 1024 * 1024); + let encode = if solid { + compress_solid_chain:: + } else { + compress_independent_members:: + }; + let actual = encode( + &sources, + &mut actual_integrity, + &plan, + 2, + 1, + &resources, + &|_| true, + &|_, error| error, + &allowance, + ) + .unwrap(); + assert_eq!(actual_integrity, expected_integrity); + assert_eq!(allowance.used(), 0); + for (mut expected, mut actual) in expected.into_iter().zip(actual) { + let mut expected_bytes = Vec::new(); + let mut actual_bytes = Vec::new(); + expected.copy_to(&mut expected_bytes).unwrap(); + actual.copy_to(&mut actual_bytes).unwrap(); + assert_eq!(actual_bytes, expected_bytes); + } + } + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn streaming_ledger_covers_repeated_waves_and_retained_history() { + use std::sync::atomic::AtomicU64; + let scratch = crate::scratch::case("stream-ledger-waves"); + let size = crate::codec::rar50::MAX_LZ_BLOCK_SIZE * 2 + 4096; + let data: Vec = (0..size).map(|i| ((i * 71) % 251) as u8).collect(); + let sources = [ + EntrySource::from_bytes(data.clone()), + EntrySource::from_bytes(data), + ]; + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + for (solid, filter_policy) in [ + (false, FilterPolicy::None), + (true, FilterPolicy::None), + (false, FilterPolicy::Auto), + ] { + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid, + method: 1, + filter_policy, + candidates: vec![options].into(), + }; + let required = super::super::streaming_lz_workspace( + plan.dictionary_size, + crate::codec::rar50::MAX_LZ_BLOCK_SIZE, + false, + ); + // Automatic mode is forced through its existing streaming fallback. + let max_workers = if plan.filter_policy == FilterPolicy::Auto { + 1 + } else { + 2 + }; + let resources = WriterResources::new(required * max_workers).with_temp_dir(&*scratch); + let collect = |members: Records| -> Vec<_> { + members + .into_iter() + .map(|mut member| { + let mut bytes = Vec::new(); + member.packed.copy_to(&mut bytes).unwrap(); + ( + bytes, + member.crc32, + member.hash, + member.store, + member.solid_continuation, + ) + }) + .collect() + }; + let expected = collect( + pool.install(|| { + compress_members_with_context(&sources, &plan, &resources, &|_| true, &|_, e| e) + }) + .unwrap(), + ); + for workers in 1..=max_workers { + let limit = required * workers + 32 * 1024 * 1024; + let ledger = Allowance::limited(limit); + let bounded = resources.clone().with_execution_allowance(ledger.clone()); + let peak = AtomicU64::new(0); + let progress = |_| { + let used = ledger.used(); + assert!(used <= limit); + peak.fetch_max(used, Ordering::Relaxed); + true + }; + let actual = pool + .install(|| { + compress_members_with_context( + &sources, + &plan, + &bounded, + &progress, + &|_, e| e, + ) + }) + .unwrap(); + assert!(peak.load(Ordering::Relaxed) >= required * workers); + // Native spools retain their path storage alongside result descriptors. + assert!( + ledger.used() + > (sources.len() * std::mem::size_of::()) as u64 + ); + assert_eq!(collect(actual), expected); + assert_eq!(ledger.used(), 0); + assert_eq!(bounded.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + } + + #[test] + fn streaming_ledger_releases_assembly_workers_and_spools_on_failure() { + use std::sync::{atomic::AtomicUsize, Arc}; + let scratch = crate::scratch::case("stream-ledger-failure"); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + for solid in [false, true] { + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 4096, + solid, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let ledger = Allowance::limited(256 * 1024 * 1024); + let resources = WriterResources::default() + .with_temp_dir(&*scratch) + .with_execution_allowance(ledger.clone()); + let opens = Arc::new(AtomicUsize::new(0)); + let failed_source = EntrySource::from_opener(4096, { + let opens = opens.clone(); + let ledger = ledger.clone(); + move || { + assert!(ledger.used() >= 4096); + opens.fetch_add(1, Ordering::Relaxed); + Err(std::io::Error::other("injected source failure").into()) + } + }); + let result = compress_members_with_context( + &[failed_source], + &plan, + &resources, + &|_| true, + &|_, e| e, + ); + assert!(result.is_err()); + assert_eq!(opens.load(Ordering::Relaxed), 1); + assert_eq!(ledger.used(), 0); + let sources = [EntrySource::from_bytes(vec![7; 8192])]; + let too_small = Allowance::limited(16 * 1024 * 1024); + let constrained = resources + .clone() + .with_execution_allowance(too_small.clone()); + assert_eq!( + compress_members_with_context(&sources, &plan, &constrained, &|_| true, &|_, e| e,) + .err() + .unwrap() + .kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(too_small.used(), 0); + let result = + compress_members_with_context(&sources, &plan, &resources, &|_| false, &|_, e| e); + assert!(matches!(result, Err(Error::Cancelled))); + assert_eq!(ledger.used(), 0); + let constrained = resources.clone().with_max_spool_bytes(1); + assert!(compress_members_with_context( + &sources, + &plan, + &constrained, + &|_| true, + &|_, e| e + ) + .is_err()); + assert_eq!(ledger.used(), 0); + drop( + compress_members_with_context(&sources, &plan, &resources, &|_| true, &|_, e| e) + .unwrap(), + ); + assert_eq!(ledger.used(), 0); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn stored_ledger_admits_read_buffer_before_opening_and_releases_it() { + use std::sync::{atomic::AtomicUsize, Arc}; + let scratch = crate::scratch::case("stored-ledger"); + let options = EncodeOptions::new(8); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid: false, + method: 0, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let opens = Arc::new(AtomicUsize::new(0)); + for limit in [32768, 131072] { + let ledger = Allowance::limited(limit); + let resources = WriterResources::default() + .with_temp_dir(&*scratch) + .with_execution_allowance(ledger.clone()); + let source = EntrySource::from_opener(8, { + let opens = opens.clone(); + let ledger = ledger.clone(); + move || { + assert!(ledger.used() >= 65536); + opens.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(std::io::Cursor::new([7u8; 8]))) + } + }); + let result = + compress_members_with_context(&[source], &plan, &resources, &|_| true, &|_, e| e); + if limit < 65536 { + assert_eq!( + result.err().unwrap().kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(opens.load(Ordering::Relaxed), 0); + } else { + let result = result.unwrap(); + assert!(result[0].store); + assert_eq!(result[0].crc32, crate::crc32::crc32(&[7u8; 8])); + drop(result); + assert_eq!(opens.load(Ordering::Relaxed), 1); + } + assert_eq!(ledger.used(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn streaming_source_failure_releases_admitted_buffers() { + let scratch = crate::scratch::case("stream-source-failure-allowance"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + for bytes in [b"short".as_slice(), b"too long for declared size"] { + let source = + EntrySource::from_opener(8, move || Ok(Box::new(std::io::Cursor::new(bytes)))); + let allowance = Allowance::limited(128); + let mut stream = + MemberStream::new(0, &source, 8, &resources, &|_| true, &allowance).unwrap(); + let error = read_block(&mut stream, 4, &|_| true).unwrap_err(); + assert!(matches!(error, Error::Io(_) | Error::SourceChanged(_))); + drop(stream); + assert_eq!(allowance.used(), 0); + } + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn admitted_workers_share_one_ledger_without_racing_for_spare_capacity() { + use crate::streaming::preparation::Bytes; + use std::sync::{atomic::AtomicU64, Barrier}; + let ledger = Allowance::limited(65536); + let resources = WriterResources::default().with_execution_allowance(ledger.clone()); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + for fail in [false, true] { + let barrier = Barrier::new(2); + let admitted = AtomicU64::new(0); + let jobs = Records::collect((0..2).map(Ok), &resources).unwrap(); + let result = pool.install(|| { + run_jobs_admitted( + jobs, + &resources, + &|_| true, + |_| 128, + |index, worker, _, allowance| { + if index == 0 { + admitted.store(ledger.used(), Ordering::Relaxed); + } + barrier.wait(); + let prepared = Bytes::zeroed(32, worker)?; + let bytes = Buffer::filled(64, index as u8, allowance)?; + assert_eq!(ledger.used(), admitted.load(Ordering::Relaxed)); + barrier.wait(); + if fail && index == 0 { + Buffer::filled(33, 0u8, allowance)?; + } + Ok((prepared, bytes)) + }, + ) + }); + if fail { + assert_eq!( + result.err().unwrap().kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(ledger.used(), 0); + } else { + let mut output = result.unwrap(); + let first = output.pop().unwrap(); + let second = output.pop().unwrap(); + drop(output); + assert_eq!( + ledger.used(), + 192 + 2 * crate::codec::workspace::RESERVATION_BYTES + ); + drop(first); + assert_eq!( + ledger.used(), + 96 + crate::codec::workspace::RESERVATION_BYTES + ); + drop(second); + assert_eq!(ledger.used(), 0); + } + } + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn whole_member_ledger_selects_fitting_waves_before_opening_sources() { + use std::sync::{atomic::AtomicUsize, Arc, Barrier}; + let scratch = crate::scratch::case("whole-member-ledger"); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 1024, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![options].into(), + }; + let data: Arc<[u8]> = (0..1024u32) + .flat_map(|n| n.to_le_bytes()) + .collect::>() + .into(); + let required = whole_member_workspace(data.len() as u64, &plan); + let expected = candidates_with_allowance( + &data, + 0, + &plan.filter_policy, + &plan.candidates, + None, + &Allowance::default(), + ) + .unwrap(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + for workers in [1, 2] { + let ledger = Allowance::limited(required * workers + 65536); + let resources = WriterResources::new(required * 2) + .with_temp_dir(&*scratch) + .with_execution_allowance(ledger.clone()); + let opens = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(workers as usize)); + let sources: Vec<_> = (0..4) + .map(|_| { + let data = data.clone(); + let ledger = ledger.clone(); + let opens = opens.clone(); + let barrier = barrier.clone(); + EntrySource::from_opener(data.len() as u64, move || { + assert!(ledger.used() >= required * workers); + opens.fetch_add(1, Ordering::Relaxed); + barrier.wait(); + Ok(Box::new(std::io::Cursor::new(data.clone()))) + }) + }) + .collect(); + let result = pool + .install(|| { + compress_members_with_context( + &sources, + &plan, + &resources, + &|_| true, + &|_, error| error, + ) + }) + .unwrap(); + assert_eq!(opens.load(Ordering::Relaxed), 4); + assert!(ledger.used() > (4 * std::mem::size_of::()) as u64); + for mut member in result { + let mut bytes = Vec::new(); + member.packed.copy_to(&mut bytes).unwrap(); + assert_eq!(&*expected, bytes); + } + assert_eq!(ledger.used(), 0); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn coordinator_slots_are_admitted_before_callbacks_and_results_stay_charged() { + use std::sync::atomic::AtomicUsize; + let calls = AtomicUsize::new(0); + let limited = WriterResources::default().with_max_preparation_bytes(16); + let jobs = Records::collect([Ok(1u64), Ok(2)].into_iter(), &limited).unwrap(); + assert!(matches!( + run_jobs(jobs, &limited, &|_| true, |job, _| { + calls.fetch_add(1, Ordering::Relaxed); + Ok(job) + }), + Err(Error::WriterPreparationLimitExceeded { .. }) + )); + assert_eq!(calls.load(Ordering::Relaxed), 0); + drop(Records::::new(16, &limited).unwrap()); + + let resources = WriterResources::default().with_max_preparation_bytes(65536); + for _ in 0..8 { + let jobs = Records::collect([Ok(1u64), Ok(2)].into_iter(), &resources).unwrap(); + let output = run_jobs(jobs, &resources, &|_| true, |job, _| Ok(job * 2)).unwrap(); + assert_eq!(&*output, &[2, 4]); + assert!(matches!( + Records::::new(65536, &resources), + Err(Error::WriterPreparationLimitExceeded { used: 16, .. }) + )); + drop(output); + drop(Records::::new(65536, &resources).unwrap()); + } + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn failed_wave_skips_queued_sources_and_releases_successful_siblings() { + use std::sync::atomic::AtomicUsize; + struct Retained<'a>(&'a AtomicUsize); + impl Drop for Retained<'_> { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + let scratch = crate::scratch::case("coordinator-failure"); + let resources = WriterResources::new(100) + .with_temp_dir(&*scratch) + .with_max_preparation_bytes(65536); + let calls = AtomicUsize::new(0); + let drops = AtomicUsize::new(0); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .unwrap(); + let permit = resources.acquire(100, 0).unwrap(); + let jobs = Records::collect((0..3).map(Ok), &resources).unwrap(); + let result = pool.install(|| { + run_jobs(jobs, &resources, &|_| true, |job, _| { + calls.fetch_add(1, Ordering::Relaxed); + assert_eq!(resources.workspace_in_use(), 100); + if job == 1 { + return Err(Error::SourceChanged("test sibling failure")); + } + Ok((Retained(&drops), Spool::create(&resources)?)) + }) + }); + assert!(matches!( + result, + Err(Error::SourceChanged("test sibling failure")) + )); + assert_eq!(calls.load(Ordering::Relaxed), 2); + assert_eq!(drops.load(Ordering::Relaxed), 1); + assert_eq!(resources.workspace_in_use(), 100); + drop(permit); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + drop(Records::::new(65536, &resources).unwrap()); + let jobs = Records::collect((0..3).map(Ok), &resources).unwrap(); + assert_eq!( + pool.install(|| run_jobs(jobs, &resources, &|_| true, |job, _| Ok(job))) + .unwrap() + .len(), + 3 + ); + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn sibling_failure_joins_running_work_and_keeps_the_original_error() { + use std::sync::{atomic::AtomicUsize, Barrier}; + let resources = WriterResources::default().with_max_preparation_bytes(65536); + let barrier = Barrier::new(2); + let joined = AtomicUsize::new(0); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + let jobs = Records::collect((0..2).map(Ok), &resources).unwrap(); + let result: Result> = pool.install(|| { + run_jobs(jobs, &resources, &|_| true, |job, progress| { + barrier.wait(); + if job == 1 { + return Err(Error::SourceChanged("original failure")); + } + while !progress.is_cancelled() { + std::thread::yield_now(); + } + joined.fetch_add(1, Ordering::Relaxed); + Err(Error::Cancelled) + }) + }); + assert!(matches!( + result, + Err(Error::SourceChanged("original failure")) + )); + assert_eq!(joined.load(Ordering::Relaxed), 1); + drop(Records::::new(65536, &resources).unwrap()); + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn whole_member_allowance_precedes_source_open_and_survives_until_spooling() { + use std::sync::{atomic::AtomicUsize, Arc}; + let scratch = crate::scratch::case("whole-member-allowance"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 1024, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![options, options.with_optimal_parse(true)].into(), + }; + let data: Vec = (0..1024u32).flat_map(|n| n.to_le_bytes()).collect(); + let opens = Arc::new(AtomicUsize::new(0)); + let source = EntrySource::from_opener(data.len() as u64, { + let data = data.clone(); + let opens = opens.clone(); + move || { + opens.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(std::io::Cursor::new(data.clone()))) + } + }); + let integrity = (data.len() as u64, 0, [0; 32]); + let allowance = Allowance::limited(data.len() as u64 - 1); + let error = compress_whole_member( + 0, + &source, + integrity, + &plan, + &resources, + &|_| true, + &allowance, + ) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(opens.load(Ordering::Relaxed), 0); + assert_eq!(allowance.used(), 0); + + let allowance = Allowance::limited(16 * 1048576); + let mut packed = compress_whole_member( + 0, + &source, + integrity, + &plan, + &resources, + &|_| { + assert!(allowance.used() >= data.len() as u64); + true + }, + &allowance, + ) + .unwrap(); + assert!(!packed.store); + assert_eq!(packed.crc32, crate::crc32::crc32(&data)); + assert_eq!(packed.hash, blake2sp::hash(&data)); + assert_eq!(allowance.used(), 0); + let expected = candidates_with_allowance( + &data, + 0, + &plan.filter_policy, + &plan.candidates, + None, + &Allowance::default(), + ) + .unwrap(); + let mut bytes = Vec::new(); + packed.packed.copy_to(&mut bytes).unwrap(); + assert_eq!(&*expected, bytes); + drop(packed); + + let refused_spool = resources.clone().with_max_spool_bytes(0); + let error = compress_whole_member( + 0, + &source, + integrity, + &plan, + &refused_spool, + &|_| true, + &allowance, + ) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(allowance.used(), 0); + let cancelled = compress_whole_member( + 0, + &source, + integrity, + &plan, + &resources, + &|_| false, + &allowance, + ); + assert!(matches!(cancelled, Err(Error::Cancelled))); + assert_eq!(allowance.used(), 0); + for actual in [b"short".as_slice(), b"longer than declared"] { + let changed = + EntrySource::from_opener(8, move || Ok(Box::new(std::io::Cursor::new(actual)))); + let result = compress_whole_member( + 0, + &changed, + (8, 0, [0; 32]), + &plan, + &resources, + &|_| true, + &allowance, + ); + assert!(matches!( + result, + Err(Error::Io(_) | Error::SourceChanged(_)) + )); + assert_eq!(allowance.used(), 0); + } + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn whole_member_sources_open_only_after_combined_admission() { + use std::sync::{atomic::AtomicUsize, Arc}; + let encode_options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options, + dictionary_size: 65536, + block_size: 65536, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![encode_options].into(), + }; + let required = whole_member_workspace(32, &plan); + let scratch = crate::scratch::case("coordinator-admission"); + let resources = WriterResources::new(required * 2).with_temp_dir(&*scratch); + let calls = Arc::new(AtomicUsize::new(0)); + let sources: Vec<_> = (0..4) + .map(|_| { + let resources = resources.clone(); + let calls = calls.clone(); + EntrySource::from_opener(32, move || { + assert_eq!(resources.workspace_in_use(), required * 2); + calls.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(std::io::Cursor::new([42u8; 32]))) + }) + }) + .collect(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap(); + drop( + pool.install(|| compress_members_reporting(&sources, plan, &resources, &|_| true)) + .unwrap(), + ); + assert_eq!(calls.load(Ordering::Relaxed), 4); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[test] + fn automatic_filter_fallback_source_failure_keeps_member_context() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + let scratch = crate::scratch::case("fallback-source-error"); + let resources = WriterResources::new(70 * 1024 * 1024).with_temp_dir(&*scratch); + let length = 2 * 1024 * 1024; + let options = EncodeOptions::new(8).with_max_match_distance(128 * 1024); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 128 * 1024, + block_size: crate::codec::rar50::LZ_BLOCK_SIZE, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![options].into(), + }; + let execution = + ExecutionPlan::with_resources(&plan, std::iter::once(length), &resources).unwrap(); + let ExecutionPlan::IndependentMembers(ref members) = execution else { + panic!("automatic filtering should plan independent members"); + }; + assert!(matches!(members[0].execution, Execution::Blocks { .. })); + drop(execution); + let opens = Arc::new(AtomicUsize::new(0)); + let source = EntrySource::from_opener(length, { + let opens = Arc::clone(&opens); + move || { + opens.fetch_add(1, Ordering::Relaxed); + Err(std::io::Error::other("fallback source refused").into()) + } + }); + let error = compress_members_with_context( + &[source], + &plan, + &resources, + &|_| true, + &|index, error| { + assert_eq!(index, 0); + error.at_entry(b"fallback.bin".to_vec(), "compressing") + }, + ) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((b"fallback.bin".as_slice(), "compressing")) + ); + assert!(error.to_string().contains("fallback source refused")); + assert_eq!(opens.load(Ordering::Relaxed), 1); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[test] + fn mixed_whole_and_fallback_members_keep_order_progress_and_cleanup() { + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Mutex, + }; + struct Progress { + events: Mutex>, + cancel: bool, + stopped: AtomicBool, + } + impl CompressionProgress for Progress { + fn advance(&self, _: u64) -> bool { + !self.is_cancelled() + } + fn is_cancelled(&self) -> bool { + self.stopped.load(Ordering::Relaxed) + } + fn started(&self, index: usize, _: u64) { + self.events.lock().unwrap().push((true, index)); + if self.cancel && index == 1 { + self.stopped.store(true, Ordering::Relaxed); + } + } + fn finished(&self, index: usize, _: u64) { + self.events.lock().unwrap().push((false, index)); + } + } + let scratch = crate::scratch::case("planned-fallback"); + let data = b"planned automatic filter fallback\n".repeat(65536); + let sources = [ + EntrySource::from_bytes(b"first".to_vec()), + EntrySource::from_bytes(data.clone()), + EntrySource::from_bytes(b"last".to_vec()), + ]; + let encode_options = EncodeOptions::new(8).with_max_match_distance(128 * 1024); + let plan = CompressPlan { + algorithm_version: 0, + encode_options, + dictionary_size: 128 * 1024, + block_size: crate::codec::rar50::LZ_BLOCK_SIZE, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![encode_options].into(), + }; + let resources = WriterResources::new(70 * 1024 * 1024).with_temp_dir(&*scratch); + for cancel in [false, true] { + let progress = Progress { + events: Mutex::new(Vec::new()), + cancel, + stopped: AtomicBool::new(false), + }; + let result = compress_members_reporting(&sources, plan.clone(), &resources, &progress); + if cancel { + assert!(matches!(result, Err(Error::Cancelled))); + assert!(!progress.events.lock().unwrap().contains(&(true, 2))); + } else { + let mut members = result.unwrap(); + assert_eq!(members.len(), 3); + for index in 0..3 { + let events = progress.events.lock().unwrap(); + assert_eq!( + events + .iter() + .filter(|event| **event == (true, index)) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|event| **event == (false, index)) + .count(), + 1 + ); + } + assert_eq!(members[1].hash, blake2sp::hash(&data)); + let mut fallback_bytes = Vec::new(); + members[1].packed.copy_to(&mut fallback_bytes).unwrap(); + let mut plain = compress_members_reporting( + &sources[1..2], + CompressPlan { + filter_policy: FilterPolicy::None, + ..plan.clone() + }, + &resources, + &|_| true, + ) + .unwrap(); + let mut plain_bytes = Vec::new(); + plain[0].packed.copy_to(&mut plain_bytes).unwrap(); + assert_eq!(fallback_bytes, plain_bytes); + } + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn explicit_filter_pass_events_do_not_change_packed_bytes() { + use crate::filter_search::EncodeProgress; + let data: Vec<_> = (0..8192u32).flat_map(u32::to_le_bytes).collect(); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plain = encode_member_with_filter_policy_candidates_and_progress( + &data, + 0, + &FilterPolicy::Auto, + &[options, options], + None, + ) + .unwrap(); + let mut events = Vec::new(); + let reported = encode_member_with_filter_policy_candidates_and_progress( + &data, + 0, + &FilterPolicy::Auto, + &[options, options], + Some(&mut |event| { + events.push(event); + true + }), + ) + .unwrap(); + assert_eq!(plain, reported); + assert!( + events + .iter() + .filter(|&&event| event == EncodeProgress::PassStarted) + .count() + > 2 + ); + assert_eq!(events.first(), Some(&EncodeProgress::PassStarted)); + } + + use super::*; + + #[cfg(target_os = "linux")] + #[test] + fn many_members_fit_a_small_descriptor_limit() { + const CHILD: &str = "RARS_TEST_LOW_FD_CHILD"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new("sh") + .args(["-c", "ulimit -n 64 && exec \"$@\"", "sh"]) + .arg(std::env::current_exe().unwrap()) + .args([ + "--exact", + "rar50::write::compress::tests::many_members_fit_a_small_descriptor_limit", + "--test-threads=1", + "--nocapture", + ]) + .env(CHILD, "1") + .env("RAYON_NUM_THREADS", "2") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let scratch = crate::scratch::case("low-fd-members"); + let input = scratch.join("input"); + let data = b"many tiny archive members\n".repeat(8); + std::fs::write(&input, &data).unwrap(); + let sources = vec![EntrySource::from_path(&input); 128]; + let resources = WriterResources::default().with_temp_dir(&*scratch); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + for (method, solid, filter_policy) in [ + (0, false, FilterPolicy::None), + (1, false, FilterPolicy::None), + (1, true, FilterPolicy::None), + (1, false, FilterPolicy::Auto), + ] { + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid, + method, + filter_policy, + candidates: vec![options].into(), + }; + let mut members = + compress_members_reporting(&sources, plan, &resources, &|_| true).unwrap(); + assert_eq!(members.len(), sources.len()); + for member in &mut members { + assert_eq!(member.input_size, data.len() as u64); + assert_eq!(member.hash, blake2sp::hash(&data)); + let mut bytes = Vec::new(); + assert_eq!( + member.packed.copy_to(&mut bytes).unwrap(), + member.packed.len() + ); + if method != 0 { + assert!(!bytes.is_empty()); + } + } + drop(members); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 1); + } + } + + #[test] + fn solid_inputs_open_one_at_a_time_and_close_on_failure() { + use std::io::{Cursor, Seek, SeekFrom}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + struct Tracked { + data: Cursor>, + live: Arc, + } + impl Read for Tracked { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + self.data.read(bytes) + } + } + impl Seek for Tracked { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.data.seek(pos) + } + } + impl Drop for Tracked { + fn drop(&mut self) { + self.live.fetch_sub(1, Ordering::SeqCst); + } + } + let live = Arc::new(AtomicUsize::new(0)); + let sources: Vec<_> = (0..128) + .map(|n| { + let live = live.clone(); + EntrySource::from_opener(32, move || { + assert_eq!(live.fetch_add(1, Ordering::SeqCst), 0); + Ok(Box::new(Tracked { + data: Cursor::new(vec![n; 32]), + live: live.clone(), + })) + }) + }) + .collect(); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid: true, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![options].into(), + }; + let scratch = crate::scratch::case("lazy-solid-inputs"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let result = + compress_members_reporting(&sources, plan.clone(), &resources, &|_| true).unwrap(); + assert_eq!(result.len(), sources.len()); + assert_eq!(live.load(Ordering::SeqCst), 0); + drop(result); + assert!(compress_members_reporting(&sources, plan, &resources, &|_| false).is_err()); + assert_eq!(live.load(Ordering::SeqCst), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[test] + fn whole_member_budget_includes_tree_and_parse_workspace() { + let size = 16 * 1024 * 1024; + let options = EncodeOptions::new(32) + .with_max_match_distance(size) + .with_optimal_parse(true); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: size as u64, + block_size: crate::codec::rar50::LZ_BLOCK_SIZE, + solid: false, + method: 3, + filter_policy: FilterPolicy::Auto, + candidates: vec![options].into(), + }; + let required = whole_member_workspace(size as u64, &plan); + assert!(required >= (size * 12) as u64); + assert!(matches!( + WriterResources::new((size * 4 + 2 * 1024 * 1024) as u64) + .acquire(required, size as u64), + Err(Error::MemoryLimitExceeded { .. }) + )); + // A large configured dictionary must not charge unreachable links. + let mut larger = plan.clone(); + larger.encode_options.max_match_distance *= 2; + assert_eq!(whole_member_workspace(size as u64, &larger), required); + } + #[cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] + #[test] + fn whole_members_use_multiple_workers_without_changing_bytes() { + use std::collections::HashSet; + use std::sync::Mutex; + let sources: Vec<_> = (0..8) + .map(|n| { + EntrySource::from_bytes( + format!("member {n}: independent text compression\n") + .repeat(1024) + .into_bytes(), + ) + }) + .collect(); + let options = EncodeOptions::new(8); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 65536, + block_size: 65536, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![options].into(), + }; + let run = |threads, budget| { + let workers = Mutex::new(HashSet::new()); + let result = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .unwrap() + .install(|| { + compress_members_reporting( + &sources, + plan.clone(), + &WriterResources::new(budget), + &|_| { + workers.lock().unwrap().insert(std::thread::current().id()); + true + }, + ) + .unwrap() + .into_iter() + .map(|mut member| { + let mut bytes = Vec::new(); + member.packed.copy_to(&mut bytes).unwrap(); + bytes + }) + .collect::>() + }); + (result, workers.into_inner().unwrap().len()) + }; + let (serial, _) = run(1, 256 * 1024 * 1024); + let (parallel, workers) = run(4, 256 * 1024 * 1024); + assert_eq!(serial, parallel); + assert!(workers > 1); + let (limited, _) = run( + 4, + whole_member_workspace(sources[0].len().unwrap(), &plan) * 2, + ); + assert_eq!(serial, limited); + } + #[test] + fn checksums_follow_the_compression_read_including_pushback_and_empty_members() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + let mut data = vec![0; 65536]; + data.extend(std::iter::repeat_n(1, 65536)); + data.extend(std::iter::repeat_n(2, 65536)); + for policy in [FilterPolicy::None, FilterPolicy::Auto] { + for solid in [false, true] { + let opens = Arc::new(AtomicUsize::new(0)); + let sources: Vec<_> = [data.clone(), Vec::new()] + .into_iter() + .map(|data| { + let opens = Arc::clone(&opens); + EntrySource::from_opener(data.len() as u64, move || { + opens.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(std::io::Cursor::new(data.clone()))) + }) + }) + .collect(); + let options = EncodeOptions::new(8).with_max_match_distance(131072); + let plan = CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 131072, + block_size: 65536, + solid, + method: 1, + filter_policy: policy.clone(), + candidates: vec![options].into(), + }; + let members = compress_members_reporting( + &sources, + plan, + &WriterResources::default(), + &|_| true, + ) + .unwrap(); + assert_eq!( + opens.load(Ordering::SeqCst), + 2, + "policy={policy:?}, solid={solid}" + ); + for (member, input) in members.iter().zip([data.as_slice(), &[]]) { + let mut crc = Crc32::new(); + crc.update(input); + assert_eq!(member.crc32, crc.finish()); + assert_eq!(member.hash, blake2sp::hash(input)); + } + } + } + } +} diff --git a/arca-rar/vendor/rars/src/rar50/write/engine.rs b/arca-rar/vendor/rars/src/rar50/write/engine.rs new file mode 100644 index 00000000..ee5ef53e --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/engine.rs @@ -0,0 +1,3223 @@ +//! Assembling a RAR 5 archive from prepared payloads. +//! +//! Native member spools use temporary files; bare-WASM spools retain bytes in +//! memory. Compression may load whole members; headers also allocate memory. +//! Service payloads borrow input and are encrypted in chunks during emission. +//! Quick-open data uses a quota-controlled spool. Once block lengths are known, +//! final archive output streams in one pass. The +//! workspace admission budget is not an aggregate RAM or disk quota. +//! +//! Stored payloads are reread and verified during emission. Recovery requires +//! a further pass over the preceding archive bytes, mirrored into a spool. + +use super::compress::{self, CompressPlan, CompressedMember}; +#[cfg(test)] +use super::headers::write_vint; +use super::headers::{ + block_header_image, encrypted_header_block, encrypted_main_header_block, file_specific, + header_encryption_keys, header_encryption_password, prepared_header_image, + stored_file_specific, write_end_header, write_extra_record, write_file_encryption_record, + write_hash_record_with_value, write_head_crypt, write_main_header, HeaderEncryptionKeys, + PreparedHeader, +}; +use super::layout::{resolve_layout, LayoutInputs}; +use super::{encrypt_reader_to, ArchiveEntry}; +use crate::crypto::rar50::{Rar50Keys, WRITE_KDF_COUNT_LOG}; +use crate::detect::RAR50_SIGNATURE; +use crate::rar50::{ + FHEXTRA_SUBDATA, HEAD_END, HEAD_FILE, HEAD_SERVICE, HFL_DATA, HFL_EXTRA, MHFL_RECOVERY, + MHFL_SOLID, +}; +#[cfg(feature = "recovery")] +use crate::recovery::rar5::{ + choose_recovery_memory_mode, plan_inline_recovery, streamed_recovery_with_allowance, + ReadWriteSeek, +}; +use crate::streaming::preparation::{Bytes, Owned, Records}; +use crate::streaming::Spool; +use crate::write_progress::{check_cancelled, CancellableIo, ProgressReporter}; +use crate::{Error, Result, WriterResources}; +use std::io::{Read, Write}; + +pub(super) struct EnginePlan<'a> { + pub(super) compress: CompressPlan, + pub(super) recovery_percent: Option, + pub(super) header_encrypted: bool, + pub(super) header_password: Option<&'a [u8]>, + pub(super) archive_comment: Option>, + pub(super) archive_metadata: Option>, + pub(super) metadata_record: Option<&'a crate::rar50::ArchiveMetadataRecord>, + pub(super) locked: bool, + pub(super) quick_open: bool, + pub(super) progress: Option>, +} + +pub(super) enum ArchiveCommentPlan<'a> { + Plain(&'a [u8]), + Encrypted { data: &'a [u8], password: &'a [u8] }, +} + +/// A block with its framing settled: the header bytes are final and the +/// payload only has to be copied. +struct PreparedBlock<'a> { + header: PreparedHeader, + payload: Payload<'a>, + payload_len: u64, + /// Quick-open repeats the headers of members and plain comments so a + /// reader can list an archive without walking it. + quick_open_cached: bool, + /// Index into the caller's entries; names are cloned only on error. + entry_index: Option, +} + +impl PreparedBlock<'_> { + fn len(&self) -> Result { + (self.header.len() as u64) + .checked_add(self.payload_len) + .ok_or(Error::InvalidArgument("RAR 5 archive block size overflows")) + } +} + +enum Payload<'a> { + /// Comments and services remain owned by the caller or builder. + Borrowed(&'a [u8]), + /// Copied straight from the source, which is re-read at write time. + Stored(PreparedSource), + Packed(Spool), + /// Encrypted on the way out, so the ciphertext is never stored anywhere. + Encrypted { + plain: Owned>, + keys: Rar50Keys, + iv: [u8; 16], + }, +} + +enum PlainPayload<'a> { + Borrowed(&'a [u8]), + Stored(PreparedSource), + Packed(Spool), +} + +// Member payloads cannot borrow comment/service bytes. Keeping their two +// forms distinct avoids an impossible borrowed arm during plain emission. +enum MemberPlainPayload { + Stored(PreparedSource), + Packed(Spool), +} + +impl From for PlainPayload<'_> { + fn from(member: MemberPlainPayload) -> Self { + match member { + MemberPlainPayload::Stored(source) => Self::Stored(source), + MemberPlainPayload::Packed(packed) => Self::Packed(packed), + } + } +} + +impl From for Payload<'_> { + fn from(member: MemberPlainPayload) -> Self { + match member { + MemberPlainPayload::Stored(source) => Self::Stored(source), + MemberPlainPayload::Packed(packed) => Self::Packed(packed), + } + } +} + +// A reopenable source is not a snapshot. Keep the size and integrity used by +// the header, and verify the actual emission read before reporting success. +struct PreparedSource { + source: crate::EntrySource, + len: u64, + crc32: u32, + hash: [u8; 32], +} + +impl PreparedSource { + fn new(source: &crate::EntrySource, member: &CompressedMember) -> Self { + Self { + source: source.clone(), + len: member.input_size, + crc32: member.crc32, + hash: member.hash, + } + } + + fn open(&self) -> Result { + Ok(CheckedReader { + reader: self.source.open()?, + integrity: ChecksumSink::default(), + }) + } + + fn verify(&self, integrity: ChecksumSink) -> Result<()> { + if integrity.crc.finish() != self.crc32 || integrity.hash.finalize() != self.hash { + return Err(Error::SourceChanged( + "entry source contents changed while writing", + )); + } + Ok(()) + } +} + +struct CheckedReader { + reader: Box, + integrity: ChecksumSink, +} + +impl Read for CheckedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.reader.read(buffer)?; + self.integrity.write_all(&buffer[..read])?; + Ok(read) + } +} + +impl CheckedReader { + fn finish(mut self, source: &PreparedSource, observed: u64) -> Result<()> { + crate::write_stream::check_source_length( + &mut *self.reader, + observed, + source.len, + "entry source size changed while writing", + )?; + source.verify(self.integrity) + } +} + +// Both archive and volume emission report and account for one aggregate input +// length. Caller-provided source lengths need not fit that sum even when each +// length fits a member header. Reject overflow before any payload read. +fn total_input_size(entries: &[ArchiveEntry]) -> Result { + entries.iter().try_fold(0u64, |total, entry| { + let length = entry + .source + .len() + .map_err(|error| member_error(error, &entry.name, "preparing"))?; + total.checked_add(length).ok_or_else(|| { + member_error( + Error::InvalidArgument("RAR 5 writer total input size overflows"), + &entry.name, + "preparing", + ) + }) + }) +} + +pub(super) fn write_archive( + entries: &[ArchiveEntry], + plan: EnginePlan<'_>, + resources: &WriterResources, + output: &mut dyn Write, +) -> Result<()> { + let mut controlled_output = CancellableIo { + inner: output, + progress: plan.progress, + }; + let output: &mut dyn Write = &mut controlled_output; + for entry in entries { + super::validate_entry(entry)?; + } + + let header_keys = if plan.header_encrypted { + let password = header_encryption_password( + plan.header_password + .into_iter() + .chain(entries.iter().filter_map(|entry| entry.password.as_deref())), + )?; + Some(header_encryption_keys(password)?) + } else { + None + }; + + let mut sources = Records::new(entries.len(), resources)?; + for entry in entries { + sources.push(entry.source.clone())?; + } + let total_input = total_input_size(entries)?; + let total_entries = entries.len(); + if let Some(progress) = plan.progress { + progress.report(crate::WriteProgressEvent::OperationStarted { + operation: crate::WriteOperation::Compression, + total_bytes: Some(total_input), + total_entries: Some(total_entries), + pass: 1, + }); + } + let work = crate::write_progress::WorkTracker::new( + plan.progress, + crate::WriteOperation::Compression, + total_input, + ); + let compressed = compress::compress_members_with_context( + &sources, + &plan.compress, + resources, + &MemberProgress { + entries, + work: &work, + }, + &|index, error| member_error(error, &entries[index].name, "compressing"), + )?; + + // Everything between the main header and the quick-open block, in order. + // Entries and their owned service vectors contain records larger than a byte + // in disjoint live storage. Their combined count fits usize, including + // the single optional archive comment. This relies on services being owned + // vectors, rather than shared or borrowed lists counted repeatedly. + let block_count = entries.iter().fold( + usize::from(plan.archive_comment.is_some()), + |total, entry| total + 1 + entry.services.len(), + ); + let mut blocks = Records::::new(block_count, resources)?; + if let Some(comment) = &plan.archive_comment { + blocks.push(prepare_comment(comment, header_keys.as_ref(), resources)?)?; + } + for (index, (entry, member)) in entries.iter().zip(compressed).enumerate() { + check_cancelled(plan.progress)?; + let mut block = prepare_member(entry, member, &plan, header_keys.as_ref(), resources) + .map_err(|error| member_error(error, &entry.name, "preparing"))?; + block.entry_index = Some(index); + blocks.push(block)?; + for service in &entry.services { + let mut block = prepare_service(service, header_keys.as_ref(), resources) + .map_err(|error| member_error(error, &entry.name, "preparing service"))?; + block.entry_index = Some(index); + blocks.push(block)?; + } + } + if !work.finish() { + return Err(Error::Cancelled); + } + if let Some(progress) = plan.progress { + progress.report(crate::WriteProgressEvent::OperationFinished { + operation: crate::WriteOperation::Compression, + total_bytes: Some(total_input), + total_entries: Some(total_entries), + pass: 1, + }); + } + + let body_len = blocks.iter().try_fold(0u64, |total, block| { + total + .checked_add(block.len()?) + .ok_or(Error::InvalidArgument("RAR 5 archive body size overflows")) + })?; + + // Quick-open stores how far back each cached header sits from the + // quick-open block itself. Both move together when the prefix grows, so + // the distances only need positions within the body. + let quick_open_payload = if plan.quick_open { + let mut payload = Spool::create(resources)?; + let mut checksum = crate::crc32::Crc32::new(); + let mut offset = 0u64; + for block in &blocks { + check_cancelled(plan.progress)?; + if block.quick_open_cached { + append_quick_open_entry( + &mut payload, + &mut checksum, + body_len - offset, + &block.header, + )?; + } + offset += block.len()?; + } + let payload_len = payload.len(); + let header = stored_service_header( + b"QO", + payload_len, + checksum.finish(), + &[], + header_keys.as_ref(), + resources, + )?; + payload.park(); + Some(PreparedBlock { + header, + payload: Payload::Packed(payload), + payload_len, + quick_open_cached: false, + entry_index: None, + }) + } else { + None + }; + + let head_crypt = match &header_keys { + Some(keys) => { + let mut block = Bytes::new(resources); + write_head_crypt(&mut block, keys, resources)?; + block + } + None => Bytes::new(resources), + }; + + let mut main_flags = if plan.locked { + crate::rar50::MHFL_LOCKED + } else { + 0 + }; + if plan.compress.solid { + main_flags |= MHFL_SOLID; + } + if plan.recovery_percent.is_some() { + main_flags |= MHFL_RECOVERY; + } + + let layout = resolve_layout( + &LayoutInputs { + header_encrypted: plan.header_encrypted, + head_crypt_len: head_crypt.len() as u64, + main_flags, + volume_number: None, + archive_metadata: plan.archive_metadata, + metadata_record: plan.metadata_record, + body_len, + quick_open_payload_len: quick_open_payload.as_ref().map(|block| block.payload_len), + recovery_percent: plan.recovery_percent, + }, + resources, + )?; + + report_emission(plan.progress, true); + // Only mirror the archive when a recovery record has to read it back. + let mut mirror = match plan.recovery_percent { + Some(_) => Some(Spool::create(resources)?), + None => None, + }; + { + let mut sink = Tee { + output, + mirror: mirror.as_mut(), + }; + + let main = match &header_keys { + Some(keys) => encrypted_main_header_block( + &keys.keys, + main_flags, + None, + &layout.main_extra, + resources, + )?, + None => { + let mut main = Bytes::new(resources); + write_main_header(&mut main, main_flags, None, &layout.main_extra, resources)?; + main + } + }; + // The layout predicted this before any of it existed. If the + // prediction is off, every offset in the locator is off with it. + debug_assert_eq!( + main.len() as u64, + layout.main_header_len, + "main header size differs from the size its layout was built on" + ); + + sink.write_all(RAR50_SIGNATURE)?; + sink.write_all(&head_crypt)?; + sink.write_all(&main)?; + + for block in blocks { + let entry_index = block.entry_index; + let result = (|| { + sink.write_all(&block.header)?; + write_payload(block.payload, &mut sink, resources, plan.progress) + })(); + result.map_err(|error| match entry_index { + Some(index) => member_error(error, &entries[index].name, "writing"), + None => error, + })?; + } + + if let Some(block) = quick_open_payload { + sink.write_all(&block.header)?; + write_payload(block.payload, &mut sink, resources, plan.progress)?; + } + } + + if let Some(recovery_percent) = plan.recovery_percent { + let mirror = mirror.as_mut().expect("recovery mirrors the archive"); + // The recovery block has to start exactly where the locator in the + // main header says it does. + debug_assert_eq!(layout.recovery_prefix_len, Some(mirror.len())); + debug_assert_eq!( + layout.recovery_offset, + Some(mirror.len() - RAR50_SIGNATURE.len() as u64), + "recovery record is not where the locator points" + ); + write_recovery_service( + recovery_percent, + mirror, + header_keys.as_ref(), + resources, + plan.progress, + output, + )?; + } + + match &header_keys { + Some(keys) => output.write_all(&encrypted_header_block( + &keys.keys, + HEAD_END, + 0, + None, + &super::end_header_specific(0), + &[], + &[], + resources, + )?)?, + None => { + let mut end = Bytes::new(resources); + write_end_header(&mut end, 0, resources)?; + output.write_all(&end)?; + } + } + check_cancelled(plan.progress)?; + report_emission(plan.progress, false); + Ok(()) +} + +/// Appends one quick-open record: how far back the header sits, then the +/// header itself. +/// +/// The wrapper is `CRC32 || BlockSize || body`, and the checksum covers +/// `BlockSize` as well as the body. That is easy to get backwards, because the +/// length is written after the checksum it is part of. Checksumming the body +/// alone costs nothing visible: readers reject the wrapper, fall back to +/// walking the block chain, and report the archive as fine while the index +/// they were handed goes unused. +fn append_quick_open_entry( + payload: &mut dyn Write, + payload_crc: &mut crate::crc32::Crc32, + distance: u64, + header: &[u8], +) -> Result<()> { + // At most three u64 vints. Keep framing on the stack and borrow the header + // instead of cloning it into body and wrapper buffers. + fn vint(out: &mut [u8], mut value: u64) -> usize { + let mut len = 0; + loop { + out[len] = (value as u8 & 0x7f) | if value >= 0x80 { 0x80 } else { 0 }; + len += 1; + value >>= 7; + if value == 0 { + return len; + } + } + } + let mut body_prefix = [0; 21]; + let mut len = 1; // Flags = 0. + len += vint(&mut body_prefix[len..], distance); + len += vint(&mut body_prefix[len..], header.len() as u64); + let body_len = (header.len() as u64) + .checked_add(len as u64) + .ok_or(Error::InvalidArgument( + "RAR 5 quick-open record size overflows", + ))?; + let mut size = [0; 10]; + let size_len = vint(&mut size, body_len); + let parts = [&size[..size_len], &body_prefix[..len], header]; + let mut crc = crate::crc32::Crc32::new(); + for part in parts { + crc.update(part); + } + // Combine the small framing fields into one write to the native spool. + let mut framing = [0; 35]; + framing[..4].copy_from_slice(&crc.finish().to_le_bytes()); + framing[4..4 + size_len].copy_from_slice(&size[..size_len]); + let framing_len = 4 + size_len + len; + framing[4 + size_len..framing_len].copy_from_slice(&body_prefix[..len]); + for part in [&framing[..framing_len], header] { + payload.write_all(part)?; + payload_crc.update(part); + } + Ok(()) +} + +/// A stored service block borrowing a named payload such as a comment. +fn stored_service_block<'a>( + name: &[u8], + data: &'a [u8], + service_data: &[u8], + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result> { + Ok(PreparedBlock { + header: stored_service_header( + name, + data.len() as u64, + crate::crc32::crc32(data), + service_data, + header_keys, + resources, + )?, + payload: Payload::Borrowed(data), + payload_len: data.len() as u64, + quick_open_cached: false, + entry_index: None, + }) +} + +fn stored_service_header( + name: &[u8], + data_len: u64, + crc32: u32, + service_data: &[u8], + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result { + let mut extra = Bytes::new(resources); + write_extra_record(&mut extra, FHEXTRA_SUBDATA, service_data)?; + let specific = stored_file_specific(name, data_len, crc32, 0, None, 0, resources)?; + prepared_header_image( + HEAD_SERVICE, + HFL_EXTRA | HFL_DATA, + Some(data_len), + &specific, + &extra, + header_keys, + resources, + ) +} + +fn prepare_comment<'a>( + comment: &ArchiveCommentPlan<'a>, + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result> { + match comment { + ArchiveCommentPlan::Plain(data) => { + let mut block = stored_service_block(b"CMT", data, &[], header_keys, resources)?; + // Plain comments are listed by quick-open; encrypted ones are not. + block.quick_open_cached = header_keys.is_none(); + Ok(block) + } + ArchiveCommentPlan::Encrypted { data, password } => { + encrypted_service_block(b"CMT", data, &[], password, header_keys, resources) + } + } +} + +fn prepare_service<'a>( + service: &'a super::ServiceEntry, + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result> { + match service.password.as_deref() { + Some(password) => encrypted_service_block( + &service.name, + &service.data, + &[], + password, + header_keys, + resources, + ), + None => stored_service_block(&service.name, &service.data, &[], header_keys, resources), + } +} + +/// A service block whose borrowed payload is encrypted during emission. +fn encrypted_service_block<'a>( + name: &[u8], + data: &'a [u8], + service_data: &[u8], + password: &[u8], + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result> { + super::validate_nonempty_password(password)?; + let mut salt = [0u8; 16]; + let mut iv = [0u8; 16]; + crate::write_stream::fill_entropy( + &mut salt, + "RAR 5 writer could not generate encryption salt", + )?; + crate::write_stream::fill_entropy(&mut iv, "RAR 5 writer could not generate encryption IV")?; + let keys = Rar50Keys::derive(password, salt, WRITE_KDF_COUNT_LOG).map_err(Error::from)?; + + let mut extra = Bytes::new(resources); + write_extra_record(&mut extra, FHEXTRA_SUBDATA, service_data)?; + write_file_encryption_record(&mut extra, salt, iv, keys.checked_password_record()?)?; + write_hash_record_with_value( + &mut extra, + keys.checked_hash_mac(crate::rar50::blake2sp::hash(data))?, + )?; + let specific = stored_file_specific( + name, + data.len() as u64, + keys.checked_crc_mac(crate::crc32::crc32(data))?, + 0, + None, + 0, + resources, + )?; + let payload_len = (data.len() as u64) + .checked_add(15) + .ok_or(Error::InvalidArgument( + "RAR 5 encrypted data size overflows", + ))? + & !15; + let header = prepared_header_image( + HEAD_SERVICE, + HFL_EXTRA | HFL_DATA, + Some(payload_len), + &specific, + &extra, + header_keys, + resources, + )?; + Ok(PreparedBlock { + header, + payload: Payload::Encrypted { + plain: Owned::new(PlainPayload::Borrowed(data), resources)?, + keys, + iv, + }, + payload_len, + quick_open_cached: false, + entry_index: None, + }) +} + +fn decoded_rar50_name_len(bytes: &[u8]) -> usize { + let Ok(text) = std::str::from_utf8(bytes) else { + return bytes.len(); + }; + if !text.contains('\u{fffe}') { + return bytes.len(); + } + text.chars() + .map(|ch| match ch { + '\u{fffe}' => 0, + '\u{e080}'..='\u{e0ff}' => 1, + _ => ch.len_utf8(), + }) + .sum() +} + +/// Builds a member's final header and decides how its payload will be written. +fn prepare_member( + entry: &ArchiveEntry, + member: CompressedMember, + plan: &EnginePlan<'_>, + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result> { + let compression_info = compress::member_compression_info(&plan.compress, &member)?; + let plain_len = if member.store { + member.input_size + } else { + member.packed.len() + }; + let plain = if member.store { + MemberPlainPayload::Stored(PreparedSource::new(&entry.source, &member)) + } else { + MemberPlainPayload::Packed(member.packed) + }; + + let mut extra = Bytes::new(resources); + let (payload, payload_len, data_crc32, hash) = match entry.password.as_deref() { + Some(password) => { + let mut salt = [0u8; 16]; + let mut iv = [0u8; 16]; + crate::write_stream::fill_entropy( + &mut salt, + "RAR 5 writer could not generate encryption salt", + )?; + crate::write_stream::fill_entropy( + &mut iv, + "RAR 5 writer could not generate encryption IV", + )?; + let keys = + Rar50Keys::derive(password, salt, WRITE_KDF_COUNT_LOG).map_err(Error::from)?; + write_file_encryption_record(&mut extra, salt, iv, keys.checked_password_record()?)?; + let crc32 = keys.checked_crc_mac(member.crc32)?; + let hash = keys.checked_hash_mac(member.hash)?; + ( + Payload::Encrypted { + plain: Owned::new(plain.into(), resources)?, + keys, + iv, + }, + // Encryption pads the payload up to the cipher block size. + plain_len.div_ceil(16) * 16, + crc32, + hash, + ) + } + None => (plain.into(), plain_len, member.crc32, member.hash), + }; + // A link has no file payload to hash; its target is protected by the header CRC. + if entry.redirection.is_none() { + write_hash_record_with_value(&mut extra, hash)?; + } + super::headers::write_mtime_record(&mut extra, entry.mtime, entry.mtime_nanoseconds)?; + if let Some(times) = entry.file_times { + write_extra_record(&mut extra, super::super::FHEXTRA_HTIME, ×.encode()?)?; + } + if let Some(link) = &entry.redirection { + let mut record = Bytes::new(resources); + record.vint(link.redirection_type)?; + record.vint(link.flags)?; + record.vint(link.target_name.len() as u64)?; + record.extend_from_slice(&link.target_name)?; + write_extra_record(&mut extra, super::super::FHEXTRA_REDIR, &record)?; + } + + let specific = file_specific( + &entry.name, + // Match Unix link stat size, while the packed payload remains empty. + entry + .redirection + .as_ref() + .map_or(member.input_size, |link| { + entry + .redirection_size + .unwrap_or_else(|| decoded_rar50_name_len(&link.target_name) as u64) + }), + data_crc32, + entry.attributes, + entry.mtime.filter(|_| entry.mtime_nanoseconds.is_none()), + compression_info, + entry.host_os, + entry.is_directory, + resources, + )?; + let header = prepared_header_image( + HEAD_FILE, + HFL_EXTRA | HFL_DATA, + Some(payload_len), + &specific, + &extra, + header_keys, + resources, + )?; + + Ok(PreparedBlock { + header, + payload, + payload_len, + quick_open_cached: true, + entry_index: None, + }) +} + +fn write_payload( + payload: Payload<'_>, + output: &mut dyn Write, + resources: &WriterResources, + progress: Option>, +) -> Result<()> { + check_cancelled(progress)?; + match payload { + Payload::Borrowed(data) => { + output.write_all(data)?; + Ok(()) + } + Payload::Stored(source) => { + let mut reader = source.open()?; + let copied = std::io::copy(&mut reader.by_ref().take(source.len), output)?; + reader.finish(&source, copied)?; + source.source.release(); + Ok(()) + } + Payload::Packed(mut packed) => { + packed.copy_to(output)?; + Ok(()) + } + Payload::Encrypted { plain, keys, iv } => { + const ENCRYPT_CHUNK: usize = 64 * 1024; + let chunk_size = match &*plain { + PlainPayload::Borrowed(data) => { + data.len().clamp(1, ENCRYPT_CHUNK).div_ceil(16) * 16 + } + _ => ENCRYPT_CHUNK, + }; + let _permit = resources.acquire_cancellable(chunk_size as u64, 0, &|| { + progress.is_some_and(ProgressReporter::is_cancelled) + })?; + match plain.into_inner() { + PlainPayload::Stored(source) => { + let mut reader = source.open()?; + encrypt_reader_to( + &mut reader, + source.len, + output, + &keys, + iv, + ENCRYPT_CHUNK, + progress, + resources, + )?; + reader.finish(&source, source.len)?; + source.source.release(); + Ok(()) + } + PlainPayload::Packed(mut packed) => { + let len = packed.len(); + packed.rewind()?; + encrypt_reader_to( + &mut packed, + len, + output, + &keys, + iv, + ENCRYPT_CHUNK, + progress, + resources, + ) + } + PlainPayload::Borrowed(mut data) => { + let len = data.len() as u64; + encrypt_reader_to( + &mut data, len, output, &keys, iv, chunk_size, progress, resources, + ) + } + } + } + } +} + +/// Computes the recovery record over `prefix` and writes its service block. +fn write_recovery_service( + recovery_percent: u64, + prefix: &mut Spool, + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, + progress: Option>, + output: &mut dyn Write, +) -> Result { + #[cfg(not(feature = "recovery"))] + { + let _ = ( + recovery_percent, + prefix, + header_keys, + resources, + progress, + output, + ); + Err(Error::FeatureDisabled { + feature: "recovery", + }) + } + #[cfg(feature = "recovery")] + { + if let Some(allowance) = &resources.execution { + return recovery_service_with_allowance( + recovery_percent, + prefix, + header_keys, + resources, + progress, + output, + allowance, + ); + } + recovery_service_with_allowance( + recovery_percent, + prefix, + header_keys, + resources, + progress, + output, + &crate::codec::workspace::Allowance::default(), + ) + } +} + +#[cfg(feature = "recovery")] +fn recovery_service_with_allowance( + recovery_percent: u64, + prefix: &mut Spool, + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, + progress: Option>, + output: &mut dyn Write, + allowance: &B, +) -> Result { + let prefix_len = prefix.len(); + let plan = plan_inline_recovery(prefix_len, recovery_percent)?; + let (mode, required) = choose_recovery_memory_mode(plan, resources.memory_limit())?; + let (mode, required) = if let Some(allowance) = &resources.execution { + crate::recovery::rar5::choose_recovery_capacity_mode( + plan, + resources.memory_limit(), + allowance, + )? + } else { + (mode, required) + }; + let _permit = resources.acquire_cancellable(required, 0, &|| { + progress.is_some_and(ProgressReporter::is_cancelled) + })?; + + let mut scratch = match mode { + crate::recovery::rar5::RecoveryMemoryMode::Striped { .. } => { + Some(Spool::create(resources)?) + } + crate::recovery::rar5::RecoveryMemoryMode::Resident => None, + }; + let mut payload = Spool::create(resources)?; + prefix.rewind()?; + let built = streamed_recovery_with_allowance( + &mut CancellableIo { + inner: prefix, + progress, + }, + prefix_len, + plan, + mode, + scratch + .as_mut() + .map(|scratch| scratch as &mut dyn ReadWriteSeek), + &mut CancellableIo { + inner: &mut payload, + progress, + }, + progress, + 1, + allowance, + ) + .map_err(|error| { + if check_cancelled(progress).is_err() { + Error::Cancelled + } else { + error.into() + } + })?; + + debug_assert_eq!(built.plan.payload_size(), Ok(built.payload_len)); + + let mut service_data = Bytes::new(resources); + service_data.vint(recovery_percent)?; + let mut extra = Bytes::new(resources); + write_extra_record(&mut extra, FHEXTRA_SUBDATA, &service_data)?; + let specific = stored_file_specific( + b"RR", + built.payload_len, + built.payload_crc32, + 0, + None, + 0, + resources, + )?; + let header = match header_keys { + Some(keys) => encrypted_header_block( + &keys.keys, + HEAD_SERVICE, + HFL_EXTRA | HFL_DATA, + Some(built.payload_len), + &specific, + &extra, + &[], + resources, + )?, + None => block_header_image( + HEAD_SERVICE, + HFL_EXTRA | HFL_DATA, + Some(built.payload_len), + &specific, + &extra, + resources, + )?, + }; + output.write_all(&header)?; + payload.copy_to(output)?; + Ok(header.len() as u64 + built.payload_len) +} + +/// Writes to the archive and, when a recovery record is coming, keeps a copy +/// for the parity pass to read. +struct Tee<'a> { + output: &'a mut dyn Write, + mirror: Option<&'a mut Spool>, +} + +impl Write for Tee<'_> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.output.write_all(buffer)?; + if let Some(mirror) = self.mirror.as_mut() { + mirror.write_all(buffer)?; + } + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.output.flush() + } +} + +/// Where a member's payload bytes come from when a volume set slices them up. +enum FragmentSource { + Packed(Spool), + Stored { + prepared: PreparedSource, + emitted: Owned, + }, +} + +impl FragmentSource { + /// Checksums of the bytes a fragment will store, read ahead of writing them. + /// + /// The fragment's header carries these and the header goes out before the + /// payload, so the range is read twice: once here and once to copy it. The + /// alternative is patching the fields after the copy, which an encrypted + /// header will not allow. + fn checksums_range( + &mut self, + start: u64, + len: u64, + progress: Option>, + ) -> Result { + let mut sink = ChecksumSink::default(); + self.copy_range_unverified( + start, + len, + &mut CancellableIo { + inner: &mut sink, + progress, + }, + )?; + Ok(FragmentChecksums { + crc32: sink.crc.finish(), + hash: sink.hash.finalize(), + }) + } + + fn emit_range( + &mut self, + start: u64, + len: u64, + output: &mut dyn Write, + expected_fragment: Option, + resources: &WriterResources, + ) -> Result<()> { + let Self::Stored { prepared, emitted } = self else { + return self.copy_range_unverified(start, len, output); + }; + let mut buffer = Bytes::zeroed(64 * 1024, resources)?; + let mut reader = prepared.source.open()?; + reader.seek(std::io::SeekFrom::Start(start))?; + let mut fragment = ChecksumSink::default(); + let mut remaining = len; + while remaining != 0 { + let want = remaining.min(buffer.len() as u64) as usize; + reader.read_exact(&mut buffer[..want])?; + output.write_all(&buffer[..want])?; + emitted.write_all(&buffer[..want])?; + if expected_fragment.is_some() { + fragment.write_all(&buffer[..want])?; + } + remaining -= want as u64; + } + if let Some(expected) = expected_fragment { + if fragment.crc.finish() != expected.crc32 || fragment.hash.finalize() != expected.hash + { + return Err(Error::SourceChanged( + "entry source contents changed while writing", + )); + } + } + if start + len == prepared.len { + crate::write_stream::check_source_length( + &mut *reader, + start + len, + prepared.len, + "entry source size changed while writing", + )?; + prepared.verify(std::mem::take(&mut **emitted))?; + } + Ok(()) + } + + fn copy_range_unverified( + &mut self, + start: u64, + len: u64, + output: &mut dyn Write, + ) -> Result<()> { + match self { + Self::Packed(spool) => { + spool.copy_range_to(start, len, output)?; + } + Self::Stored { prepared, .. } => { + let mut reader = prepared.source.open()?; + reader.seek(std::io::SeekFrom::Start(start))?; + let copied = std::io::copy(&mut reader.by_ref().take(len), output)?; + if copied != len { + return Err(Error::SourceChanged( + "entry source size changed while writing", + )); + } + } + } + Ok(()) + } +} + +/// What a fragment that is not the last one reports about its own bytes. +#[derive(Clone, Copy)] +struct FragmentChecksums { + crc32: u32, + hash: [u8; 32], +} + +/// Discards what it is given and keeps the running checksums. +struct ChecksumSink { + crc: crate::crc32::Crc32, + hash: crate::rar50::blake2sp::Hasher, +} + +impl Default for ChecksumSink { + fn default() -> Self { + Self { + crc: crate::crc32::Crc32::new(), + hash: crate::rar50::blake2sp::Hasher::new(), + } + } +} + +impl Write for ChecksumSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.crc.update(buf); + self.hash.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// A member ready to be sliced across volumes. +struct VolumeMember<'a> { + name: &'a [u8], + is_directory: bool, + mtime: Option, + mtime_nanoseconds: Option, + file_times: Option, + attributes: u64, + host_os: u64, + unpacked_size: u64, + crc32: u32, + hash: [u8; 32], + compression_info: u64, + payload_len: u64, + source: FragmentSource, + /// Encryption record for the file header, when the payload is encrypted. + encryption: Option<([u8; 16], [u8; 16], [u8; 12])>, +} + +struct MemberProgress<'a, 'p> { + entries: &'a [ArchiveEntry], + work: &'a crate::write_progress::WorkTracker<'p>, +} +impl compress::CompressionProgress for MemberProgress<'_, '_> { + fn is_cancelled(&self) -> bool { + self.work.is_cancelled() + } + fn advance(&self, bytes: u64) -> bool { + self.work.advance(bytes) + } + fn started(&self, index: usize, size: u64) { + self.work + .entry_started(index, self.entries.len(), &self.entries[index].name, size); + } + fn finished(&self, index: usize, size: u64) { + self.work + .entry_finished(index, self.entries.len(), &self.entries[index].name, size); + } +} + +/// Writes a multi-volume archive, handing each volume to `sink` as it is +/// finished rather than keeping the set in memory. +pub(super) fn write_volumes( + entries: &[ArchiveEntry], + plan: EnginePlan<'_>, + max_payload_per_volume: u64, + sink: &mut dyn super::VolumeSink, + resources: &WriterResources, +) -> Result<()> { + if max_payload_per_volume == 0 { + return Err(Error::InvalidArgument("RAR 5 volume payload size is zero")); + } + for entry in entries { + super::validate_entry(entry)?; + } + + let header_keys = if plan.header_encrypted { + let password = header_encryption_password( + plan.header_password + .into_iter() + .chain(entries.iter().filter_map(|entry| entry.password.as_deref())), + )?; + Some(header_encryption_keys(password)?) + } else { + None + }; + + let mut sources = Records::new(entries.len(), resources)?; + for entry in entries { + sources.push(entry.source.clone())?; + } + let total_input = total_input_size(entries)?; + let total_entries = entries.len(); + if let Some(progress) = plan.progress { + progress.report(crate::WriteProgressEvent::OperationStarted { + operation: crate::WriteOperation::Compression, + total_bytes: Some(total_input), + total_entries: Some(total_entries), + pass: 1, + }); + } + let work = crate::write_progress::WorkTracker::new( + plan.progress, + crate::WriteOperation::Compression, + total_input, + ); + let compressed = compress::compress_members_with_context( + &sources, + &plan.compress, + resources, + &MemberProgress { + entries, + work: &work, + }, + &|index, error| member_error(error, &entries[index].name, "compressing"), + )?; + + let mut members = Records::new(entries.len(), resources)?; + for (entry, member) in entries.iter().zip(compressed) { + check_cancelled(plan.progress)?; + members.push( + prepare_volume_member(entry, member, &plan, resources) + .map_err(|error| member_error(error, &entry.name, "preparing volume payload"))?, + )?; + } + if !work.finish() { + return Err(Error::Cancelled); + } + if let Some(progress) = plan.progress { + progress.report(crate::WriteProgressEvent::OperationFinished { + operation: crate::WriteOperation::Compression, + total_bytes: Some(total_input), + total_entries: Some(total_entries), + pass: 1, + }); + } + + report_emission(plan.progress, true); + let mut writer = VolumeWriter { + max_payload_per_volume, + solid: plan.compress.solid, + recovery_percent: plan.recovery_percent, + header_keys: header_keys.as_ref(), + progress: plan.progress, + resources, + sink, + body: None, + payload_in_volume: 0, + volume_index: 0, + }; + + for mut member in members { + writer + .write_member(&mut member) + .map_err(|error| member_error(error, member.name, "writing volume member"))?; + } + writer.finish()?; + check_cancelled(plan.progress)?; + report_emission(plan.progress, false); + Ok(()) +} + +/// Compresses and, if needed, encrypts one member into a form that can be cut +/// at any byte boundary. +fn prepare_volume_member<'a>( + entry: &'a ArchiveEntry, + member: CompressedMember, + plan: &EnginePlan<'_>, + resources: &WriterResources, +) -> Result> { + let progress = plan.progress; + check_cancelled(progress)?; + let compression_info = compress::member_compression_info(&plan.compress, &member)?; + let plain_len = if member.store { + member.input_size + } else { + member.packed.len() + }; + + match entry.password.as_deref() { + Some(password) => { + let mut salt = [0u8; 16]; + let mut iv = [0u8; 16]; + crate::write_stream::fill_entropy( + &mut salt, + "RAR 5 writer could not generate encryption salt", + )?; + crate::write_stream::fill_entropy( + &mut iv, + "RAR 5 writer could not generate encryption IV", + )?; + let keys = + Rar50Keys::derive(password, salt, WRITE_KDF_COUNT_LOG).map_err(Error::from)?; + + // A volume boundary can fall anywhere, and the cipher runs as one + // chain over the member, so encrypt it up front into scratch + // storage and slice the ciphertext. + let mut encrypted = Spool::create(resources)?; + const ENCRYPT_CHUNK: usize = 64 * 1024; + let _permit = resources.acquire_cancellable(ENCRYPT_CHUNK as u64, 0, &|| { + progress.is_some_and(ProgressReporter::is_cancelled) + })?; + if member.store { + let source = PreparedSource::new(&entry.source, &member); + let mut reader = source.open()?; + encrypt_reader_to( + &mut reader, + plain_len, + &mut encrypted, + &keys, + iv, + ENCRYPT_CHUNK, + progress, + resources, + )?; + reader.finish(&source, source.len)?; + } else { + let mut packed = member.packed; + packed.rewind()?; + encrypt_reader_to( + &mut packed, + plain_len, + &mut encrypted, + &keys, + iv, + ENCRYPT_CHUNK, + progress, + resources, + )?; + } + let payload_len = encrypted.len(); + encrypted.park(); + Ok(VolumeMember { + name: &entry.name, + is_directory: entry.is_directory, + mtime: entry.mtime, + mtime_nanoseconds: entry.mtime_nanoseconds, + file_times: entry.file_times, + attributes: entry.attributes, + host_os: entry.host_os, + unpacked_size: member.input_size, + crc32: keys.checked_crc_mac(member.crc32)?, + hash: keys.checked_hash_mac(member.hash)?, + compression_info, + payload_len, + source: FragmentSource::Packed(encrypted), + encryption: Some((salt, iv, keys.checked_password_record()?)), + }) + } + None => Ok(VolumeMember { + name: &entry.name, + is_directory: entry.is_directory, + mtime: entry.mtime, + mtime_nanoseconds: entry.mtime_nanoseconds, + file_times: entry.file_times, + attributes: entry.attributes, + host_os: entry.host_os, + unpacked_size: member.input_size, + crc32: member.crc32, + hash: member.hash, + compression_info, + payload_len: plain_len, + source: if member.store { + FragmentSource::Stored { + prepared: PreparedSource::new(&entry.source, &member), + emitted: Owned::new(ChecksumSink::default(), resources)?, + } + } else { + FragmentSource::Packed(member.packed) + }, + encryption: None, + }), + } +} + +struct VolumeWriter<'a> { + max_payload_per_volume: u64, + solid: bool, + recovery_percent: Option, + header_keys: Option<&'a HeaderEncryptionKeys>, + progress: Option>, + resources: &'a WriterResources, + sink: &'a mut dyn super::VolumeSink, + /// Body of the volume being filled, held on disk rather than in memory. + body: Option, + payload_in_volume: u64, + volume_index: u64, +} + +impl VolumeWriter<'_> { + /// Cuts one member across as many volumes as it takes. + fn write_member(&mut self, member: &mut VolumeMember) -> Result<()> { + check_cancelled(self.progress)?; + let mut start = 0u64; + let mut split_before = false; + // Zero-length members still need a header of their own. + loop { + // A volume that filled exactly is left open until something asks + // for room, because until then we cannot say whether it is the last + // one, and its end-of-archive block has to say which. + if self.body.is_some() && self.payload_in_volume == self.max_payload_per_volume { + self.finish_volume(true)?; + } + if self.body.is_none() { + self.start_volume()?; + } + let room = self.max_payload_per_volume - self.payload_in_volume; + let remaining = member.payload_len - start; + let fragment_len = room.min(remaining); + let split_after = start + fragment_len < member.payload_len; + // A fragment that is not the last one has no whole-member checksum + // to report, so the field carries the CRC32 of the bytes this + // volume stores. That is what WinRAR puts there, and it lets a + // single volume be checked on its own. + let fragment_checksums = match split_after { + true => Some( + member + .source + .checksums_range(start, fragment_len, self.progress)?, + ), + false => None, + }; + + let header = fragment_header( + member, + fragment_len, + split_before, + fragment_checksums, + self.header_keys, + self.resources, + )?; + let body = self.body.as_mut().expect("volume started"); + body.write_all(&header)?; + member.source.emit_range( + start, + fragment_len, + &mut CancellableIo { + inner: body, + progress: self.progress, + }, + fragment_checksums, + self.resources, + )?; + + self.payload_in_volume += fragment_len; + start += fragment_len; + split_before = true; + + if start >= member.payload_len { + return Ok(()); + } + } + } + + fn finish(mut self) -> Result<()> { + // The public volume entry point rejects empty member lists; each + // nonempty member starts a volume, even when its payload is empty. + self.finish_volume(false) + } + + fn start_volume(&mut self) -> Result<()> { + self.body = Some(Spool::create(self.resources)?); + self.payload_in_volume = 0; + Ok(()) + } + + fn finish_volume(&mut self, more_volumes_follow: bool) -> Result<()> { + let mut body = self.body.take().expect("volume started"); + let volume_number = self.volume_index; + self.volume_index += 1; + self.payload_in_volume = 0; + + let head_crypt = match self.header_keys { + Some(keys) => { + let mut block = Bytes::new(self.resources); + write_head_crypt(&mut block, keys, self.resources)?; + block + } + None => Bytes::new(self.resources), + }; + + let mut main_flags = crate::rar50::MHFL_VOLUME | crate::rar50::MHFL_VOLUME_NUMBER; + if self.solid { + main_flags |= MHFL_SOLID; + } + if self.recovery_percent.is_some() { + main_flags |= MHFL_RECOVERY; + } + + let layout = resolve_layout( + &LayoutInputs { + header_encrypted: self.header_keys.is_some(), + head_crypt_len: head_crypt.len() as u64, + main_flags, + volume_number: Some(volume_number), + archive_metadata: None, + metadata_record: None, + body_len: body.len(), + quick_open_payload_len: None, + recovery_percent: self.recovery_percent, + }, + self.resources, + )?; + + check_cancelled(self.progress)?; + let raw_output = self.sink.start_volume(volume_number)?; + let mut output = CancellableIo { + inner: raw_output, + progress: self.progress, + }; + let mut mirror = match self.recovery_percent { + Some(_) => Some(Spool::create(self.resources)?), + None => None, + }; + let mut written; + { + let mut tee = Tee { + output: &mut output, + mirror: mirror.as_mut(), + }; + let main = match self.header_keys { + Some(keys) => encrypted_main_header_block( + &keys.keys, + main_flags, + Some(volume_number), + &layout.main_extra, + self.resources, + )?, + None => { + let mut main = Bytes::new(self.resources); + write_main_header( + &mut main, + main_flags, + Some(volume_number), + &layout.main_extra, + self.resources, + )?; + main + } + }; + debug_assert_eq!(main.len() as u64, layout.main_header_len); + + tee.write_all(RAR50_SIGNATURE)?; + tee.write_all(&head_crypt)?; + tee.write_all(&main)?; + body.rewind()?; + std::io::copy(&mut body, &mut tee)?; + written = RAR50_SIGNATURE.len() as u64 + + head_crypt.len() as u64 + + main.len() as u64 + + body.len(); + } + + if let Some(recovery_percent) = self.recovery_percent { + let mirror = mirror.as_mut().expect("recovery mirrors the volume"); + debug_assert_eq!(layout.recovery_prefix_len, Some(mirror.len())); + written += write_recovery_service( + recovery_percent, + mirror, + self.header_keys, + self.resources, + self.progress, + &mut output, + )?; + } + + let end_flags = if more_volumes_follow { + crate::rar50::EFL_NEXT_VOLUME + } else { + 0 + }; + let end = match self.header_keys { + Some(keys) => encrypted_header_block( + &keys.keys, + HEAD_END, + 0, + None, + &super::end_header_specific(end_flags), + &[], + &[], + self.resources, + )?, + None => { + let mut end = Bytes::new(self.resources); + write_end_header(&mut end, end_flags, self.resources)?; + end + } + }; + output.write_all(&end)?; + written += end.len() as u64; + output.flush()?; + drop(output); + + self.sink.finish_volume(volume_number, written) + } +} + +/// Builds the file header for one fragment of a member. +/// +/// Every fragment repeats the member's metadata. The last one carries the +/// member's own checksum and hash; the rest carry `fragment` over the bytes +/// they store, which is what WinRAR puts there and lets one volume be checked +/// on its own. Encrypted fragments checksum the ciphertext, so that check needs +/// no password. +/// +/// Both a CRC32 and a hash record go on every fragment. A reader takes the hash +/// record as the member's checksum type when it has one, and unrar picks that +/// type from the first fragment and compares it against the last, so a fragment +/// that offered only a CRC32 while the last offered a hash would fail a member +/// that is perfectly intact. +fn fragment_header( + member: &VolumeMember, + fragment_len: u64, + split_before: bool, + fragment: Option, + header_keys: Option<&HeaderEncryptionKeys>, + resources: &WriterResources, +) -> Result { + let split_after = fragment.is_some(); + let mut extra = Bytes::new(resources); + if let Some((salt, iv, check_value)) = member.encryption { + write_file_encryption_record(&mut extra, salt, iv, check_value)?; + } + write_hash_record_with_value(&mut extra, fragment.map_or(member.hash, |f| f.hash))?; + super::headers::write_mtime_record(&mut extra, member.mtime, member.mtime_nanoseconds)?; + if let Some(times) = member.file_times { + write_extra_record(&mut extra, super::super::FHEXTRA_HTIME, ×.encode()?)?; + } + let specific = file_specific( + member.name, + member.unpacked_size, + fragment.map_or(member.crc32, |f| f.crc32), + member.attributes, + member.mtime.filter(|_| member.mtime_nanoseconds.is_none()), + member.compression_info, + member.host_os, + member.is_directory, + resources, + )?; + + // Every fragment has a hash record, so its extra area is never empty. + let mut flags = HFL_DATA | HFL_EXTRA; + if split_before { + flags |= crate::rar50::HFL_SPLIT_BEFORE; + } + if split_after { + flags |= crate::rar50::HFL_SPLIT_AFTER; + } + match header_keys { + Some(keys) => encrypted_header_block( + &keys.keys, + HEAD_FILE, + flags, + Some(fragment_len), + &specific, + &extra, + &[], + resources, + ), + None => block_header_image( + HEAD_FILE, + flags, + Some(fragment_len), + &specific, + &extra, + resources, + ), + } +} + +fn report_emission(progress: Option>, started: bool) { + use crate::{WriteOperation, WriteProgressEvent}; + if let Some(progress) = progress { + progress.report(if started { + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Emission, + total_bytes: None, + total_entries: None, + pass: 1, + } + } else { + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Emission, + total_bytes: None, + total_entries: None, + pass: 1, + } + }); + } +} + +fn member_error(error: Error, name: &[u8], operation: &'static str) -> Error { + // Cancellation remains an operation-level result. A streaming fallback can + // already have supplied the same member context; do not duplicate it. + if error.kind() == crate::ErrorKind::Cancelled + || error + .entry_context() + .is_some_and(|(existing, _)| existing == name) + { + error + } else { + error.at_entry(name.to_vec(), operation) + } +} + +#[cfg(test)] +mod input_size_tests { + use super::*; + + #[test] + fn writer_adapters_flush_the_output_and_preserve_checksums() { + struct FlushingSink { + bytes: Vec, + flushes: usize, + fail: bool, + } + + impl Write for FlushingSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.flushes += 1; + if self.fail { + Err(std::io::Error::other("injected flush failure")) + } else { + Ok(()) + } + } + } + + let mut output = FlushingSink { + bytes: Vec::new(), + flushes: 0, + fail: false, + }; + { + let mut tee = Tee { + output: &mut output, + mirror: None, + }; + tee.write_all(b"payload").unwrap(); + tee.flush().unwrap(); + } + assert_eq!(output.bytes, b"payload"); + assert_eq!(output.flushes, 1); + output.fail = true; + assert_eq!( + Tee { + output: &mut output, + mirror: None, + } + .flush() + .unwrap_err() + .to_string(), + "injected flush failure" + ); + assert_eq!(output.flushes, 2); + + let mut checksum = ChecksumSink::default(); + checksum.write_all(b"payload").unwrap(); + checksum.flush().unwrap(); + assert_eq!(checksum.crc.finish(), crate::crc32::crc32(b"payload")); + } + + fn virtual_entry(name: &[u8], length: u64) -> ArchiveEntry { + ArchiveEntry::new( + name.to_vec(), + crate::EntrySource::from_opener(length, || panic!("measuring input opened a source")), + ) + } + + #[test] + fn total_input_size_accepts_empty_and_exact_u64_maximum() { + assert_eq!(total_input_size(&[]).unwrap(), 0); + let entries = [ + virtual_entry(b"first", u64::MAX - 1), + virtual_entry(b"last", 1), + virtual_entry(b"empty", 0), + ]; + assert_eq!(total_input_size(&entries).unwrap(), u64::MAX); + } + + #[test] + fn total_input_size_preserves_source_errors_and_cancellation() { + struct Broken(bool); + impl crate::streaming::SourceFactory for Broken { + fn len(&self) -> Result { + if self.0 { + Err(Error::Cancelled) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected metadata failure", + ) + .into()) + } + } + fn open(&self) -> Result> { + panic!("measuring input opened a source") + } + } + for cancelled in [false, true] { + let entry = ArchiveEntry::new( + b"broken".to_vec(), + crate::EntrySource::from_factory(Broken(cancelled)), + ); + let error = total_input_size(&[entry]).unwrap_err(); + if cancelled { + assert_eq!(error, Error::Cancelled); + assert!(error.entry_context().is_none()); + } else { + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.entry_context().unwrap(), + (b"broken".as_slice(), "preparing") + ); + } + } + } +} + +#[cfg(test)] +mod quick_open_tests { + use super::*; + + #[test] + fn streamed_records_match_buffered_encoding_at_vint_boundaries() { + for distance in [0, 127, 128, 16383, 16384, u64::MAX] { + for size in [0, 1, 123, 127, 128, 16383, 16384] { + let header = vec![0xa5; size]; + let mut body = Vec::new(); + write_vint(&mut body, 0); + write_vint(&mut body, distance); + write_vint(&mut body, size as u64); + body.extend_from_slice(&header); + let mut framed = Vec::new(); + write_vint(&mut framed, body.len() as u64); + framed.extend_from_slice(&body); + let mut expected = crate::crc32::crc32(&framed).to_le_bytes().to_vec(); + expected.extend_from_slice(&framed); + let mut actual = Vec::new(); + let mut crc = crate::crc32::Crc32::new(); + append_quick_open_entry(&mut actual, &mut crc, distance, &header).unwrap(); + assert_eq!(actual, expected); + assert_eq!(crc.finish(), crate::crc32::crc32(&actual)); + } + } + } +} + +#[cfg(test)] +mod service_payload_tests { + use super::*; + + #[test] + fn prepared_services_borrow_input_and_stream_identical_ciphertext() { + for size in [0usize, 1, 15, 16, 65535, 65536, 65537, 131089] { + let data: Vec = (0..size).map(|i| (i % 251) as u8).collect(); + let plain = stored_service_block(b"CMT", &data, &[], None, &WriterResources::default()) + .unwrap(); + let Payload::Borrowed(borrowed) = plain.payload else { + panic!("expected borrowed service") + }; + assert_eq!(borrowed.as_ptr(), data.as_ptr()); + assert_eq!(borrowed.len(), data.len()); + #[cfg(feature = "encryption")] + { + let encrypted = encrypted_service_block( + b"CMT", + &data, + &[], + b"secret", + None, + &WriterResources::default(), + ) + .unwrap(); + let Payload::Encrypted { plain, keys, iv } = &encrypted.payload else { + panic!("expected streaming encryption") + }; + let PlainPayload::Borrowed(borrowed) = plain.as_ref() else { + panic!("expected borrowed plaintext") + }; + assert_eq!(borrowed.as_ptr(), data.as_ptr()); + let mut expected = data.clone(); + expected.resize(size.div_ceil(16) * 16, 0); + crate::crypto::rar50::Rar50Cipher::new(keys.key, *iv) + .encrypt_in_place(&mut expected) + .unwrap(); + assert_eq!(encrypted.payload_len, expected.len() as u64); + let mut actual = Vec::new(); + write_payload( + encrypted.payload, + &mut actual, + &WriterResources::default(), + None, + ) + .unwrap(); + assert_eq!(actual, expected, "service size {size}"); + } + } + } +} + +#[cfg(test)] +mod preparation_name_tests { + use super::*; + #[test] + fn allocation_free_target_length_preserves_rar50_mapping_rules() { + for bytes in [ + b"plain".as_slice(), + b"\xffinvalid", + "é/å��å­—".as_bytes(), + "\u{e080}".as_bytes(), + "\u{fffe}a\u{e080}\u{e0ff}é".as_bytes(), + "\u{fffe}\u{fffe}".as_bytes(), + ] { + assert_eq!( + decoded_rar50_name_len(bytes), + crate::filename::decode_rar50(bytes).len() + ); + } + } +} + +#[cfg(test)] +mod emission_ledger_tests { + use super::*; + use crate::codec::workspace::Allowance; + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn source_disappearing_at_compression_start_retains_member_context() { + use std::sync::atomic::{AtomicBool, Ordering}; + for volumes in [false, true] { + let scratch = crate::scratch::case("source-disappears-before-compression"); + let path = scratch.join("source"); + std::fs::write(&path, b"payload").unwrap(); + let entries = [ArchiveEntry::new( + b"missing.bin".to_vec(), + crate::EntrySource::from_path(&path), + )]; + let resources = WriterResources::new(128 * 1024 * 1024) + .with_max_memory_bytes(128 * 1024 * 1024) + .with_temp_dir(&*scratch); + let removed = AtomicBool::new(false); + let callback = |event: crate::WriteProgressEvent<'_>| { + if matches!( + event, + crate::WriteProgressEvent::OperationStarted { + operation: crate::WriteOperation::Compression, + .. + } + ) && !removed.swap(true, Ordering::Relaxed) + { + std::fs::remove_file(&path).unwrap(); + } + }; + let mut plan = plan(false); + plan.progress = Some(ProgressReporter(&callback)); + let error = if volumes { + let mut output = super::super::CollectedVolumes::new(); + let error = write_volumes(&entries, plan, 64, &mut output, &resources).unwrap_err(); + assert!(output.take().is_empty()); + error + } else { + let mut output = Vec::new(); + let error = write_archive(&entries, plan, &resources, &mut output).unwrap_err(); + assert!(output.is_empty()); + error + }; + assert!(removed.load(Ordering::Relaxed)); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((b"missing.bin".as_slice(), "compressing")) + ); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(resources.managed_memory_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn zero_volume_payload_is_rejected_before_opening_a_sink() { + let entries = [ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + )]; + let mut sink = super::super::CollectedVolumes::new(); + let error = write_volumes( + &entries, + plan(false), + 0, + &mut sink, + &WriterResources::default(), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument); + assert!(sink.take().is_empty()); + } + + #[test] + fn prepared_record_counts_include_empty_archives_comments_and_services() { + // The aggregate bound uses distinct live storage for owned records. + assert!(std::mem::size_of::() >= 2); + assert!(std::mem::size_of::() >= 2); + for count in 0..=2 { + for comment in [false, true] { + let entries: Vec<_> = (0..count) + .map(|index| { + let entry = ArchiveEntry::new( + format!("file{index}").into_bytes(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + ); + if index == 1 { + entry.with_service(super::super::ServiceEntry::new(b"CMT", b"note")) + } else { + entry + } + }) + .collect(); + let mut settings = plan(false); + settings.compress.method = 0; + settings.quick_open = false; + settings.recovery_percent = None; + if !comment { + settings.archive_comment = None; + } + let mut bytes = Vec::new(); + write_archive(&entries, settings, &WriterResources::default(), &mut bytes).unwrap(); + let archive = crate::ArchiveReader::read_owned(bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + assert_eq!(raw.files().count(), count); + assert_eq!( + raw.services().count(), + usize::from(comment) + usize::from(count == 2) + ); + for index in 0..count { + assert_eq!( + archive + .read_member(format!("file{index}").as_bytes(), None) + .unwrap() + .unwrap(), + b"payload" + ); + } + } + } + } + + #[test] + fn compression_completion_cancellation_precedes_archive_and_volume_emission() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct CancelAtCompletion { + armed: AtomicBool, + cancelled: AtomicBool, + finished: AtomicBool, + emitted: AtomicBool, + } + impl crate::WriteProgress for CancelAtCompletion { + fn report(&self, event: crate::WriteProgressEvent<'_>) { + use crate::{WriteOperation, WriteProgressEvent}; + match event { + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Compression, + total_entries: Some(0), + .. + } + | WriteProgressEvent::EntryFinished { + operation: WriteOperation::Compression, + .. + } => self.armed.store(true, Ordering::Relaxed), + WriteProgressEvent::Advanced { + operation: WriteOperation::Compression, + .. + } if self.armed.load(Ordering::Relaxed) => { + self.cancelled.store(true, Ordering::Relaxed); + } + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Compression, + .. + } => self.finished.store(true, Ordering::Relaxed), + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Emission, + .. + } => self.emitted.store(true, Ordering::Relaxed), + _ => {} + } + } + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } + } + for empty in [false, true] { + for volumes in [false, true] { + let scratch = crate::scratch::case("completion-cancellation"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let progress = CancelAtCompletion { + armed: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + finished: AtomicBool::new(false), + emitted: AtomicBool::new(false), + }; + let entries = if empty { + Vec::new() + } else { + vec![ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + )] + }; + let mut settings = plan(false); + settings.archive_comment = None; + settings.recovery_percent = None; + settings.progress = Some(ProgressReporter(&progress)); + let error = if volumes { + let mut sink = super::super::CollectedVolumes::new(); + let error = + write_volumes(&entries, settings, 1024, &mut sink, &resources).unwrap_err(); + assert!(sink.take().is_empty()); + error + } else { + let mut output = Vec::new(); + let error = + write_archive(&entries, settings, &resources, &mut output).unwrap_err(); + assert!(output.is_empty()); + error + }; + assert_eq!(error, Error::Cancelled); + assert!(progress.armed.load(Ordering::Relaxed)); + assert!(progress.cancelled.load(Ordering::Relaxed)); + assert!(!progress.finished.load(Ordering::Relaxed)); + assert!(!progress.emitted.load(Ordering::Relaxed)); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + } + + #[test] + fn quick_open_skips_file_services_but_keeps_plain_comments() { + let entry = ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + ) + .with_service(super::super::ServiceEntry::new(b"CMT", b"file note")); + let mut plan = plan(false); + plan.quick_open = true; + plan.recovery_percent = None; + let mut bytes = Vec::new(); + write_archive(&[entry], plan, &WriterResources::default(), &mut bytes).unwrap(); + + let archive = crate::ArchiveReader::read_owned(bytes).unwrap(); + let raw = archive.as_rar50().unwrap(); + assert!(raw.main.locator().unwrap().quick_open_offset.is_some()); + let quick_open = raw + .services() + .find(|service| service.name == b"QO") + .unwrap(); + let payload = quick_open.packed_data(raw).unwrap(); + fn vint(data: &[u8], cursor: &mut usize) -> usize { + let mut value = 0usize; + for shift in (0..70).step_by(7) { + let byte = data[*cursor]; + *cursor += 1; + value |= ((byte & 0x7f) as usize) << shift; + if byte & 0x80 == 0 { + return value; + } + } + panic!("invalid quick-open record length") + } + let mut cursor = 0; + let mut records = 0; + while cursor < payload.len() { + cursor += 4; // Per-record CRC32. + let body_len = vint(&payload, &mut cursor); + cursor += body_len; + assert!(cursor <= payload.len()); + records += 1; + } + // The plain archive comment and file are cached; the attached service is not. + assert_eq!(records, 2); + assert_eq!( + archive.member_comment_at(0, None).unwrap(), + Some(b"file note".to_vec()) + ); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } + + struct FailingSink; + impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("injected emission failure")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn archive_emission_propagates_each_output_write_failure() { + struct FailOnWrite { + fail_at: usize, + writes: usize, + } + impl Write for FailOnWrite { + fn write(&mut self, data: &[u8]) -> std::io::Result { + let call = self.writes; + self.writes += 1; + if call == self.fail_at { + Err(std::io::Error::other("injected output failure")) + } else { + Ok(data.len()) + } + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let entries = [ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + )]; + let settings = || { + let mut plan = plan(false); + plan.recovery_percent = None; + plan.quick_open = true; + plan + }; + let resources = WriterResources::default(); + let mut counting = FailOnWrite { + fail_at: usize::MAX, + writes: 0, + }; + write_archive(&entries, settings(), &resources, &mut counting).unwrap(); + assert!(counting.writes >= 6); + + let mut member_failure = false; + let mut archive_failure = false; + for fail_at in 0..counting.writes { + let mut sink = FailOnWrite { fail_at, writes: 0 }; + let error = write_archive(&entries, settings(), &resources, &mut sink).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io, "write {fail_at}"); + assert_eq!(sink.writes, fail_at + 1); + if error.entry_context() == Some((b"file".as_slice(), "writing")) { + member_failure = true; + } else { + archive_failure = true; + } + } + assert!(member_failure); + assert!(archive_failure); + } + + #[test] + fn volume_emission_propagates_sink_failures_before_handoff() { + use std::sync::{Arc, Mutex}; + + #[derive(Default)] + struct State { + writes: usize, + finishes: usize, + } + struct Sink { + state: Arc>, + fail_write: Option, + fail_start: bool, + fail_flush: bool, + fail_finish: bool, + } + struct Volume { + state: Arc>, + fail_write: Option, + fail_flush: bool, + } + impl Write for Volume { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let mut state = self.state.lock().unwrap(); + let call = state.writes; + state.writes += 1; + if self.fail_write == Some(call) { + Err(std::io::Error::other("injected volume write failure")) + } else { + Ok(bytes.len()) + } + } + fn flush(&mut self) -> std::io::Result<()> { + if self.fail_flush { + Err(std::io::Error::other("injected volume flush failure")) + } else { + Ok(()) + } + } + } + impl super::super::VolumeSink for Sink { + fn start_volume(&mut self, _: u64) -> Result> { + if self.fail_start { + return Err(std::io::Error::other("injected volume start failure").into()); + } + Ok(Box::new(Volume { + state: self.state.clone(), + fail_write: self.fail_write, + fail_flush: self.fail_flush, + })) + } + fn finish_volume(&mut self, _: u64, _: u64) -> Result<()> { + self.state.lock().unwrap().finishes += 1; + if self.fail_finish { + Err(std::io::Error::other("injected volume finish failure").into()) + } else { + Ok(()) + } + } + } + + let entries = [ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + )]; + let settings = || { + let mut plan = plan(false); + plan.recovery_percent = None; + plan + }; + let resources = WriterResources::default(); + let make_sink = || Sink { + state: Arc::new(Mutex::new(State::default())), + fail_write: None, + fail_start: false, + fail_flush: false, + fail_finish: false, + }; + let mut success = make_sink(); + write_volumes(&entries, settings(), 4096, &mut success, &resources).unwrap(); + let writes = success.state.lock().unwrap().writes; + assert!(writes >= 4); + assert_eq!(success.state.lock().unwrap().finishes, 1); + + for fail_at in 0..writes { + let mut sink = make_sink(); + sink.fail_write = Some(fail_at); + let error = + write_volumes(&entries, settings(), 4096, &mut sink, &resources).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io, "write {fail_at}"); + assert_eq!(sink.state.lock().unwrap().writes, fail_at + 1); + assert_eq!(sink.state.lock().unwrap().finishes, 0); + } + for failure in 0..3 { + let mut sink = make_sink(); + sink.fail_start = failure == 0; + sink.fail_flush = failure == 1; + sink.fail_finish = failure == 2; + let error = + write_volumes(&entries, settings(), 4096, &mut sink, &resources).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + sink.state.lock().unwrap().finishes, + usize::from(failure == 2) + ); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encryption_emission_counts_retained_preparation_and_releases_chunk() { + let data = vec![7; 65537]; + for limit in [65536, 131072] { + let ledger = Allowance::limited(limit); + let resources = WriterResources::default().with_execution_allowance(ledger.clone()); + let retained = Bytes::zeroed(128, &resources).unwrap(); + for fail_sink in [false, true] { + let keys = Rar50Keys::derive(b"secret", [1; 16], WRITE_KDF_COUNT_LOG).unwrap(); + let mut expected = data.clone(); + expected.resize(data.len().div_ceil(16) * 16, 0); + crate::crypto::rar50::Rar50Cipher::new(keys.key, [2; 16]) + .encrypt_in_place(&mut expected) + .unwrap(); + let payload = Payload::Encrypted { + plain: Owned::new(PlainPayload::Borrowed(&data), &resources).unwrap(), + keys, + iv: [2; 16], + }; + let mut actual = Vec::new(); + let result = if fail_sink { + write_payload(payload, &mut FailingSink, &resources, None) + } else { + write_payload(payload, &mut actual, &resources, None) + }; + if limit == 65536 { + assert_eq!(result.unwrap_err().kind(), crate::ErrorKind::ResourceLimit); + assert!(actual.is_empty()); + } else if fail_sink { + assert!(result.is_err()); + } else { + result.unwrap(); + assert_eq!(actual, expected); + } + assert_eq!(ledger.used(), 128); + assert_eq!(resources.workspace_in_use(), 0); + } + drop(retained); + assert_eq!(ledger.used(), 0); + } + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_emission_shares_ledger_in_resident_and_striped_modes() { + let scratch = crate::scratch::case("recovery-emission-ledger"); + let data = vec![7; 131072]; + for estimated_limit in [16384, 8 * 1024 * 1024] { + let base = WriterResources::new(estimated_limit).with_temp_dir(&*scratch); + let mut prefix = Spool::create(&base).unwrap(); + prefix.write_all(&data).unwrap(); + let mut expected = Vec::new(); + write_recovery_service(10, &mut prefix, None, &base, None, &mut expected).unwrap(); + for limit in [131072, 8 * 1024 * 1024] { + let ledger = Allowance::limited(limit); + let resources = base.clone().with_execution_allowance(ledger.clone()); + let retained = Bytes::zeroed(128, &resources).unwrap(); + let mut actual = Vec::new(); + let result = + write_recovery_service(10, &mut prefix, None, &resources, None, &mut actual); + if limit == 131072 { + assert_eq!(result.unwrap_err().kind(), crate::ErrorKind::ResourceLimit); + assert!(actual.is_empty()); + } else { + result.unwrap(); + assert_eq!(actual, expected); + let cancel = CancelRecovery { + cancelled: std::sync::atomic::AtomicBool::new(false), + }; + let error = write_recovery_service( + 10, + &mut prefix, + None, + &resources, + Some(ProgressReporter(&cancel)), + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Cancelled); + assert_eq!(ledger.used(), 128); + + assert!(write_recovery_service( + 10, + &mut prefix, + None, + &resources, + None, + &mut FailingSink + ) + .is_err()); + } + assert_eq!(ledger.used(), 128); + assert_eq!(resources.workspace_in_use(), 0); + drop(retained); + assert_eq!(ledger.used(), 0); + // The caller's prefix survives; temporary parity and payload files do not. + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 1); + } + drop(prefix); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[cfg(feature = "recovery")] + struct CancelRecovery { + cancelled: std::sync::atomic::AtomicBool, + } + #[cfg(feature = "recovery")] + impl crate::WriteProgress for CancelRecovery { + fn report(&self, event: crate::WriteProgressEvent<'_>) { + if matches!( + event, + crate::WriteProgressEvent::Advanced { + operation: crate::WriteOperation::Recovery, + .. + } + ) { + self.cancelled + .store(true, std::sync::atomic::Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.cancelled.load(std::sync::atomic::Ordering::Relaxed) + } + } + + fn plan<'a>(encrypted: bool) -> EnginePlan<'a> { + let options = crate::codec::rar50::EncodeOptions::new(8).with_max_match_distance(131072); + EnginePlan { + compress: CompressPlan { + algorithm_version: 0, + encode_options: options, + dictionary_size: 131072, + block_size: 4096, + solid: false, + method: 0, + filter_policy: super::super::FilterPolicy::None, + candidates: vec![options].into(), + }, + recovery_percent: cfg!(feature = "recovery").then_some(10), + header_encrypted: encrypted, + header_password: encrypted.then_some(b"secret".as_slice()), + archive_comment: Some(if encrypted { + ArchiveCommentPlan::Encrypted { + data: b"comment", + password: b"secret", + } + } else { + ArchiveCommentPlan::Plain(b"comment") + }), + archive_metadata: None, + metadata_record: None, + locked: false, + quick_open: false, + progress: None, + } + } + + #[test] + fn volume_emission_reports_compression_completion() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let finished = AtomicBool::new(false); + let report = |event: crate::WriteProgressEvent<'_>| { + if matches!( + event, + crate::WriteProgressEvent::OperationFinished { + operation: crate::WriteOperation::Compression, + total_bytes: Some(7), + total_entries: Some(1), + .. + } + ) { + finished.store(true, Ordering::Relaxed); + } + }; + let mut plan = plan(false); + plan.progress = Some(ProgressReporter(&report)); + plan.recovery_percent = None; + let entries = [ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + )]; + let mut sink = super::super::CollectedVolumes::new(); + write_volumes(&entries, plan, 4096, &mut sink, &WriterResources::default()).unwrap(); + assert!(finished.load(Ordering::Relaxed)); + assert_eq!(sink.take().len(), 1); + } + + #[test] + fn an_existing_member_context_is_not_wrapped_again() { + let original = + Error::InvalidArgument("source changed").at_entry(b"file".to_vec(), "reading"); + let error = member_error(original, b"file", "writing"); + assert_eq!(error.entry_context(), Some((b"file".as_slice(), "reading"))); + } + + #[test] + fn archive_and_volume_emission_keep_the_ledger_reusable() { + let scratch = crate::scratch::case("archive-emission-ledger"); + for encrypted in crate::test_features::encryption_cases() { + let mut entry = ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(vec![7; 16384]), + ); + if encrypted { + entry = entry.with_password(b"secret"); + } + let entries = [entry]; + let base = WriterResources::default().with_temp_dir(&*scratch); + let ledger = Allowance::limited(8 * 1024 * 1024); + let resources = base.clone().with_execution_allowance(ledger.clone()); + let mut expected = Vec::new(); + write_archive(&entries, plan(encrypted), &base, &mut expected).unwrap(); + let mut actual = Vec::new(); + write_archive(&entries, plan(encrypted), &resources, &mut actual).unwrap(); + if !encrypted { + assert_eq!(actual, expected); + } else { + assert_eq!(actual.len(), expected.len()); + } + assert_eq!(ledger.used(), 0); + assert!( + write_archive(&entries, plan(encrypted), &resources, &mut FailingSink).is_err() + ); + assert_eq!(ledger.used(), 0); + let mut expected = super::super::CollectedVolumes::new(); + write_volumes(&entries, plan(encrypted), 4096, &mut expected, &base).unwrap(); + let small = Allowance::limited(32768); + let constrained = base.clone().with_execution_allowance(small.clone()); + assert_eq!( + write_volumes( + &entries, + plan(encrypted), + 4096, + &mut super::super::CollectedVolumes::new(), + &constrained, + ) + .unwrap_err() + .kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(small.used(), 0); + let mut actual = super::super::CollectedVolumes::new(); + write_volumes(&entries, plan(encrypted), 4096, &mut actual, &resources).unwrap(); + let expected = expected.take(); + let actual = actual.take(); + assert_eq!(actual.len(), 4); + if !encrypted { + assert_eq!(actual, expected); + } else { + assert_eq!( + actual.iter().map(Vec::len).collect::>(), + expected.iter().map(Vec::len).collect::>() + ); + } + assert_eq!(ledger.used(), 0); + assert_eq!(resources.workspace_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + + #[test] + fn archive_preparation_refusals_release_capacity_at_each_boundary() { + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + )]; + let mut limit = 0; + let mut refusals = 0; + loop { + let resources = WriterResources::default().with_max_preparation_bytes(limit); + let mut settings = plan(false); + settings.compress.method = 0; + settings.recovery_percent = None; + settings.quick_open = true; + let mut output = Vec::new(); + match write_archive(&entries, settings, &resources, &mut output) { + Ok(()) => { + assert!(!output.is_empty()); + break; + } + Err(error) => match error.root_cause() { + Error::WriterPreparationLimitExceeded { + limit: actual_limit, + required, + .. + } => { + assert_eq!(*actual_limit, limit); + assert!(*required > limit); + refusals += 1; + drop(Records::::new(limit as usize, &resources).unwrap()); + limit = *required; + assert!(limit < 64 * 1024, "tiny archive used too much preparation"); + } + _ => panic!("unexpected preparation failure: {error}"), + }, + } + } + assert!(refusals > 3, "the test must cross several admission sites"); + } + + #[test] + fn volume_preparation_refusals_release_capacity_at_each_boundary() { + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + )]; + let mut limit = 0; + let mut refusals = 0; + loop { + let resources = WriterResources::default().with_max_preparation_bytes(limit); + let mut settings = plan(false); + settings.compress.method = 0; + settings.recovery_percent = None; + let mut volumes = super::super::CollectedVolumes::new(); + match write_volumes(&entries, settings, 4096, &mut volumes, &resources) { + Ok(()) => { + assert_eq!(volumes.take().len(), 1); + break; + } + Err(error) => match error.root_cause() { + Error::WriterPreparationLimitExceeded { + limit: actual_limit, + required, + .. + } => { + assert_eq!(*actual_limit, limit); + assert!(*required > limit); + refusals += 1; + drop(Records::::new(limit as usize, &resources).unwrap()); + limit = *required; + assert!( + limit < 1024 * 1024, + "tiny volumes used {limit} bytes of preparation" + ); + } + _ => panic!("unexpected preparation failure: {error}"), + }, + } + } + assert!(refusals > 3, "the test must cross several admission sites"); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_archive_preparation_refusals_release_capacity() { + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + ) + .with_password(b"secret")]; + let mut limit = 0; + let mut refusals = 0; + loop { + let resources = WriterResources::default().with_max_preparation_bytes(limit); + let mut settings = plan(true); + settings.compress.method = 0; + settings.recovery_percent = None; + let mut output = Vec::new(); + match write_archive(&entries, settings, &resources, &mut output) { + Ok(()) => { + assert!(!output.is_empty()); + break; + } + Err(error) => match error.root_cause() { + Error::WriterPreparationLimitExceeded { required, .. } => { + assert!(*required > limit); + refusals += 1; + drop(Records::::new(limit as usize, &resources).unwrap()); + limit = *required; + assert!( + limit < 64 * 1024, + "encrypted archive used too much preparation" + ); + } + _ => panic!("unexpected preparation failure: {error}"), + }, + } + } + assert!(refusals > 3); + } + + #[test] + fn oversized_archive_metadata_is_refused_during_layout_before_output() { + let name = vec![b'm'; 4096]; + let mut settings = plan(false); + settings.archive_comment = None; + settings.recovery_percent = None; + settings.archive_metadata = Some(super::super::ArchiveMetadataEntry { + name: Some(&name), + creation_time: Some(1), + }); + let resources = WriterResources::default().with_max_preparation_bytes(128); + let mut output = Vec::new(); + let error = write_archive(&[], settings, &resources, &mut output).unwrap_err(); + assert!(matches!( + error, + Error::WriterPreparationLimitExceeded { limit: 128, .. } + )); + assert!(output.is_empty()); + drop(Records::::new(128, &resources).unwrap()); + } + + #[test] + fn archive_service_preparation_failure_preserves_member_context() { + for encrypted in crate::test_features::encryption_cases() { + let mut service = super::super::ServiceEntry::new(b"CMT", b"note"); + if encrypted { + service = service.with_password(b"secret"); + } + let mut entry = ArchiveEntry::new( + b"member".to_vec(), + crate::EntrySource::from_bytes(b"payload".to_vec()), + ); + for _ in 0..8 { + entry = entry.with_service(service.clone()); + } + let mut limit = 0; + let mut service_refused = false; + loop { + let mut settings = plan(false); + settings.archive_comment = None; + settings.recovery_percent = None; + let resources = WriterResources::default().with_max_preparation_bytes(limit); + let mut output = Vec::new(); + let result = write_archive( + std::slice::from_ref(&entry), + settings, + &resources, + &mut output, + ); + assert_eq!(resources.workspace_in_use(), 0); + drop(Records::::new(limit as usize, &resources).unwrap()); + match result { + Ok(()) => { + assert!(!output.is_empty()); + assert!(service_refused); + break; + } + Err(error) => { + assert!(output.is_empty()); + let Error::WriterPreparationLimitExceeded { + required, + limit: actual, + .. + } = error.root_cause() + else { + panic!("unexpected preparation failure: {error}"); + }; + assert_eq!(*actual, limit); + assert!(*required > limit); + service_refused |= error.entry_context() + == Some((b"member".as_slice(), "preparing service")); + limit = *required; + assert!(limit < 64 * 1024); + } + } + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_service_rejects_oversized_name_after_extra_preparation() { + let name = vec![b'n'; 4096]; + let resources = WriterResources::default().with_max_preparation_bytes(512); + let error = encrypted_service_block(&name, b"payload", &[], b"secret", None, &resources) + .err() + .expect("the service name exceeds the preparation quota"); + assert!(matches!( + error, + Error::WriterPreparationLimitExceeded { limit: 512, .. } + )); + drop(Records::::new(512, &resources).unwrap()); + } + + #[test] + fn archive_releases_preparation_for_each_injected_admission_failure() { + use std::sync::atomic::Ordering; + + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + )]; + let run = |resources: &WriterResources| -> Result> { + let mut settings = plan(false); + settings.compress.method = 0; + settings.recovery_percent = None; + settings.quick_open = true; + let mut output = Vec::new(); + write_archive(&entries, settings, resources, &mut output)?; + Ok(output) + }; + let (resources, attempts) = + WriterResources::default().refuse_preparation_growth_at(usize::MAX); + assert!(!run(&resources).unwrap().is_empty()); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 10); + assert!(count < 200, "unexpectedly many preparation admissions"); + assert_eq!(resources.preparation_in_use(), 0); + + for index in 0..count { + let (resources, _) = WriterResources::default().refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + } + } + + #[test] + fn volumes_release_preparation_for_each_injected_admission_failure() { + use std::sync::atomic::Ordering; + + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + )]; + let run = |resources: &WriterResources| -> Result { + let mut settings = plan(false); + settings.compress.method = 0; + settings.recovery_percent = None; + let mut volumes = super::super::CollectedVolumes::new(); + write_volumes(&entries, settings, 4096, &mut volumes, resources)?; + Ok(volumes.take().len()) + }; + let (resources, attempts) = + WriterResources::default().refuse_preparation_growth_at(usize::MAX); + assert_eq!(run(&resources).unwrap(), 1); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 10); + assert!(count < 200, "unexpectedly many preparation admissions"); + assert_eq!(resources.preparation_in_use(), 0); + + for index in 0..count { + let (resources, _) = WriterResources::default().refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + } + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_releases_preparation_for_each_injected_admission_failure() { + use std::sync::atomic::Ordering; + + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + )]; + let run = |resources: &WriterResources| -> Result> { + let mut settings = plan(false); + settings.compress.method = 0; + let mut output = Vec::new(); + write_archive(&entries, settings, resources, &mut output)?; + Ok(output) + }; + let (resources, attempts) = + WriterResources::default().refuse_preparation_growth_at(usize::MAX); + assert!(!run(&resources).unwrap().is_empty()); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 10); + assert!(count < 200, "unexpectedly many preparation admissions"); + assert_eq!(resources.preparation_in_use(), 0); + + for index in 0..count { + let (resources, _) = WriterResources::default().refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + } + } + + #[test] + #[cfg(feature = "recovery")] + fn volume_recovery_releases_preparation_for_each_injected_admission_failure() { + use std::sync::atomic::Ordering; + + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + )]; + let run = |resources: &WriterResources| -> Result { + let mut settings = plan(false); + settings.compress.method = 0; + let mut volumes = super::super::CollectedVolumes::new(); + write_volumes(&entries, settings, 4096, &mut volumes, resources)?; + Ok(volumes.take().len()) + }; + let (resources, attempts) = + WriterResources::default().refuse_preparation_growth_at(usize::MAX); + assert!(run(&resources).unwrap() > 0); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 10); + assert!(count < 200, "unexpectedly many preparation admissions"); + assert_eq!(resources.preparation_in_use(), 0); + + for index in 0..count { + let (resources, _) = WriterResources::default().refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_outputs_release_late_preparation_refusals() { + use std::sync::atomic::Ordering; + + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + ) + .with_password(b"secret")]; + for volumes in [false, true] { + let run = |resources: &WriterResources| -> Result<()> { + let mut settings = plan(true); + settings.compress.method = 0; + settings.recovery_percent = None; + settings.archive_comment = None; + if volumes { + write_volumes( + &entries, + settings, + 4096, + &mut super::super::CollectedVolumes::new(), + resources, + ) + } else { + write_archive(&entries, settings, resources, &mut Vec::new()) + } + }; + let (resources, attempts) = + WriterResources::default().refuse_preparation_growth_at(usize::MAX); + run(&resources).unwrap(); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 3); + assert!(count < 200); + assert_eq!(resources.preparation_in_use(), 0); + + for index in count - 3..count { + let (resources, _) = WriterResources::default().refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "volume mode {volumes}, admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + } + } + } + + #[test] + #[cfg(feature = "encryption")] + #[cfg(feature = "recovery")] + fn encrypted_recovery_releases_late_preparation_refusals() { + use std::sync::atomic::Ordering; + + let entries = [ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(b"contents".to_vec()), + ) + .with_password(b"secret")]; + let run = |resources: &WriterResources| -> Result<()> { + let mut settings = plan(true); + settings.compress.method = 0; + settings.archive_comment = None; + write_archive(&entries, settings, resources, &mut Vec::new()) + }; + let (resources, attempts) = + WriterResources::default().refuse_preparation_growth_at(usize::MAX); + run(&resources).unwrap(); + let count = attempts.load(Ordering::Relaxed); + assert!(count > 5); + assert_eq!(resources.preparation_in_use(), 0); + + for index in count - 5..count { + let (resources, _) = WriterResources::default().refuse_preparation_growth_at(index); + let error = run(&resources).unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::WriterFailure("injected preparation admission failure"), + "admission {index}" + ); + assert_eq!(resources.preparation_in_use(), 0, "admission {index}"); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_volume_reread_failure_keeps_member_context() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + let opens = Arc::new(AtomicUsize::new(0)); + let source = crate::EntrySource::from_opener(8, { + let opens = opens.clone(); + move || { + if opens.fetch_add(1, Ordering::Relaxed) == 0 { + Ok(Box::new(std::io::Cursor::new(b"contents".to_vec()))) + } else { + Err(std::io::Error::other("injected encrypted reread failure").into()) + } + } + }); + let entries = [ArchiveEntry::new(b"payload".to_vec(), source).with_password(b"secret")]; + let mut settings = plan(false); + settings.compress.method = 0; + settings.archive_comment = None; + settings.recovery_percent = None; + let mut volumes = super::super::CollectedVolumes::new(); + let error = write_volumes( + &entries, + settings, + 4096, + &mut volumes, + &WriterResources::default(), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((b"payload".as_slice(), "preparing volume payload")) + ); + assert_eq!(opens.load(Ordering::Relaxed), 2); + assert!(volumes.take().is_empty()); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_compressed_volume_payload_propagates_spool_refusal() { + let scratch = crate::scratch::case("encrypted-volume-spool-refusal"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let entry = ArchiveEntry::new( + b"payload".to_vec(), + crate::EntrySource::from_bytes(vec![b'A'; 64 * 1024]), + ) + .with_password(b"secret"); + let mut settings = plan(false); + settings.compress.method = 1; + settings.recovery_percent = None; + let member = compress::compress_members_reporting( + std::slice::from_ref(&entry.source), + settings.compress.clone(), + &resources, + &|_| true, + ) + .unwrap() + .remove(0); + assert!(!member.store); + + let constrained = resources.clone().with_max_spool_bytes(0); + let error = prepare_volume_member(&entry, member, &settings, &constrained) + .err() + .expect("encrypted ciphertext must be charged to its spool"); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/rar50/write/filter_policy.rs b/arca-rar/vendor/rars/src/rar50/write/filter_policy.rs new file mode 100644 index 00000000..1bbbc678 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/filter_policy.rs @@ -0,0 +1,975 @@ +use super::*; +use crate::codec::rar50::EncodeOptions; +#[cfg(test)] +use crate::codec::rar50::{ + encode_lz_member_with_options, encode_lz_member_with_options_and_progress, Unpack50Encoder, +}; +#[cfg(test)] +use crate::codec::workspace::Allowance; +use crate::codec::workspace::{Budget, Buffer}; +use crate::filter_search::{encode_pass, EncodeProgress, OwnedSearch}; + +fn borrow_progress<'a>( + progress: &'a mut Option<&mut dyn FnMut(EncodeProgress) -> bool>, +) -> Option<&'a mut dyn FnMut(EncodeProgress) -> bool> { + match progress { + Some(report) => Some(&mut **report), + None => None, + } +} + +#[cfg(test)] +pub(super) fn encode_member_with_filter_policy_and_progress( + data: &[u8], + algorithm_version: u8, + policy: &FilterPolicy, + options: EncodeOptions, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, +) -> Result> { + policy_with_allowance( + data, + algorithm_version, + policy, + options, + progress, + &Allowance::default(), + ) + .map(Buffer::into_vec) +} +#[cfg(test)] +pub(super) fn encode_member_with_filter_policy_candidates_and_progress( + data: &[u8], + algorithm_version: u8, + policy: &FilterPolicy, + candidates: &[EncodeOptions], + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, +) -> Result> { + candidates_with_allowance( + data, + algorithm_version, + policy, + candidates, + progress, + &Allowance::default(), + ) + .map(Buffer::into_vec) +} +pub(super) fn candidates_with_allowance( + data: &[u8], + algorithm_version: u8, + policy: &FilterPolicy, + candidates: &[EncodeOptions], + mut progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, + allowance: &B, +) -> Result> { + let mut remaining = candidates.iter().copied(); + let first = remaining.next().ok_or(Error::WriterFailure( + "RAR 5 compression level has no encoder options", + ))?; + + // Searching for a filter once and then trying the encoder settings against + // it is the difference between a handful of passes over the member and one + // whole search per setting. + if *policy == FilterPolicy::Auto && auto_size_filter_search_applies(data) { + let (specs, mut best) = choose_owned_filter( + data, + algorithm_version, + first, + borrow_progress(&mut progress), + allowance, + )?; + // The search already encoded the winner at the first setting, so only + // the remaining settings are left to try. + for options in remaining { + let packed = specs_with_allowance( + data, + algorithm_version, + &specs, + options, + borrow_progress(&mut progress), + allowance, + )?; + if packed.len() < best.len() { + best = packed; + } + } + return Ok(best); + } + + let mut best = policy_with_allowance( + data, + algorithm_version, + policy, + first, + borrow_progress(&mut progress), + allowance, + )?; + for options in remaining { + let packed = policy_with_allowance( + data, + algorithm_version, + policy, + options, + borrow_progress(&mut progress), + allowance, + )?; + if packed.len() < best.len() { + best = packed; + } + } + Ok(best) +} + +use crate::filter_search::search_applies as auto_size_filter_search_applies; + +/// How many bytes the encoder will walk while packing this member, for progress +/// to scale by. +pub(super) fn filter_policy_walk_bytes( + data: &[u8], + policy: &FilterPolicy, + _algorithm_version: u8, + encoder_candidates: usize, +) -> u64 { + let member = data.len() as u64; + if *policy != FilterPolicy::Auto || !auto_size_filter_search_applies(data) { + return member * encoder_candidates.max(1) as u64; + } + crate::filter_search::walk_bytes_for_kinds( + data, + SCREENED_KINDS.len() as u64, + encoder_candidates, + ) +} + +/// Whether a member compression did not help is better off stored. +/// +/// Takes lengths rather than the bytes, because the streaming path decides this +/// for a payload it has already spilled to disk. +pub(super) fn should_store_compressed_payload( + unpacked: u64, + packed: u64, + solid: bool, + policy: &FilterPolicy, +) -> bool { + // A solid member is decoded against the dictionary the members before it + // filled, so this writer can never go back and store one. + crate::write_plan::StoreFallback::new() + .filter_requested(matches!(policy, FilterPolicy::Explicit(_))) + .applies(solid, unpacked as usize, packed as usize) +} + +/// How hard each level looks for a match, and where the default sits. +/// +/// The ladder used to run 8, 32, 64, 48, 64 with the absent level on 256, so +/// asking for the most compression got you less of it than asking for nothing: +/// `--level 5` searched 64 positions where the default searched 256, and level +/// 4 searched fewer than level 3. On 4 MiB of man pages the default packed +/// 679,368 bytes and `--level 5` packed 691,274. +/// +/// Measured on that member, packed bytes against seconds, at lookahead 2: +/// +/// ```text +/// candidates 64 128 256 512 1024 +/// packed 686,891 680,315 675,746 672,645 670,061 +/// seconds 1.67 2.35 3.53 4.71 6.65 +/// ``` +/// +/// It flattens out: the last doubling buys 0.4% for another two seconds. So +/// the top of the ladder sits at 512 rather than chasing it, and level 3 stays +/// near what the default already cost. +fn match_candidates_for_level(level: u8) -> Result { + match level { + 0 => Ok(0), + 1 => Ok(8), + 2 => Ok(32), + 3 => Ok(32), + 4 => Ok(256), + 5 => Ok(512), + _ => Err(Error::InvalidArgument( + "RAR 5 compression level must be in the range 0..5", + )), + } +} + +/// The level an absent `--level` means. Resolved once, so the effort spent, the +/// method byte written and the fallback ladder cannot disagree about it. +pub(super) const RAR50_DEFAULT_LEVEL: u8 = 3; + +pub(super) fn resolved_level(level: Option) -> u8 { + level.unwrap_or(RAR50_DEFAULT_LEVEL) +} + +pub(super) fn encode_options_for_level( + level: Option, + dictionary_size: u64, +) -> Result { + let level = resolved_level(level); + let candidates = match_candidates_for_level(level)?; + let max_match_distance = usize::try_from(dictionary_size).map_err(|_| { + Error::InvalidArgument("RAR 5 dictionary size exceeds this platform's address space") + })?; + Ok(EncodeOptions::new(candidates) + // Looking one byte further ahead before settling for the match at hand + // is both smaller and quicker than not: at 64 candidates it took 4,383 + // bytes off that member and half a second off the encode, because a + // longer match covers ground the finder then does not have to search. + // Four is worse than one, which is its own question (#83). + .with_lazy_matching(level >= 2) + .with_lazy_lookahead(2) + // Everything above level 1 parses by shortest path. + // + // This used to start at level 5, which left three rungs in the middle + // doing something that could not close the distance to WinRAR whatever + // it was tuned to. Measured over the bench corpus against each level's + // own WinRAR, byte weighted, as the parse moved down the ladder: + // + // ```text + // parse from m2 m3 m4 m5 + // level 5 +7.27% +4.13% +4.16% -0.38% + // level 4 +7.27% +4.13% -0.39% -0.38% + // level 3 +3.36% -0.62% -0.39% -0.38% + // level 2 -2.07% -0.65% -0.42% -0.41% + // ``` + // + // Search depth is worth tenths of a percent across that whole range and + // the parse is worth several, so there is no setting of the other knobs + // that makes a greedy rung competitive. What the depth still buys is + // the ordering: at 32, 128, 256 and 512 candidates the parse packs + // steadily smaller for steadily more time, which is what a level is. + // + // Level 1 keeps the greedy parse. It is the rung that exists to be + // quick, it already beats WinRAR's own level 1 by 2.71%, and it is the + // only place left to go when the parse is too slow for the job. + .with_optimal_parse(level >= 2) + .with_max_match_distance(max_match_distance) + // Arca: levels 2 and 3 stop looking at 64 bytes and price each block + // once or twice rather than three times. On Arca's 144 MiB mixed corpus + // that took level 3 from about 130 s to about 25 s for 3% more output; + // levels 4 and 5 keep the full parse for the smallest archive. + .with_nice_length(if matches!(level, 2 | 3) { 64 } else { NICE_LENGTH_FULL }) + .with_parse_passes(match level { + 2 => 1, + 3 => 2, + _ => PARSE_PASSES_FULL, + })) +} + +/// What levels 4 and 5 keep: the encoder's own defaults. +const NICE_LENGTH_FULL: usize = EncodeOptions::new(0).nice_length; +const PARSE_PASSES_FULL: usize = EncodeOptions::new(0).parse_passes; + +/// The encoder settings to try for a level, smallest output kept. +/// +/// One, now. Level 5 used to try levels 4 down to 1 as well, to catch a member +/// that packs smaller with less search. Over the whole bench corpus it caught +/// one byte, on one member, for four extra whole-member encodes; dropping it +/// took 28% off level 5's time and nothing off its output. +/// +/// The shape stays because the caller still asks a list, and because a future +/// setting that genuinely competes rather than merely searches less would go +/// here. +pub(super) fn encode_option_candidates_for_level( + level: Option, + dictionary_size: u64, +) -> Result<[EncodeOptions; 1]> { + Ok([encode_options_for_level(level, dictionary_size)?]) +} + +pub(super) fn rar50_algorithm_version(options: WriterOptions, dictionary_size: u64) -> Result { + match options.target { + crate::ArchiveVersion::Rar50 => Ok(0), + crate::ArchiveVersion::Rar70 => { + if dictionary_size_fields(0, dictionary_size).is_ok() { + Ok(0) + } else { + Ok(1) + } + } + _ => Err(Error::UnsupportedVersion(options.target)), + } +} + +/// The level written into the member header. +/// +/// It is the resolved level, so an archive says what it was compressed at. An +/// absent level used to write 1 while spending more search than `--level 5` +/// did, which described neither what was asked for nor what was done. +pub(super) fn compression_method_for_level(level: Option) -> Result { + let level = resolved_level(level); + match_candidates_for_level(level)?; + Ok(level) +} + +/// The largest dictionary the writer picks on its own at each level. +/// +/// The format goes far higher and `--dict-size` still does. What stops the +/// writer reaching for it is that the hash chains lengthen with the window, so +/// a wider one costs time that is not always repaid. How often it is repaid +/// depends entirely on the data. Over 16 MiB of manpage text and 16 MiB of +/// shared libraries at level 3: +/// +/// ```text +/// dictionary text seconds binary seconds +/// 1 MiB 3,480,749 6.6 4,646,481 14.1 +/// 2 MiB 2,223,009 8.1 4,590,488 25.3 +/// 4 MiB 2,061,307 14.8 4,514,109 53.0 +/// 8 MiB 2,043,138 21.7 4,505,410 83.2 +/// 16 MiB 2,040,730 27.6 4,507,232 105.0 +/// ``` +/// +/// A 1 MiB cap costs text 41% and buys binaries most of their speed. One +/// number cannot serve both, so the level picks, which is what a level is for. +/// Text keeps paying up to about 8 MiB; binaries stop at 4 and are slightly +/// worse by 16. +/// +/// Level 2 was raised off the 1 MiB floor when it gained the shortest-path +/// parse: on 4 MiB of manpage text it took 24,315 bytes off for a tenth of a +/// second, which is the same trade level 3 was already making. +fn fitted_dictionary_cap(level: u8) -> u64 { + let megabytes = match level { + 0..=1 => 1, + 2 => 4, + 3 => 4, + 4 => 8, + _ => 16, + }; + megabytes * 1024 * 1024 +} + +/// The smallest dictionary that still reaches past `content`. +/// +/// A window larger than the data cannot match anything extra, so a small member +/// keeps a small window and stays as quick as it was. Sizes are the format's +/// own: 128 KiB doubled. +pub(super) fn fitted_dictionary_size(content: u64, level: u8) -> u64 { + let cap = fitted_dictionary_cap(level); + let mut size = DEFAULT_RAR50_DICTIONARY_SIZE; + while size < cap && size <= content { + size *= 2; + } + size +} + +/// The dictionary to write with, either the caller's or one fitted to the data. +/// +/// `content` is what one window has to reach across: the whole archive when the +/// members share a dictionary, otherwise the largest member. `memory_limit` is +/// the workspace budget the write has to stay inside. +/// +/// A dictionary the caller named is validated and used as given: if it does not +/// fit the budget the write fails saying so, which is the answer to a request +/// that cannot be met. A fitted one is shrunk to fit instead, because it has no +/// business failing a write that the smaller default would have finished. +pub(super) fn dictionary_size_for_options( + options: WriterOptions, + content: u64, + memory_limit: u64, +) -> Result { + let size = match options.dictionary_size { + Some(size) => size, + None => { + let level = resolved_level(options.compression_level); + let mut fitted = fitted_dictionary_size(content, level); + // Whether the level parses optimally decides how much a window + // costs, and the level owns that answer. The size passed here does + // not change it. + let optimal_parse = encode_options_for_level(Some(level), fitted)?.optimal_parse; + while fitted > DEFAULT_RAR50_DICTIONARY_SIZE + && super::streaming_lz_workspace( + fitted, + crate::codec::rar50::MAX_LZ_BLOCK_SIZE, + optimal_parse, + ) > memory_limit + { + fitted /= 2; + } + fitted + } + }; + validate_dictionary_size(options.target, size)?; + Ok(size) +} + +pub(super) fn validate_dictionary_size(target: crate::ArchiveVersion, size: u64) -> Result<()> { + match target { + crate::ArchiveVersion::Rar50 => dictionary_size_fields(0, size).map(|_| ()), + crate::ArchiveVersion::Rar70 => dictionary_size_fields(0, size) + .or_else(|_| dictionary_size_fields(1, size)) + .map(|_| ()), + _ => Err(Error::UnsupportedVersion(target)), + } +} + +pub(super) fn dictionary_size_fields(algorithm_version: u8, size: u64) -> Result<(u8, u8)> { + if size == 0 { + return Err(Error::InvalidArgument( + "RAR 5 dictionary size must be non-zero", + )); + } + match algorithm_version { + 0 => { + if size < DEFAULT_RAR50_DICTIONARY_SIZE { + return Err(Error::InvalidArgument( + "RAR 5 v0 dictionary size must be at least 128 KiB", + )); + } + if !size.is_multiple_of(DEFAULT_RAR50_DICTIONARY_SIZE) { + return Err(Error::InvalidArgument( + "RAR 5 v0 dictionary size must be a power-of-two multiple of 128 KiB", + )); + } + let multiple = size / DEFAULT_RAR50_DICTIONARY_SIZE; + if !multiple.is_power_of_two() { + return Err(Error::InvalidArgument( + "RAR 5 v0 dictionary size must be a power-of-two multiple of 128 KiB", + )); + } + let power = multiple.trailing_zeros(); + if power > 15 { + return Err(Error::InvalidArgument( + "RAR 5 v0 dictionary size exceeds 4 GiB", + )); + } + Ok((power as u8, 0)) + } + 1 => { + if !size.is_multiple_of(4096) { + return Err(Error::InvalidArgument( + "RAR 7 dictionary size must be a multiple of 4 KiB", + )); + } + let mut units = size / 4096; + let mut power = 0u8; + while units > 63 { + if !units.is_multiple_of(2) || power == 31 { + return Err(Error::InvalidArgument( + "RAR 7 dictionary size is not encodable", + )); + } + units /= 2; + power += 1; + } + if units < 32 { + return Err(Error::InvalidArgument( + "RAR 7 dictionary size must be at least 128 KiB", + )); + } + Ok((power, (units - 32) as u8)) + } + _ => Err(Error::InvalidArgument( + "RAR 5 unknown compression algorithm version", + )), + } +} + +#[cfg(test)] +mod dictionary_encoding_tests { + use super::*; + + #[cfg(target_pointer_width = "32")] + #[test] + fn encode_options_refuse_a_valid_dictionary_above_the_host_limit() { + let size = 1u64 << 32; + assert_eq!(dictionary_size_fields(0, size).unwrap(), (15, 0)); + assert_eq!( + encode_options_for_level(Some(3), size).unwrap_err(), + Error::InvalidArgument("RAR 5 dictionary size exceeds this platform's address space") + ); + } + + #[test] + fn rar5_dictionary_boundaries_and_rejections() { + let unit = DEFAULT_RAR50_DICTIONARY_SIZE; + assert_eq!(dictionary_size_fields(0, unit).unwrap(), (0, 0)); + assert_eq!(dictionary_size_fields(0, 1u64 << 32).unwrap(), (15, 0)); + for (size, reason) in [ + ( + unit - 1, + "RAR 5 v0 dictionary size must be at least 128 KiB", + ), + ( + unit + 4096, + "RAR 5 v0 dictionary size must be a power-of-two multiple of 128 KiB", + ), + ( + unit * 3, + "RAR 5 v0 dictionary size must be a power-of-two multiple of 128 KiB", + ), + (1u64 << 33, "RAR 5 v0 dictionary size exceeds 4 GiB"), + ] { + assert_eq!( + dictionary_size_fields(0, size).unwrap_err(), + Error::InvalidArgument(reason), + "size {size}" + ); + } + } + + #[test] + fn rar7_dictionary_boundaries_and_rejections() { + assert_eq!(dictionary_size_fields(1, 128 * 1024).unwrap(), (0, 0)); + assert_eq!(dictionary_size_fields(1, 63 * 4096).unwrap(), (0, 31)); + assert_eq!(dictionary_size_fields(1, 64 * 4096).unwrap(), (1, 0)); + for (size, reason) in [ + (4096, "RAR 7 dictionary size must be at least 128 KiB"), + (65 * 4096, "RAR 7 dictionary size is not encodable"), + ( + 64u64 * (1u64 << 31) * 4096, + "RAR 7 dictionary size is not encodable", + ), + ] { + assert_eq!( + dictionary_size_fields(1, size).unwrap_err(), + Error::InvalidArgument(reason), + "size {size}" + ); + } + } + + #[test] + fn dictionary_and_compression_options_reject_unsupported_encodings() { + for level in [6, u8::MAX] { + assert_eq!( + encode_options_for_level(Some(level), 128 * 1024).unwrap_err(), + Error::InvalidArgument("RAR 5 compression level must be in the range 0..5") + ); + } + let rar5 = WriterOptions::default(); + let rar7 = WriterOptions::new( + crate::ArchiveVersion::Rar70, + crate::FeatureSet::store_only(), + ); + let fractional = 160 * 1024; + assert!(validate_dictionary_size(rar5.target, fractional).is_err()); + validate_dictionary_size(rar7.target, fractional).unwrap(); + assert_eq!(rar50_algorithm_version(rar7, 128 * 1024).unwrap(), 0); + assert_eq!(rar50_algorithm_version(rar7, fractional).unwrap(), 1); + assert_eq!( + rar50_algorithm_version( + WriterOptions::new( + crate::ArchiveVersion::Rar40, + crate::FeatureSet::store_only(), + ), + 128 * 1024, + ) + .unwrap_err(), + Error::UnsupportedVersion(crate::ArchiveVersion::Rar40) + ); + assert_eq!( + validate_dictionary_size(crate::ArchiveVersion::Rar40, 128 * 1024).unwrap_err(), + Error::UnsupportedVersion(crate::ArchiveVersion::Rar40) + ); + assert_eq!( + dictionary_size_fields(2, 128 * 1024).unwrap_err(), + Error::InvalidArgument("RAR 5 unknown compression algorithm version") + ); + } + + #[test] + fn filter_trials_propagate_codec_allocation_refusals() { + use crate::codec::workspace::Allowance; + use crate::filter_search::OwnedSearch; + + let search = Rar50OwnedSearch { + algorithm_version: 0, + allowance: Allowance::limited(0), + }; + let filter = FilterSpec::whole(FilterKind::Delta { channels: 1 }); + let data = vec![42; 4096]; + assert_eq!( + search + .filtered_bytes(&data, std::slice::from_ref(&filter)) + .unwrap_err() + .kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!( + search + .encode_filtered(&data, &[filter], EncodeOptions::new(8), None) + .unwrap_err() + .kind(), + crate::ErrorKind::ResourceLimit + ); + } +} + +pub(super) fn compression_info( + algorithm_version: u8, + method: u8, + dictionary_size: u64, + solid_continuation: bool, +) -> Result { + let (dictionary_power, dictionary_fraction) = + dictionary_size_fields(algorithm_version, dictionary_size)?; + Ok(u64::from(algorithm_version) + | (u64::from(method) << 7) + | (u64::from(dictionary_power) << 10) + | (u64::from(dictionary_fraction) << 15) + | solid_compression_flag(solid_continuation)) +} + +const SCREENED_KINDS: [FilterKind; 5] = [ + FilterKind::Arm, + FilterKind::Delta { channels: 1 }, + FilterKind::Delta { channels: 2 }, + FilterKind::Delta { channels: 3 }, + FilterKind::Delta { channels: 4 }, +]; + +struct Rar50OwnedSearch { + algorithm_version: u8, + allowance: B, +} +impl OwnedSearch for Rar50OwnedSearch { + type Options = EncodeOptions; + type Memory = B; + fn allowance(&self) -> &B { + &self.allowance + } + fn screened_kinds(&self, _: &[u8]) -> Result> { + Ok(Buffer::collect(SCREENED_KINDS, &self.allowance)?) + } + fn detects_x86(&self) -> bool { + true + } + fn max_delta_channels(&self) -> usize { + crate::codec::rar50::MAX_DELTA_CHANNELS + } + fn screen_options(&self, options: EncodeOptions) -> EncodeOptions { + options.with_optimal_parse(false) + } + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result> { + Ok(crate::codec::rar50::filtered_owned_member( + data, + filters, + &self.allowance, + )?) + } + fn encode_plain( + &self, + data: &[u8], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Ok(crate::codec::rar50::encode_owned_member( + data, + self.algorithm_version, + options, + None, + progress, + &self.allowance, + )?) + } + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + Ok(crate::codec::rar50::encode_owned_member( + data, + self.algorithm_version, + options, + Some(filters), + progress, + &self.allowance, + )?) + } +} +fn choose_owned_filter( + data: &[u8], + algorithm_version: u8, + options: EncodeOptions, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, + allowance: &B, +) -> Result> { + crate::filter_search::choose_owned( + &Rar50OwnedSearch { + algorithm_version, + allowance: allowance.clone(), + }, + data, + options, + progress, + ) +} +fn specs_with_allowance( + data: &[u8], + algorithm_version: u8, + filters: &[FilterSpec], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, + allowance: &B, +) -> Result> { + encode_pass(progress, |progress| { + Ok(crate::codec::rar50::encode_owned_member( + data, + algorithm_version, + options, + (!filters.is_empty()).then_some(filters), + progress, + allowance, + )?) + }) +} +fn policy_with_allowance( + data: &[u8], + algorithm_version: u8, + policy: &FilterPolicy, + options: EncodeOptions, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, + allowance: &B, +) -> Result> { + match policy { + FilterPolicy::Auto if auto_size_filter_search_applies(data) => { + choose_owned_filter(data, algorithm_version, options, progress, allowance) + .map(|(_, bytes)| bytes) + } + FilterPolicy::Explicit(filter) => specs_with_allowance( + data, + algorithm_version, + std::slice::from_ref(filter), + options, + progress, + allowance, + ), + _ => specs_with_allowance(data, algorithm_version, &[], options, progress, allowance), + } +} + +/// How RAR 5 measures a filter candidate, for the shared search. +#[cfg(test)] +#[derive(Clone, Copy)] +pub(crate) struct Rar50Search { + pub(crate) algorithm_version: u8, +} + +#[cfg(test)] +impl crate::filter_search::FilterSearch for Rar50Search { + type Options = EncodeOptions; + + fn screened_kinds(&self, _data: &[u8]) -> Vec { + SCREENED_KINDS.to_vec() + } + + fn max_delta_channels(&self) -> usize { + crate::codec::rar50::MAX_DELTA_CHANNELS + } + + fn filtered_bytes(&self, data: &[u8], filters: &[FilterSpec]) -> Result> { + crate::codec::rar50::filtered_lz_member(data, filters) + .map(|(filtered, _)| filtered) + .map_err(Error::from) + } + + /// Screens rank filters against each other, and the optimal parse costs six + /// encodes' worth of time to sharpen a ranking that the same parse on both + /// sides already gets right. Everything else about the settings stays, so a + /// screen still measures what the writer will do. + fn screen_options(&self, options: EncodeOptions) -> EncodeOptions { + options.with_optimal_parse(false) + } + + fn encode_plain( + &self, + data: &[u8], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + match progress { + Some(progress) => encode_lz_member_with_options_and_progress( + data, + self.algorithm_version, + options, + progress, + ), + None => encode_lz_member_with_options(data, self.algorithm_version, options), + } + .map_err(Error::from) + } + + fn encode_filtered( + &self, + data: &[u8], + filters: &[FilterSpec], + options: EncodeOptions, + progress: Option<&mut dyn FnMut(usize) -> bool>, + ) -> Result> { + let mut encoder = Unpack50Encoder::with_options(options); + match progress { + Some(progress) => encoder.encode_member_with_filters_and_progress( + data, + self.algorithm_version, + filters, + progress, + ), + None => encoder.encode_member_with_filters(data, self.algorithm_version, filters), + } + .map_err(Error::from) + } +} + +#[cfg(test)] +pub(super) fn encode_member_with_auto_size_filter_progress( + data: &[u8], + algorithm_version: u8, + options: EncodeOptions, + progress: Option<&mut dyn FnMut(EncodeProgress) -> bool>, +) -> Result> { + policy_with_allowance( + data, + algorithm_version, + &FilterPolicy::Auto, + options, + progress, + &Allowance::default(), + ) + .map(Buffer::into_vec) +} + +pub(super) fn solid_compression_flag(solid_continuation: bool) -> u64 { + if solid_continuation { + 0x40 + } else { + 0 + } +} + +#[cfg(test)] +mod allowance_tests { + use super::*; + + #[test] + fn candidate_search_keeps_the_winner_charged_and_releases_refused_trials() { + let data: Vec = (0..1024u32) + .flat_map(|n| ((n * 71) % 32749).to_le_bytes()) + .collect(); + let candidates = [ + EncodeOptions::new(8).with_max_match_distance(65536), + EncodeOptions::new(16).with_max_match_distance(65536), + ]; + for version in [0, 1] { + for policy in [ + FilterPolicy::None, + FilterPolicy::Auto, + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::Delta { channels: 4 })), + ] { + let expected = candidates_with_allowance( + &data, + version, + &policy, + &candidates, + None, + &Allowance::default(), + ) + .unwrap(); + let mut successes = 0; + let mut refusals = 0; + for limit in [0, 4096, 16384, 65536, 262144, 1048576, 8 * 1048576] { + let allowance = Allowance::limited(limit); + match candidates_with_allowance( + &data, + version, + &policy, + &candidates, + None, + &allowance, + ) { + Ok(packed) => { + successes += 1; + assert_eq!(&*packed, &*expected); + assert!(allowance.used() >= packed.len() as u64); + assert!(allowance.used() > 0); + drop(packed); + } + Err(error) => { + refusals += 1; + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit, "{error}"); + } + } + assert_eq!(allowance.used(), 0); + } + assert!(successes > 0 && refusals >= 3); + } + } + } + + #[test] + fn bounded_sampled_search_preserves_code_and_table_candidates() { + let mut data = vec![0x90u8; 65536]; + for pos in (0..65000).step_by(32) { + data[pos] = 0xe8; + data[pos + 1..pos + 5].copy_from_slice(&(1024i32 - pos as i32).to_le_bytes()); + } + for index in 0..8192u64 { + data.extend_from_slice(&(0x7f80_1234_0000 + index * 24).to_le_bytes()); + data.extend_from_slice(&0x0102_0304_0506_0708u64.to_le_bytes()); + data.extend_from_slice(&(0x4455_6677_0000 | ((index * 7) & 0xffff)).to_le_bytes()); + } + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let expected = choose_owned_filter(&data, 0, options, None, &Allowance::default()).unwrap(); + assert!(!expected.0.is_empty()); + assert!(expected + .0 + .iter() + .any(|spec| matches!(spec.kind, FilterKind::Delta { channels: 24 }))); + let allowance = Allowance::limited(32 * 1048576); + let (specs, packed) = choose_owned_filter(&data, 0, options, None, &allowance).unwrap(); + assert_eq!(&*specs, &*expected.0); + assert_eq!(&*packed, &*expected.1); + let retained = allowance.used(); + assert!(retained > packed.len() as u64); + drop(specs); + assert!(allowance.used() < retained); + assert!(allowance.used() >= packed.len() as u64); + drop(packed); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn cancellation_drops_retained_screens_and_previous_encoder_candidates() { + let data: Vec = (0..1024u32).flat_map(|n| n.to_le_bytes()).collect(); + let options = EncodeOptions::new(8).with_max_match_distance(65536); + let candidates = [options, options.with_optimal_parse(true)]; + for policy in [FilterPolicy::None, FilterPolicy::Auto] { + let mut passes = 0; + let expected = candidates_with_allowance( + &data, + 0, + &policy, + &candidates, + Some(&mut |event| { + passes += usize::from(event == EncodeProgress::PassStarted); + true + }), + &Allowance::default(), + ) + .unwrap(); + assert!(passes >= 2); + for stop in 1..=passes { + let allowance = Allowance::limited(16 * 1048576); + let mut pass = 0; + let result = candidates_with_allowance( + &data, + 0, + &policy, + &candidates, + Some(&mut |event| { + pass += usize::from(event == EncodeProgress::PassStarted); + pass < stop + }), + &allowance, + ); + assert!(matches!(result, Err(Error::Cancelled))); + assert_eq!(allowance.used(), 0, "pass {stop}"); + let retried = + candidates_with_allowance(&data, 0, &policy, &candidates, None, &allowance) + .unwrap(); + assert_eq!(&*retried, &*expected); + drop(retried); + assert_eq!(allowance.used(), 0); + } + } + } +} diff --git a/arca-rar/vendor/rars/src/rar50/write/headers.rs b/arca-rar/vendor/rars/src/rar50/write/headers.rs new file mode 100644 index 00000000..a4b1f5fd --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/headers.rs @@ -0,0 +1,1027 @@ +//! RAR 5 header serialization. +//! +//! Plain header encodings and lengths depend only on their inputs, letting the +//! layout resolver predict positions before emission. Encrypted headers obtain +//! fresh IVs. Prepared images additionally reserve their allocation capacity +//! against the writer resource policy. + +use super::ArchiveMetadataEntry; +#[cfg(test)] +use crate::crc32::crc32; +#[cfg(all(test, feature = "encryption"))] +use crate::crypto::rar50::Rar50Cipher; +use crate::crypto::rar50::{Rar50Keys, WRITE_KDF_COUNT_LOG}; +use crate::rar50::framing::{checked_image_len, image_size_error, HeaderImage, HeaderScratch}; +use crate::rar50::{ + FHEXTRA_CRYPT, FHEXTRA_HASH, FHFL_CRC32, FHFL_DIRECTORY, FHFL_MTIME, HEAD_CRYPT, HEAD_END, + HEAD_MAIN, HFL_EXTRA, MHEXTRA_ARCHIVE_METADATA, MHEXTRA_ARCHIVE_METADATA_NAME, + MHEXTRA_ARCHIVE_METADATA_TIME, MHEXTRA_LOCATOR, MHEXTRA_LOCATOR_QUICK_OPEN, + MHEXTRA_LOCATOR_RECOVERY, +}; +use crate::streaming::preparation::Bytes; +use crate::WriterResources; +use crate::{Error, Result}; + +pub(crate) trait HeaderOutput { + fn append(&mut self, bytes: &[u8]) -> Result<()>; + fn vint(&mut self, value: u64) -> Result<()>; +} +impl HeaderOutput for Bytes { + fn append(&mut self, bytes: &[u8]) -> Result<()> { + self.extend_from_slice(bytes) + } + fn vint(&mut self, value: u64) -> Result<()> { + self.vint(value) + } +} +#[cfg(test)] +impl HeaderOutput for Vec { + fn append(&mut self, bytes: &[u8]) -> Result<()> { + self.extend_from_slice(bytes); + Ok(()) + } + fn vint(&mut self, value: u64) -> Result<()> { + write_vint(self, value); + Ok(()) + } +} + +#[cfg(test)] +pub(super) fn write_vint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +pub(super) fn write_extra_record( + out: &mut impl HeaderOutput, + record_type: u64, + data: &[u8], +) -> Result<()> { + let mut kind = HeaderScratch::<10>::new(); + kind.vint(record_type); + // A slice length is at most isize::MAX, so adding a ten-byte vint fits u64. + out.vint(data.len() as u64 + kind.len() as u64)?; + out.append(kind.as_slice())?; + out.append(data)?; + + Ok(()) +} + +pub(super) fn write_hash_record_with_value( + out: &mut impl HeaderOutput, + hash: [u8; 32], +) -> Result<()> { + let mut record = HeaderScratch::<33>::new(); + record.vint(0); + record.extend_from_slice(&hash); + write_extra_record(out, FHEXTRA_HASH, record.as_slice())?; + + Ok(()) +} + +pub(super) fn write_file_encryption_record( + out: &mut impl HeaderOutput, + salt: [u8; 16], + iv: [u8; 16], + check_value: [u8; 12], +) -> Result<()> { + let mut record = HeaderScratch::<47>::new(); + record.vint(0); + record.vint(0x0003); + record.extend_from_slice(&[WRITE_KDF_COUNT_LOG]); + record.extend_from_slice(&salt); + record.extend_from_slice(&iv); + record.extend_from_slice(&check_value); + write_extra_record(out, FHEXTRA_CRYPT, record.as_slice())?; + + Ok(()) +} + +fn join_record(parts: &[&[u8]], resources: &WriterResources) -> Result { + let len = parts + .iter() + .try_fold(0usize, |total, part| total.checked_add(part.len())) + .ok_or_else(image_size_error)?; + let mut out = Bytes::zeroed(checked_image_len(&[len])?, resources)?; + let mut offset = 0; + for part in parts { + out[offset..offset + part.len()].copy_from_slice(part); + offset += part.len(); + } + Ok(out) +} + +impl HeaderImage<'_> { + fn render( + &self, + keys: Option<&Rar50Keys>, + data: &[u8], + resources: &WriterResources, + ) -> Result { + let mut out = Bytes::zeroed(self.image_len(keys.is_some(), data.len())?, resources)?; + self.render_into(keys, data, &mut out)?; + Ok(out) + } +} + +pub(super) fn block_header_image( + header_type: u64, + flags: u64, + data_size: Option, + type_specific: &[u8], + extra: &[u8], + resources: &WriterResources, +) -> Result { + HeaderImage::new(header_type, flags, data_size, type_specific, extra)?.render( + None, + &[], + resources, + ) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn write_block( + out: &mut impl HeaderOutput, + header_type: u64, + flags: u64, + data_size: Option, + type_specific: &[u8], + extra: &[u8], + data: &[u8], + resources: &WriterResources, +) -> Result<()> { + let header = block_header_image( + header_type, + flags, + data_size, + type_specific, + extra, + resources, + )?; + out.append(&header)?; + out.append(data)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_header_admission_failure_leaves_output_empty() { + let resources = WriterResources::default().with_max_preparation_bytes(0); + let mut output = Bytes::new(&resources); + let error = + write_block(&mut output, HEAD_MAIN, 0, None, &[], &[], &[], &resources).unwrap_err(); + assert!(matches!( + error, + Error::WriterPreparationLimitExceeded { + limit: 0, + used: 0, + .. + } + )); + assert!(output.is_empty()); + } + + #[test] + fn archive_metadata_extra_releases_each_refused_preparation_charge() { + let metadata = ArchiveMetadataEntry { + name: Some(b"archive"), + creation_time: Some(1), + }; + let mut limit = 0; + let mut refusals = 0; + loop { + let resources = WriterResources::default().with_max_preparation_bytes(limit); + match resolved_main_extra(Some(metadata), None, None, &resources) { + Ok(extra) => { + assert!(!extra.is_empty()); + break; + } + Err(Error::WriterPreparationLimitExceeded { required, .. }) => { + assert!(required > limit); + refusals += 1; + drop(Bytes::zeroed(limit as usize, &resources).unwrap()); + limit = required; + assert!(limit < 4096); + } + Err(error) => panic!("unexpected metadata error: {error}"), + } + } + assert!(refusals > 2); + } +} + +/// Encrypts a header block under `keys`, returning `iv || ciphertext || data`. +/// +/// Each block gets its own IV and its own CBC chain, so headers can be built +/// and emitted one at a time without any cross-block state. +#[allow(clippy::too_many_arguments)] +pub(crate) fn encrypted_header_block( + keys: &Rar50Keys, + header_type: u64, + flags: u64, + data_size: Option, + type_specific: &[u8], + extra: &[u8], + data: &[u8], + resources: &WriterResources, +) -> Result { + HeaderImage::new(header_type, flags, data_size, type_specific, extra)?.render( + Some(keys), + data, + resources, + ) +} + +pub(super) fn stored_file_specific( + name: &[u8], + unpacked_size: u64, + data_crc32: u32, + attributes: u64, + mtime: Option, + host_os: u64, + resources: &WriterResources, +) -> Result { + file_specific( + name, + unpacked_size, + data_crc32, + attributes, + mtime, + 0, + host_os, + false, + resources, + ) +} + +// Keep the on-disk file header fields explicit at this serialization boundary. +#[allow(clippy::too_many_arguments)] +pub(super) fn file_specific( + name: &[u8], + unpacked_size: u64, + data_crc32: u32, + attributes: u64, + mtime: Option, + compression_info: u64, + host_os: u64, + is_directory: bool, + resources: &WriterResources, +) -> Result { + if name.is_empty() { + return Err(Error::InvalidArgument("RAR 5 file name is empty")); + } + let mut file_flags = FHFL_CRC32; + if is_directory { + file_flags |= FHFL_DIRECTORY; + } + if mtime.is_some() { + file_flags |= FHFL_MTIME; + } + + let mut specific = HeaderScratch::<68>::new(); + specific.vint(file_flags); + specific.vint(unpacked_size); + specific.vint(attributes); + if let Some(mtime) = mtime { + specific.extend_from_slice(&mtime.to_le_bytes()); + } + specific.extend_from_slice(&data_crc32.to_le_bytes()); + specific.vint(compression_info); + specific.vint(host_os); + specific.vint(name.len() as u64); + join_record(&[specific.as_slice(), name], resources) +} + +pub(super) fn write_mtime_record( + extra: &mut impl HeaderOutput, + seconds: Option, + nanos: Option, +) -> Result<()> { + if let (Some(seconds), Some(nanos)) = (seconds, nanos) { + // Unix time + mtime + nanoseconds. Emit the complete timestamp here, + // with no base-header time competing with the higher precision record. + let mut record = HeaderScratch::<9>::new(); + record.extend_from_slice(&[0x13]); + record.extend_from_slice(&seconds.to_le_bytes()); + record.extend_from_slice(&nanos.to_le_bytes()); + write_extra_record(extra, super::super::FHEXTRA_HTIME, record.as_slice())?; + } + + Ok(()) +} + +fn metadata_image( + flags: u64, + name: Option<&[u8]>, + time: &[u8], + resources: &WriterResources, +) -> Result { + let mut fields = HeaderScratch::<20>::new(); + fields.vint(flags); + if let Some(name) = name { + fields.vint(name.len() as u64); + } + let name = name.unwrap_or_default(); + let mut kind = HeaderScratch::<10>::new(); + kind.vint(MHEXTRA_ARCHIVE_METADATA); + let body_len = checked_image_len(&[kind.len(), fields.len(), name.len(), time.len()])?; + let mut size = HeaderScratch::<10>::new(); + size.vint(body_len as u64); + join_record( + &[ + size.as_slice(), + kind.as_slice(), + fields.as_slice(), + name, + time, + ], + resources, + ) +} + +pub(super) fn archive_metadata_record( + metadata: ArchiveMetadataEntry<'_>, + resources: &WriterResources, +) -> Result { + if metadata.name.is_none() && metadata.creation_time.is_none() { + return Err(Error::InvalidArgument( + "RAR 5 archive metadata writer needs a name or creation time", + )); + } + if metadata.name.is_some() && metadata.creation_time.is_none() { + return Err(Error::InvalidArgument( + "RAR 5 archive metadata name needs a creation time", + )); + } + let mut flags = 0; + if metadata.name.is_some() { + flags |= MHEXTRA_ARCHIVE_METADATA_NAME; + } + // A creation time is required, with or without a name. + flags |= MHEXTRA_ARCHIVE_METADATA_TIME; + + if metadata.name.is_some_and(|name| name.is_empty()) { + return Err(Error::InvalidArgument( + "RAR 5 archive metadata name is empty", + )); + } + let time = metadata.creation_time.map(u64::to_le_bytes); + metadata_image( + flags, + metadata.name, + time.as_ref().map_or(&[], |bytes| bytes.as_slice()), + resources, + ) +} + +pub(super) fn write_locator_record( + out: &mut impl HeaderOutput, + quick_open_offset: Option, + recovery_record_offset: Option, +) -> Result<()> { + let mut flags = 0; + if quick_open_offset.is_some() { + flags |= MHEXTRA_LOCATOR_QUICK_OPEN; + } + if recovery_record_offset.is_some() { + flags |= MHEXTRA_LOCATOR_RECOVERY; + } + + let mut record = HeaderScratch::<21>::new(); + record.vint(flags); + if let Some(quick_open_offset) = quick_open_offset { + record.vint(quick_open_offset); + } + if let Some(recovery_record_offset) = recovery_record_offset { + record.vint(recovery_record_offset); + } + write_extra_record(out, MHEXTRA_LOCATOR, record.as_slice())?; + + Ok(()) +} + +pub(super) fn resolved_main_extra( + archive_metadata: Option>, + quick_open_offset: Option, + recovery_offset: Option, + resources: &WriterResources, +) -> Result { + let mut main_extra = Bytes::new(resources); + let locator_quick_open_offset = quick_open_offset.or_else(|| archive_metadata.map(|_| 0)); + if locator_quick_open_offset.is_some() || recovery_offset.is_some() { + write_locator_record(&mut main_extra, locator_quick_open_offset, recovery_offset)?; + } + if let Some(archive_metadata) = archive_metadata { + main_extra.extend_from_slice(&archive_metadata_record(archive_metadata, resources)?)?; + } + Ok(main_extra) +} + +pub(super) fn write_main_header( + out: &mut impl HeaderOutput, + archive_flags: u64, + volume_number: Option, + extra: &[u8], + resources: &WriterResources, +) -> Result<()> { + let mut specific = HeaderScratch::<20>::new(); + specific.vint(archive_flags); + if let Some(volume_number) = volume_number { + specific.vint(volume_number); + } + write_block( + out, + HEAD_MAIN, + if extra.is_empty() { 0 } else { HFL_EXTRA }, + None, + specific.as_slice(), + extra, + &[], + resources, + ) +} + +pub(super) fn encrypted_main_header_block( + keys: &Rar50Keys, + archive_flags: u64, + volume_number: Option, + extra: &[u8], + resources: &WriterResources, +) -> Result { + let mut specific = HeaderScratch::<20>::new(); + specific.vint(archive_flags); + if let Some(volume_number) = volume_number { + specific.vint(volume_number); + } + encrypted_header_block( + keys, + HEAD_MAIN, + if extra.is_empty() { 0 } else { HFL_EXTRA }, + None, + specific.as_slice(), + extra, + &[], + resources, + ) +} + +pub(crate) struct HeaderEncryptionKeys { + pub(super) keys: Rar50Keys, + pub(super) salt: [u8; 16], +} + +pub(super) fn header_encryption_keys(password: &[u8]) -> Result { + crate::crypto::require_encryption()?; + let mut salt = [0u8; 16]; + crate::write_stream::fill_entropy( + &mut salt, + "RAR 5 writer could not generate encryption salt", + )?; + let keys = Rar50Keys::derive(password, salt, WRITE_KDF_COUNT_LOG).map_err(Error::from)?; + Ok(HeaderEncryptionKeys { keys, salt }) +} + +/// Header encryption covers the whole archive, so every member, service and +/// comment has to be locked with the same password. +pub(super) fn header_encryption_password<'a>( + mut passwords: impl Iterator, +) -> Result<&'a [u8]> { + let first = passwords.next().ok_or(Error::NeedPassword)?; + for password in passwords { + if password != first { + return Err(Error::InvalidArgument( + "RAR 5 header-encrypted writer needs one shared password", + )); + } + } + Ok(first) +} + +pub(super) fn write_head_crypt( + out: &mut impl HeaderOutput, + header_keys: &HeaderEncryptionKeys, + resources: &WriterResources, +) -> Result<()> { + let mut specific = HeaderScratch::<31>::new(); + specific.vint(0); + specific.vint(0x0001); + specific.extend_from_slice(&[WRITE_KDF_COUNT_LOG]); + specific.extend_from_slice(&header_keys.salt); + specific.extend_from_slice(&header_keys.keys.checked_password_record()?); + write_block( + out, + HEAD_CRYPT, + 0, + None, + specific.as_slice(), + &[], + &[], + resources, + ) +} + +pub(crate) fn write_end_header( + out: &mut impl HeaderOutput, + end_flags: u64, + resources: &WriterResources, +) -> Result<()> { + write_block( + out, + HEAD_END, + 0, + None, + &end_header_specific(end_flags), + &[], + &[], + resources, + ) +} + +pub(crate) fn end_header_specific(end_flags: u64) -> HeaderScratch<10> { + let mut specific = HeaderScratch::new(); + specific.vint(end_flags); + specific +} + +pub(crate) fn retained_archive_metadata( + metadata: &crate::rar50::ArchiveMetadataRecord, + resources: &WriterResources, +) -> Result { + if metadata.flags & !15 != 0 + || (metadata.flags & 8 != 0 && metadata.flags & 4 == 0) + || (metadata.flags & 1 != 0) != metadata.name.is_some() + || (metadata.flags & 2 != 0) != metadata.creation_time.is_some() + || metadata.flags & 3 == 0 + || metadata + .name + .as_ref() + .is_some_and(|name| name.is_empty() || std::str::from_utf8(name).is_err()) + { + return Err(Error::InvalidArgument( + "unsupported archive metadata record", + )); + } + let mut time_bytes = HeaderScratch::<8>::new(); + if let Some(time) = metadata.creation_time { + if metadata.flags & 4 != 0 && metadata.flags & 8 == 0 { + time_bytes.extend_from_slice( + &u32::try_from(time) + .map_err(|_| Error::InvalidArgument("archive Unix timestamp exceeds 32 bits"))? + .to_le_bytes(), + ); + } else { + time_bytes.extend_from_slice(&time.to_le_bytes()); + } + } + metadata_image( + metadata.flags, + metadata.name.as_deref(), + time_bytes.as_slice(), + resources, + ) +} + +/// An immutable final header image. Free bytes before releasing admission. +pub(super) struct PreparedHeader { + bytes: Bytes, + _charge: Option, +} + +impl std::ops::Deref for PreparedHeader { + type Target = [u8]; + fn deref(&self) -> &[u8] { + &self.bytes + } +} + +pub(super) fn prepared_header_image( + header_type: u64, + flags: u64, + data_size: Option, + type_specific: &[u8], + extra: &[u8], + keys: Option<&HeaderEncryptionKeys>, + resources: &crate::WriterResources, +) -> Result { + let image = HeaderImage::new(header_type, flags, data_size, type_specific, extra)?; + let length = image.header_len(keys.is_some())?; + let charge = resources.reserve_prepared_header(length as u64)?; + let bytes = image.render(keys.map(|keys| &keys.keys), &[], resources)?; + Ok(PreparedHeader { + bytes, + _charge: charge, + }) +} + +#[cfg(test)] +mod prepared_header_tests { + use super::*; + use crate::{ErrorKind, WriterResources}; + + #[test] + fn exact_images_match_existing_serialization_and_include_encryption_padding() { + #[cfg(feature = "encryption")] + let keys = header_encryption_keys(b"secret").unwrap(); + for len in [0, 1, 15, 16, 120, 127, 128, 16384] { + let specific = vec![0x51; len]; + for encrypted in crate::test_features::encryption_cases() { + let key = { + #[cfg(feature = "encryption")] + { + encrypted.then_some(&keys) + } + #[cfg(not(feature = "encryption"))] + { + None + } + }; + let expected = block_header_image( + 2, + HFL_EXTRA, + Some(u64::MAX), + &specific, + b"extra", + &crate::WriterResources::default(), + ) + .unwrap(); + let size = if encrypted { + 16 + expected.len().div_ceil(16) * 16 + } else { + expected.len() + }; + let limited = + WriterResources::default().with_max_prepared_header_bytes(size as u64 - 1); + let error = prepared_header_image( + 2, + HFL_EXTRA, + Some(u64::MAX), + &specific, + b"extra", + key, + &limited, + ) + .err() + .unwrap(); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!( + error, + Error::WriterPreparedHeaderLimitExceeded { + limit: size as u64 - 1, + required: size as u64, + used: 0 + } + ); + let resources = + WriterResources::default().with_max_prepared_header_bytes(size as u64); + let header = prepared_header_image( + 2, + HFL_EXTRA, + Some(u64::MAX), + &specific, + b"extra", + key, + &resources, + ) + .unwrap(); + assert_eq!(header.bytes.capacity(), size); + assert!(resources.clone().reserve_prepared_header(1).is_err()); + if encrypted { + #[cfg(feature = "encryption")] + { + let iv = header[..16].try_into().unwrap(); + let mut plain = header[16..].to_vec(); + Rar50Cipher::new(keys.keys.key, iv) + .decrypt_in_place(&mut plain) + .unwrap(); + assert_eq!(&plain[..expected.len()], expected.as_slice()); + assert!(plain[expected.len()..].iter().all(|byte| *byte == 0)); + } + } else { + assert_eq!(&*header, expected.as_slice()); + } + drop(header); + assert!(resources.reserve_prepared_header(size as u64).is_ok()); + } + } + } + + #[test] + fn shared_header_quota_is_released_on_unwind() { + let resources = WriterResources::default().with_max_prepared_header_bytes(1024); + let failure = std::panic::catch_unwind(|| { + let _header = + prepared_header_image(2, 0, None, b"data", &[], None, &resources).unwrap(); + assert!(resources.reserve_prepared_header(1024).is_err()); + panic!("injected failure"); + }); + assert!(failure.is_err()); + assert!(resources.reserve_prepared_header(1024).is_ok()); + } +} + +#[cfg(test)] +mod scratch_tests { + use super::*; + + fn buffered_record(kind: u64, data: &[u8]) -> Vec { + let mut body = Vec::new(); + write_vint(&mut body, kind); + body.extend_from_slice(data); + let mut out = Vec::new(); + write_vint(&mut out, body.len() as u64); + out.extend_from_slice(&body); + out + } + + #[test] + fn stack_framing_matches_buffered_encoding_at_vint_boundaries() { + for len in [0, 1, 127, 128, 16384] { + let data = vec![0xa5; len]; + for kind in [0, 127, 128, u64::MAX] { + let mut record = vec![0x5a]; + write_extra_record(&mut record, kind, &data).unwrap(); + let mut expected = vec![0x5a]; + expected.extend(buffered_record(kind, &data)); + assert_eq!(record, expected); + for flags in [0, HFL_EXTRA, u64::MAX] { + for size in [None, Some(u64::MAX)] { + let mut body = Vec::new(); + write_vint(&mut body, kind); + write_vint(&mut body, flags); + if flags & HFL_EXTRA != 0 { + write_vint(&mut body, data.len() as u64); + } + if let Some(size) = size { + write_vint(&mut body, size); + } + body.extend_from_slice(b"specific"); + body.extend_from_slice(&data); + let mut expected = vec![0; 4]; + write_vint(&mut expected, body.len() as u64); + expected.extend_from_slice(&body); + let crc = crc32(&expected[4..]); + expected[..4].copy_from_slice(&crc.to_le_bytes()); + assert_eq!( + block_header_image( + kind, + flags, + size, + b"specific", + &data, + &crate::WriterResources::default() + ) + .unwrap(), + expected + ); + } + } + } + } + } + + #[test] + fn fixed_records_fit_their_stack_capacity_and_keep_wire_bytes() { + let mut actual = Vec::new(); + write_file_encryption_record(&mut actual, [1; 16], [2; 16], [3; 12]).unwrap(); + let mut body = vec![0, 3, WRITE_KDF_COUNT_LOG]; + body.extend_from_slice(&[1; 16]); + body.extend_from_slice(&[2; 16]); + body.extend_from_slice(&[3; 12]); + assert_eq!(actual, buffered_record(FHEXTRA_CRYPT, &body)); + actual.clear(); + write_hash_record_with_value(&mut actual, [7; 32]).unwrap(); + let mut body = vec![0]; + body.extend_from_slice(&[7; 32]); + assert_eq!(actual, buffered_record(FHEXTRA_HASH, &body)); + actual.clear(); + write_mtime_record(&mut actual, Some(u32::MAX), Some(u32::MAX)).unwrap(); + let mut body = vec![0x13]; + body.extend_from_slice(&[0xff; 8]); + assert_eq!( + actual, + buffered_record(super::super::super::FHEXTRA_HTIME, &body) + ); + actual.clear(); + write_locator_record(&mut actual, Some(u64::MAX), Some(u64::MAX)).unwrap(); + let mut body = Vec::new(); + write_vint( + &mut body, + MHEXTRA_LOCATOR_QUICK_OPEN | MHEXTRA_LOCATOR_RECOVERY, + ); + write_vint(&mut body, u64::MAX); + write_vint(&mut body, u64::MAX); + assert_eq!(actual, buffered_record(MHEXTRA_LOCATOR, &body)); + } +} + +#[cfg(test)] +mod image_tests { + use super::*; + + #[test] + fn header_length_checks_reject_overflow_without_allocating() { + assert_eq!(checked_image_len(&[0, 7]).unwrap(), 7); + for parts in [[usize::MAX, 1], [isize::MAX as usize, 1]] { + assert_eq!( + checked_image_len(&parts).unwrap_err(), + Error::InvalidArgument("RAR 5 header size overflows") + ); + } + } + + #[test] + fn invalid_archive_metadata_is_rejected_at_serialization() { + let resources = crate::WriterResources::default(); + for (name, time, reason) in [ + ( + None, + None, + "RAR 5 archive metadata writer needs a name or creation time", + ), + ( + Some(b"name".as_slice()), + None, + "RAR 5 archive metadata name needs a creation time", + ), + ( + Some(b"".as_slice()), + Some(1), + "RAR 5 archive metadata name is empty", + ), + ] { + let error = archive_metadata_record( + ArchiveMetadataEntry { + name, + creation_time: time, + }, + &resources, + ) + .unwrap_err(); + assert_eq!(error, Error::InvalidArgument(reason)); + } + assert_eq!( + file_specific(b"", 0, 0, 0, None, 0, 0, false, &resources).unwrap_err(), + Error::InvalidArgument("RAR 5 file name is empty") + ); + } + + #[test] + fn retained_metadata_rejects_inconsistent_flags_and_names() { + let resources = crate::WriterResources::default(); + for (flags, name, time) in [ + (16, None, None), + (8, None, None), + (1, None, None), + (2, None, None), + (0, Some(b"name".to_vec()), None), + (0, None, Some(1)), + (4, None, None), + (1, Some(Vec::new()), None), + (1, Some(vec![0xff]), None), + ] { + let record = crate::rar50::ArchiveMetadataRecord { + flags, + name, + creation_time: time, + }; + assert_eq!( + retained_archive_metadata(&record, &resources).unwrap_err(), + Error::InvalidArgument("unsupported archive metadata record"), + "flags {flags}" + ); + } + let record = crate::rar50::ArchiveMetadataRecord { + flags: 6, + name: None, + creation_time: Some(u64::from(u32::MAX) + 1), + }; + assert_eq!( + retained_archive_metadata(&record, &resources).unwrap_err(), + Error::InvalidArgument("archive Unix timestamp exceeds 32 bits") + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_images_match_buffered_ciphertext_and_leave_trailing_data_plain() { + let keys = header_encryption_keys(b"secret").unwrap(); + for size in [0, 1, 15, 16, 17, 127, 128, 16384] { + let specific = vec![0x35; size]; + let data = b"unencrypted trailing payload"; + let actual = encrypted_header_block( + &keys.keys, + u64::MAX, + HFL_EXTRA, + Some(data.len() as u64), + &specific, + b"extra", + data, + &crate::WriterResources::default(), + ) + .unwrap(); + let iv = actual[..16].try_into().unwrap(); + let expected = block_header_image( + u64::MAX, + HFL_EXTRA, + Some(data.len() as u64), + &specific, + b"extra", + &crate::WriterResources::default(), + ) + .unwrap(); + let mut expected = expected.to_vec(); + expected.resize(expected.len().div_ceil(16) * 16, 0); + Rar50Cipher::new(keys.keys.key, iv) + .encrypt_in_place(&mut expected) + .unwrap(); + assert_eq!(&actual[16..16 + expected.len()], expected.as_slice()); + assert_eq!(&actual[16 + expected.len()..], data); + assert_eq!(actual.capacity(), actual.len()); + } + } + + #[test] + fn metadata_images_keep_all_supported_time_encodings_and_name_lengths() { + for length in [1, 127, 128, 16384] { + let name = vec![b'n'; length]; + for flags in [1, 2, 3, 6, 7, 14, 15] { + let time = if flags & 4 != 0 && flags & 8 == 0 { + u32::MAX as u64 + } else { + u64::MAX + }; + let record = crate::rar50::ArchiveMetadataRecord { + flags, + name: (flags & 1 != 0).then(|| name.clone()), + creation_time: (flags & 2 != 0).then_some(time), + }; + let mut body = Vec::new(); + write_vint(&mut body, flags); + if let Some(name) = &record.name { + write_vint(&mut body, name.len() as u64); + body.extend_from_slice(name); + } + if record.creation_time.is_some() { + if flags & 4 != 0 && flags & 8 == 0 { + body.extend_from_slice(&(time as u32).to_le_bytes()); + } else { + body.extend_from_slice(&time.to_le_bytes()); + } + } + let mut expected = Vec::new(); + write_extra_record(&mut expected, MHEXTRA_ARCHIVE_METADATA, &body).unwrap(); + let actual = + retained_archive_metadata(&record, &crate::WriterResources::default()).unwrap(); + assert_eq!(actual, expected); + assert_eq!(actual.capacity(), actual.len()); + if flags == 2 || flags == 3 { + assert_eq!( + archive_metadata_record( + ArchiveMetadataEntry { + name: record.name.as_deref(), + creation_time: record.creation_time, + }, + &crate::WriterResources::default() + ) + .unwrap(), + expected + ); + } + } + } + } + + #[test] + fn file_specific_fields_keep_maximum_widths_without_growing_the_record() { + let name = vec![b'n'; 16384]; + let actual = file_specific( + &name, + u64::MAX, + u32::MAX, + u64::MAX, + Some(u32::MAX), + u64::MAX, + u64::MAX, + true, + &crate::WriterResources::default(), + ) + .unwrap(); + let mut expected = Vec::new(); + for value in [FHFL_CRC32 | FHFL_DIRECTORY | FHFL_MTIME, u64::MAX, u64::MAX] { + write_vint(&mut expected, value); + } + expected.extend_from_slice(&[0xff; 8]); + for value in [u64::MAX, u64::MAX, name.len() as u64] { + write_vint(&mut expected, value); + } + expected.extend_from_slice(&name); + assert_eq!(actual, expected); + assert_eq!(actual.capacity(), actual.len()); + } +} diff --git a/arca-rar/vendor/rars/src/rar50/write/layout.rs b/arca-rar/vendor/rars/src/rar50/write/layout.rs new file mode 100644 index 00000000..cbe09f07 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/layout.rs @@ -0,0 +1,446 @@ +//! Where each archive block lands, worked out before anything is written. +//! +//! The main header carries a locator record holding the offsets of the +//! quick-open and recovery blocks, and those offsets are stored as variable +//! length integers. So the main header's own size depends on values that +//! depend on the main header's size. The writer used to break that circle by +//! emitting the entire archive up to four times until the offsets stopped +//! moving, which meant recomputing every recovery record from scratch on each +//! attempt. +//! +//! It is only a circle over two integers, though. Everything else in the +//! archive has a size that is already known: block headers are a pure function +//! of their fields, and the quick-open payload stores offsets *relative* to +//! itself, so growing the prefix shifts its position and its targets equally +//! and leaves its length alone. That makes the fixed point cheap to solve here, +//! over a few dozen bytes, instead of over the whole archive. + +use super::headers::{block_header_image, resolved_main_extra, stored_file_specific}; +use super::ArchiveMetadataEntry; +use crate::detect::RAR50_SIGNATURE; +use crate::rar50::{FHEXTRA_SUBDATA, HEAD_MAIN, HEAD_SERVICE, HFL_DATA, HFL_EXTRA}; +use crate::streaming::preparation::Bytes; +use crate::{Error, Result, WriterResources}; + +#[derive(Debug, Clone, Copy)] +pub(super) struct LayoutInputs<'a> { + /// Header encryption pads every header to a 16-byte boundary and prefixes + /// an IV, which changes sizes but keeps them predictable. + pub(super) header_encrypted: bool, + /// Size of the plaintext HEAD_CRYPT block, or zero when absent. + pub(super) head_crypt_len: u64, + /// Archive flags, including MHFL_RECOVERY and any volume flags. + pub(super) main_flags: u64, + pub(super) volume_number: Option, + pub(super) archive_metadata: Option>, + pub(super) metadata_record: Option<&'a crate::rar50::ArchiveMetadataRecord>, + /// Everything between the main header and the quick-open block: comments, + /// members and their services. + pub(super) body_len: u64, + pub(super) quick_open_payload_len: Option, + pub(super) recovery_percent: Option, +} + +#[derive(Debug)] +pub(super) struct ResolvedLayout { + /// The extra area to put in the main header, with settled offsets. + pub(super) main_extra: Bytes, + pub(super) main_header_len: u64, + /// Value stored in the locator: the block's position measured from the end + /// of the signature. The quick-open offset is settled the same way but is + /// only ever written into `main_extra`, so it is not repeated here. + pub(super) recovery_offset: Option, + /// Bytes the recovery record protects, i.e. everything before it. + pub(super) recovery_prefix_len: Option, +} + +pub(super) fn resolve_layout( + inputs: &LayoutInputs<'_>, + resources: &WriterResources, +) -> Result { + let signature_len = RAR50_SIGNATURE.len() as u64; + let quick_open_block_len = match inputs.quick_open_payload_len { + Some(payload_len) => { + stored_service_block_len(b"QO", payload_len, &[], inputs.header_encrypted, resources)? + } + None => 0, + }; + + let mut quick_open_offset = inputs.quick_open_payload_len.map(|_| 0); + let mut recovery_offset = inputs.recovery_percent.map(|_| 0); + + // Starting with zero offsets makes header sizes and offsets monotone. + // The two locator vints can grow by at most 18 bytes altogether; their + // record-size vint stays one byte. The extra-size and header-size vints + // can each grow by one byte, so the plaintext header grows by at most 20 + // bytes, or 32 after encryption padding. After the first offset update, + // each offset can therefore cross at most one further vint boundary + // (consecutive boundaries are at least 16,256 bytes apart). Those two + // possible width changes, plus the initial update and propagation, settle + // within five passes. Allocation or arithmetic failures return via `?`. + loop { + let mut main_extra = resolved_main_extra( + inputs.archive_metadata, + quick_open_offset, + recovery_offset, + resources, + )?; + if let Some(metadata) = inputs.metadata_record { + main_extra.extend_from_slice(&super::headers::retained_archive_metadata( + metadata, resources, + )?)?; + } + let main_header_len = main_header_len(inputs, &main_extra, resources)?; + + let quick_open_position = signature_len + .checked_add(inputs.head_crypt_len) + .and_then(|value| value.checked_add(main_header_len)) + .and_then(|value| value.checked_add(inputs.body_len)) + .ok_or(Error::InvalidArgument("RAR 5 archive layout overflows"))?; + let recovery_position = quick_open_position + .checked_add(quick_open_block_len) + .ok_or(Error::InvalidArgument("RAR 5 archive layout overflows"))?; + + let next_quick_open = quick_open_offset.map(|_| quick_open_position - signature_len); + let next_recovery = recovery_offset.map(|_| recovery_position - signature_len); + + if next_quick_open == quick_open_offset && next_recovery == recovery_offset { + return Ok(ResolvedLayout { + main_extra, + main_header_len, + recovery_offset, + recovery_prefix_len: inputs.recovery_percent.map(|_| recovery_position), + }); + } + + quick_open_offset = next_quick_open; + recovery_offset = next_recovery; + } +} + +/// Size of a main header carrying `extra`, as it will appear in the archive. +fn main_header_len( + inputs: &LayoutInputs<'_>, + extra: &[u8], + resources: &WriterResources, +) -> Result { + let mut specific = Bytes::new(resources); + specific.vint(inputs.main_flags)?; + if let Some(volume_number) = inputs.volume_number { + specific.vint(volume_number)?; + } + let header = block_header_image( + HEAD_MAIN, + if extra.is_empty() { 0 } else { HFL_EXTRA }, + None, + &specific, + extra, + resources, + )?; + Ok(emitted_header_len( + header.len() as u64, + inputs.header_encrypted, + )) +} + +/// Size of a stored service block, header and payload together. +pub(super) fn stored_service_block_len( + name: &[u8], + data_len: u64, + service_data: &[u8], + header_encrypted: bool, + resources: &WriterResources, +) -> Result { + let mut extra = Bytes::new(resources); + super::headers::write_extra_record(&mut extra, FHEXTRA_SUBDATA, service_data)?; + // The CRC is a fixed-width field, so any value gives the right size. + let specific = stored_file_specific(name, data_len, 0, 0, None, 0, resources)?; + let header = block_header_image( + HEAD_SERVICE, + HFL_EXTRA | HFL_DATA, + Some(data_len), + &specific, + &extra, + resources, + )?; + emitted_header_len(header.len() as u64, header_encrypted) + .checked_add(data_len) + .ok_or(Error::InvalidArgument("RAR 5 service block size overflows")) +} + +/// An encrypted header is a 16-byte IV followed by the plaintext padded up to +/// the AES block size. +fn emitted_header_len(plain_len: u64, header_encrypted: bool) -> u64 { + if header_encrypted { + 16 + plain_len.div_ceil(16) * 16 + } else { + plain_len + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn layout_header_sizes_release_preparation_charge_at_every_limit() { + for measure in [ + &(|resources: &WriterResources| main_header_len(&inputs(0), &[], resources)) + as &dyn Fn(&WriterResources) -> Result, + &(|resources: &WriterResources| { + stored_service_block_len(b"QO", 16, &[], false, resources) + }), + ] { + let mut succeeded = false; + let mut failed_after_admission = false; + for limit in 0..512 { + let resources = WriterResources::default().with_max_preparation_bytes(limit); + match measure(&resources) { + Ok(_) => { + succeeded = true; + break; + } + Err(Error::WriterPreparationLimitExceeded { used, .. }) => { + failed_after_admission |= used > 0; + drop(Bytes::zeroed(limit as usize, &resources).unwrap()); + } + Err(error) => panic!("unexpected limit {limit} failure: {error}"), + } + } + assert!(failed_after_admission); + assert!(succeeded); + } + } + + fn inputs(body_len: u64) -> LayoutInputs<'static> { + LayoutInputs { + header_encrypted: false, + head_crypt_len: 0, + main_flags: 0, + volume_number: None, + archive_metadata: None, + metadata_record: None, + body_len, + quick_open_payload_len: None, + recovery_percent: Some(5), + } + } + + /// The offsets a layout reports must be exactly where the blocks land when + /// the archive is assembled from the sizes it computed. + fn assert_self_consistent(inputs: &LayoutInputs<'_>, layout: &ResolvedLayout) { + let signature_len = RAR50_SIGNATURE.len() as u64; + let quick_open_position = + signature_len + inputs.head_crypt_len + layout.main_header_len + inputs.body_len; + + // Rebuilding the header with the settled offsets must not resize it. + let rebuilt = super::main_header_len( + inputs, + &layout.main_extra, + &crate::WriterResources::default(), + ) + .unwrap(); + assert_eq!(rebuilt, layout.main_header_len, "main header size moved"); + + let (records, complete) = crate::rar50::parse_main_extra_area( + &layout.main_extra, + 0..layout.main_extra.len(), + &crate::read_control::ReadControl::new(None), + ) + .unwrap(); + assert!(complete); + for record in records { + if let crate::rar50::MainExtraRecord::Locator(locator) = record { + assert_eq!( + locator.quick_open_offset, + inputs + .quick_open_payload_len + .map(|_| quick_open_position - signature_len) + .or_else(|| inputs.archive_metadata.map(|_| 0)) + ); + assert_eq!(locator.recovery_record_offset, layout.recovery_offset); + } + } + + if let Some(offset) = layout.recovery_offset { + let quick_open_block_len = match inputs.quick_open_payload_len { + Some(len) => stored_service_block_len( + b"QO", + len, + &[], + inputs.header_encrypted, + &crate::WriterResources::default(), + ) + .unwrap(), + None => 0, + }; + assert_eq!( + offset, + quick_open_position + quick_open_block_len - signature_len + ); + assert_eq!( + layout.recovery_prefix_len, + Some(offset + signature_len), + "recovery protects everything before its own block" + ); + } + } + + #[test] + fn layout_settles_across_vint_width_boundaries() { + // Body sizes chosen so the resulting offsets sit either side of each + // vint width step, which is where a naive single pass gets it wrong. + for boundary in [0x7fu64, 0x3fff, 0x1f_ffff, 0x0fff_ffff] { + for delta in [-3i64, -2, -1, 0, 1, 2, 3] { + let body_len = (boundary as i64 + delta).max(0) as u64; + let inputs = inputs(body_len); + let layout = resolve_layout(&inputs, &crate::WriterResources::default()).unwrap(); + assert_self_consistent(&inputs, &layout); + } + } + } + + #[test] + fn layout_settles_at_every_u64_offset_width_with_metadata_and_padding() { + let resources = WriterResources::default(); + for name_len in [0, 104, 120, 16_360] { + let metadata = crate::rar50::ArchiveMetadataRecord { + flags: 1, + name: Some(vec![b'x'; name_len]), + creation_time: None, + }; + for encrypted in [false, true] { + for features in 1..=3 { + let mut inputs = inputs(0); + inputs.header_encrypted = encrypted; + inputs.head_crypt_len = if encrypted { 60 } else { 0 }; + inputs.quick_open_payload_len = (features & 1 != 0).then_some(17); + inputs.recovery_percent = (features & 2 != 0).then_some(5); + inputs.metadata_record = (name_len != 0).then_some(&metadata); + let initial = resolve_layout(&inputs, &resources).unwrap(); + for bits in (7..=63).step_by(7) { + let boundary = 1u64 << bits; + let base = boundary + .saturating_sub(inputs.head_crypt_len + initial.main_header_len); + for delta in -32i64..=32 { + inputs.body_len = base.saturating_add_signed(delta); + let layout = resolve_layout(&inputs, &resources).unwrap(); + assert_self_consistent(&inputs, &layout); + } + } + } + } + } + } + + #[test] + fn layout_settles_with_quick_open_and_recovery_together() { + for body_len in [0u64, 100, 0x3ffe, 0x4001, 1 << 20] { + let mut inputs = inputs(body_len); + inputs.quick_open_payload_len = Some(4096); + let layout = resolve_layout(&inputs, &crate::WriterResources::default()).unwrap(); + + // assert_self_consistent checks that the recovery offset leaves + // room for the whole quick-open block ahead of it. + assert!(layout.recovery_offset.is_some()); + assert_self_consistent(&inputs, &layout); + } + } + + #[test] + fn layout_accounts_for_encrypted_header_padding() { + let mut inputs = inputs(1024); + inputs.header_encrypted = true; + inputs.head_crypt_len = 60; + let layout = resolve_layout(&inputs, &crate::WriterResources::default()).unwrap(); + + assert_eq!( + layout.main_header_len % 16, + 0, + "an IV plus padded ciphertext is a multiple of the block size" + ); + assert_self_consistent(&inputs, &layout); + } + + #[test] + fn layout_without_locator_features_has_no_offsets() { + let mut inputs = inputs(4096); + inputs.recovery_percent = None; + let layout = resolve_layout(&inputs, &crate::WriterResources::default()).unwrap(); + + assert_eq!(layout.recovery_offset, None); + assert_eq!(layout.recovery_prefix_len, None); + assert!(layout.main_extra.is_empty()); + } + + #[test] + fn layout_includes_retained_metadata_before_solving_offsets() { + let metadata = crate::rar50::ArchiveMetadataRecord { + flags: 2, + name: None, + creation_time: Some(123), + }; + let mut inputs = inputs(4096); + inputs.metadata_record = Some(&metadata); + inputs.quick_open_payload_len = Some(17); + let layout = resolve_layout(&inputs, &crate::WriterResources::default()).unwrap(); + assert!(!layout.main_extra.is_empty()); + assert_self_consistent(&inputs, &layout); + } + + #[test] + fn retained_layout_metadata_obeys_preparation_limit() { + let metadata = crate::rar50::ArchiveMetadataRecord { + flags: 2, + name: None, + creation_time: Some(123), + }; + let mut inputs = inputs(0); + inputs.quick_open_payload_len = None; + inputs.recovery_percent = None; + inputs.metadata_record = Some(&metadata); + let resources = crate::WriterResources::default().with_max_preparation_bytes(0); + assert!(matches!( + resolve_layout(&inputs, &resources), + Err(Error::WriterPreparationLimitExceeded { limit: 0, .. }) + )); + } + + #[test] + fn layout_rejects_offset_and_service_length_overflow() { + let resources = crate::WriterResources::default(); + for (body_len, head_crypt_len, quick_open_payload_len) in + [(u64::MAX, 0, None), (0, u64::MAX, None)] + { + let mut inputs = inputs(body_len); + inputs.head_crypt_len = head_crypt_len; + inputs.quick_open_payload_len = quick_open_payload_len; + assert_eq!( + resolve_layout(&inputs, &resources).unwrap_err(), + Error::InvalidArgument("RAR 5 archive layout overflows") + ); + } + assert_eq!( + stored_service_block_len(b"QO", u64::MAX, &[], false, &resources).unwrap_err(), + Error::InvalidArgument("RAR 5 service block size overflows") + ); + } + + #[test] + fn layout_and_service_sizing_release_refused_admission() { + let ledger = crate::codec::workspace::Allowance::limited(0); + let resources = WriterResources::default().with_execution_allowance(ledger.clone()); + assert_eq!( + resolve_layout(&inputs(0), &resources).err().unwrap().kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(ledger.used(), 0); + assert_eq!( + stored_service_block_len(b"QO", 1, &[], false, &resources) + .unwrap_err() + .kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(ledger.used(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/rar50/write/mod.rs b/arca-rar/vendor/rars/src/rar50/write/mod.rs new file mode 100644 index 00000000..23224563 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/mod.rs @@ -0,0 +1,2875 @@ +use super::*; +use crate::crc32::Crc32; +#[cfg(feature = "encryption")] +use crate::crypto::rar50::Rar50Cipher; +use crate::crypto::rar50::Rar50Keys; +pub use crate::filter::{FilterKind, FilterPolicy, FilterSpec}; +use crate::write_plan::{PlanShape, WriterOption}; +use crate::write_progress::ProgressReporter; +use crate::{EntrySource, WriteProgress, WriterResources}; +use std::io::{Read, Write}; + +mod compress; +mod engine; +mod filter_policy; +pub(crate) mod headers; +mod layout; +mod plan; +#[cfg(test)] +use filter_policy::encode_options_for_level; +use filter_policy::{ + compression_method_for_level, dictionary_size_for_options, encode_option_candidates_for_level, + filter_policy_walk_bytes, rar50_algorithm_version, +}; +pub(super) use headers::end_header_specific; + +const DEFAULT_RAR50_DICTIONARY_SIZE: u64 = 128 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct WriterOptions { + pub target: crate::ArchiveVersion, + pub features: crate::FeatureSet, + pub compression_level: Option, + pub dictionary_size: Option, +} + +impl WriterOptions { + pub const fn new(target: crate::ArchiveVersion, features: crate::FeatureSet) -> Self { + Self { + target, + features, + compression_level: None, + dictionary_size: None, + } + } + + pub const fn with_compression_level(mut self, level: u8) -> Self { + self.compression_level = Some(level); + self + } + + pub const fn with_dictionary_size(mut self, size: u64) -> Self { + self.dictionary_size = Some(size); + self + } +} + +impl Default for WriterOptions { + fn default() -> Self { + Self { + target: crate::ArchiveVersion::Rar50, + features: crate::FeatureSet::store_only(), + compression_level: None, + dictionary_size: None, + } + } +} + +/// An archive member, read from a reopenable source when it is needed. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ArchiveEntry { + pub name: Vec, + pub source: EntrySource, + /// Explicit directory entry; its source must be empty. + pub is_directory: bool, + /// Optional redirection metadata; the source must be empty. + pub redirection: Option, + /// Logical size from a retained redirection header. + pub redirection_size: Option, + pub mtime: Option, + /// Optional nanosecond fraction; requires mtime and a value below one second. + pub mtime_nanoseconds: Option, + pub file_times: Option, + pub attributes: u64, + pub host_os: u64, + /// Encrypts this member's payload. With header encryption every member + /// must use the same password. + pub password: Option>, + /// Service records attached to this member, such as a file comment. + pub services: Vec, +} + +/// A small named record attached to an archive or a member. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct ServiceEntry { + /// Service name, such as `CMT`, `ACL` or `STM`. + pub name: Vec, + pub data: Vec, + pub password: Option>, +} + +impl ServiceEntry { + pub fn new(name: impl Into>, data: impl Into>) -> Self { + Self { + name: name.into(), + data: data.into(), + password: None, + } + } + + pub fn with_password(mut self, password: impl Into>) -> Self { + self.password = Some(password.into()); + self + } +} + +impl ArchiveEntry { + pub fn new(name: impl Into>, source: EntrySource) -> Self { + Self { + name: name.into(), + source, + is_directory: false, + redirection: None, + redirection_size: None, + mtime: None, + mtime_nanoseconds: None, + file_times: None, + attributes: 0, + host_os: 0, + password: None, + services: Vec::new(), + } + } + + pub fn with_directory(mut self, is_directory: bool) -> Self { + self.is_directory = is_directory; + self + } + + pub fn with_redirection_size(mut self, size: Option) -> Self { + self.redirection_size = size; + self + } + + pub fn with_redirection(mut self, redirection: Option) -> Self { + self.redirection = redirection; + self + } + + pub fn with_service(mut self, service: ServiceEntry) -> Self { + self.services.push(service); + self + } + + pub fn with_file_times(mut self, times: Option) -> Self { + self.file_times = times; + self + } + + pub fn with_mtime(mut self, mtime: Option) -> Self { + self.mtime = mtime; + self + } + + pub fn with_mtime_nanoseconds(mut self, nanoseconds: Option) -> Self { + self.mtime_nanoseconds = nanoseconds; + self + } + + pub fn with_attributes(mut self, attributes: u64) -> Self { + self.attributes = attributes; + self + } + + pub fn with_host_os(mut self, host_os: u64) -> Self { + self.host_os = host_os; + self + } + + pub fn with_password(mut self, password: impl Into>) -> Self { + self.password = Some(password.into()); + self + } +} + +/// Receives each volume of a multi-volume archive as it is finished. +pub trait VolumeSink { + /// Opens the output for volume `index`, numbered from zero. + fn start_volume(&mut self, index: u64) -> Result>; + + /// Called once volume `index` has been written and flushed. + fn finish_volume(&mut self, index: u64, len: u64) -> Result<()> { + let _ = (index, len); + Ok(()) + } +} + +/// A [`VolumeSink`] that keeps the whole set in memory. +/// +/// The point of the sink is that a caller need not, so this is for the callers +/// that want the bytes back anyway: the Python bindings hand Python a list, and +/// tests inspect what was written. Four hand-rolled copies of it had grown up, +/// and only two of them checked that volumes arrived in order. +#[derive(Debug, Default)] +pub struct CollectedVolumes { + volumes: std::sync::Arc>>>, +} + +struct CollectedVolume { + volumes: std::sync::Arc>>>, + index: usize, +} + +impl CollectedVolumes { + pub fn new() -> Self { + Self::default() + } + + /// The finished set, in order. Takes the volumes rather than copying them, + /// so a set that only just fitted in memory does not need twice the room. + pub fn take(&mut self) -> Vec> { + std::mem::take( + &mut *self + .volumes + .lock() + .expect("volume collector is not poisoned"), + ) + } +} + +impl Write for CollectedVolume { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.volumes + .lock() + .expect("volume collector is not poisoned")[self.index] + .extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl VolumeSink for CollectedVolumes { + fn start_volume(&mut self, index: u64) -> Result> { + let mut volumes = self + .volumes + .lock() + .expect("volume collector is not poisoned"); + if volumes.len() as u64 != index { + return Err(Error::WriterFailure("RAR 5 volumes arrived out of order")); + } + volumes.push(Vec::new()); + drop(volumes); + Ok(Box::new(CollectedVolume { + volumes: std::sync::Arc::clone(&self.volumes), + index: index as usize, + })) + } + + fn finish_volume(&mut self, index: u64, len: u64) -> Result<()> { + let volumes = self + .volumes + .lock() + .expect("volume collector is not poisoned"); + if volumes[index as usize].len() as u64 != len { + return Err(Error::WriterFailure( + "RAR 5 volume length does not match what was written", + )); + } + Ok(()) + } +} + +/// Writes a multi-volume RAR 5 or RAR 7 archive, handing each volume to `sink` +/// as it completes so the set is never held in memory. +pub fn write_streaming_volumes_to( + entries: &[ArchiveEntry], + options: WriterOptions, + extras: ArchiveExtras<'_>, + max_payload_per_volume: u64, + sink: &mut dyn VolumeSink, + resources: &WriterResources, +) -> Result<()> { + write_streaming_volumes_with_progress( + entries, + options, + extras, + max_payload_per_volume, + sink, + resources, + None, + ) +} + +/// As [`write_streaming_volumes_to`], reporting compression progress as it +/// goes. +pub fn write_streaming_volumes_with_progress( + entries: &[ArchiveEntry], + options: WriterOptions, + extras: ArchiveExtras<'_>, + max_payload_per_volume: u64, + sink: &mut dyn VolumeSink, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, +) -> Result<()> { + let control = + crate::write_progress::ResourceProgress::new(resources, progress.map(ProgressReporter)); + let progress: Option<&dyn WriteProgress> = + (progress.is_some() || resources.has_cancellation()).then_some(&control); + crate::write_progress::check_cancelled(progress.map(ProgressReporter))?; + let encrypted = entries.iter().any(|entry| entry.password.is_some()); + if encrypted && !entries.iter().all(|entry| entry.password.is_some()) { + return Err(Error::UnsupportedFeature { + version: options.target, + feature: "RAR 5 writer mixing encrypted and plain members", + }); + } + let shape = PlanShape::new() + .compressed(true) + .volumes(true) + .filtered(extras.filter_policy != FilterPolicy::None); + // One option at a time, so a refusal names the one that was asked for. + // This used to be a single check over all three, which reported a string + // naming every option it covered whichever one the caller had set. Quick + // open rides on the feature set, so `validate_plan` catches that one. + validate_plan(options, shape)?; + if extras.locked { + return Err(Error::InvalidArgument( + "archive lock flag is not supported in volume output", + )); + } + if extras.comment.is_some() { + crate::write_plan::validate_option(options.target, WriterOption::ArchiveComment, shape)?; + } + if extras.metadata.is_some() || extras.metadata_record.is_some() { + crate::write_plan::validate_option(options.target, WriterOption::ArchiveMetadata, shape)?; + } + // The volume preparation path does not carry redirection metadata. + if entries.iter().any(|entry| entry.redirection.is_some()) { + return Err(Error::InvalidArgument( + "symbolic links are not supported in volume output", + )); + } + // A member's services are validated with it and then had nowhere to go: + // `prepare_volume_member` never reads them, so a file comment asked for on + // a split set was accepted and then dropped without a word. + if entries.iter().any(|entry| !entry.services.is_empty()) { + crate::write_plan::validate_option(options.target, WriterOption::FileComment, shape)?; + } + // A set with no members produces no volumes at all: the writer would + // return success having never asked the sink for a single one. + if entries.is_empty() { + return Err(Error::InvalidArgument( + "RAR 5 volume writer needs at least one member", + )); + } + if let Some(percent) = extras.recovery_percent { + validate_recovery_percent(percent)?; + } + if options.features.header_encryption && !encrypted && extras.header_password.is_none() { + return Err(Error::NeedPassword); + } + + engine::write_volumes( + entries, + engine::EnginePlan { + compress: compression_plan( + options, + extras.filter_policy, + dictionary_reach(entries, options.features.solid), + resources.memory_limit(), + resources, + )?, + recovery_percent: extras.recovery_percent, + header_encrypted: options.features.header_encryption, + header_password: extras.header_password, + archive_comment: None, + archive_metadata: None, + metadata_record: None, + locked: extras.locked, + // No volume writer emits the index, and `validate_plan` refuses a + // set that asks for one, so this is the only value that can get + // here rather than a decision taken quietly on the caller's behalf. + quick_open: false, + progress: progress.map(ProgressReporter), + }, + max_payload_per_volume, + sink, + resources, + ) +} + +/// Archive-level options that sit alongside the members. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct ArchiveExtras<'a> { + /// Archive comment, stored as a `CMT` service record. + pub comment: Option<&'a [u8]>, + /// Encrypts the comment. Without it the comment is stored in the clear. + pub comment_password: Option<&'a [u8]>, + /// Password for encrypted headers, including archives without members. + /// When omitted, uses the shared member password. When supplied, must + /// agree with any member passwords; ignored unless header encryption is set. + pub header_password: Option<&'a [u8]>, + pub metadata: Option>, + /// Complete retained metadata, mutually exclusive with `metadata`. + pub metadata_record: Option<&'a super::ArchiveMetadataRecord>, + /// Retains the advisory archive lock flag. + pub locked: bool, + /// Whether to look for a data filter that makes members compress better. + pub filter_policy: FilterPolicy, + /// Percentage of the archive to spend on a recovery record. + pub recovery_percent: Option, +} + +impl<'a> ArchiveExtras<'a> { + pub fn with_comment(mut self, comment: &'a [u8]) -> Self { + self.comment = Some(comment); + self + } + + pub fn with_encrypted_comment(mut self, comment: &'a [u8], password: &'a [u8]) -> Self { + self.comment = Some(comment); + self.comment_password = Some(password); + self + } + + pub fn with_metadata(mut self, metadata: ArchiveMetadataEntry<'a>) -> Self { + self.metadata = Some(metadata); + self + } + + pub fn with_filter_policy(mut self, policy: FilterPolicy) -> Self { + self.filter_policy = policy; + self + } + + pub fn with_recovery_percent(mut self, percent: Option) -> Self { + self.recovery_percent = percent; + self + } +} + +/// Writes a RAR 5 or RAR 7 archive to `output`, admitting estimated compression +/// workspace through `resources`. Packed payloads are retained in spools before +/// emission. This is not a total RAM or temporary-disk limit; bare-WASM spools +/// remain in memory. +/// +/// Supports solid compression, per-member and header encryption, and recovery +/// records, in any combination. +pub fn write_streaming_archive_to( + entries: &[ArchiveEntry], + options: WriterOptions, + extras: ArchiveExtras<'_>, + resources: &WriterResources, + output: &mut dyn Write, +) -> Result<()> { + write_streaming_archive_reporting(entries, options, extras, resources, None, output) +} + +/// As [`write_streaming_archive_to`], reporting compression progress as it +/// goes. +pub fn write_streaming_archive_with_progress( + entries: &[ArchiveEntry], + options: WriterOptions, + extras: ArchiveExtras<'_>, + resources: &WriterResources, + progress: Option<&dyn WriteProgress>, + output: &mut dyn Write, +) -> Result<()> { + write_streaming_archive_reporting( + entries, + options, + extras, + resources, + progress.map(ProgressReporter), + output, + ) +} + +pub(crate) fn write_streaming_archive_reporting( + entries: &[ArchiveEntry], + options: WriterOptions, + extras: ArchiveExtras<'_>, + resources: &WriterResources, + progress: Option>, + output: &mut dyn Write, +) -> Result<()> { + let control = crate::write_progress::ResourceProgress::new(resources, progress); + let progress = + (progress.is_some() || resources.has_cancellation()).then_some(ProgressReporter(&control)); + crate::write_progress::check_cancelled(progress)?; + let encrypted = entries.iter().any(|entry| entry.password.is_some()); + if extras.metadata.is_some() && extras.metadata_record.is_some() { + return Err(Error::InvalidArgument( + "duplicate archive metadata settings", + )); + } + let recovery_percent = extras.recovery_percent; + validate_plan( + options, + PlanShape::new() + .compressed(true) + .filtered(extras.filter_policy != FilterPolicy::None), + )?; + if let Some(percent) = recovery_percent { + validate_recovery_percent(percent)?; + } + if options.features.header_encryption && !encrypted && extras.header_password.is_none() { + return Err(Error::NeedPassword); + } + engine::write_archive( + entries, + engine::EnginePlan { + compress: compression_plan( + options, + extras.filter_policy, + dictionary_reach(entries, options.features.solid), + resources.memory_limit(), + resources, + )?, + recovery_percent, + header_encrypted: options.features.header_encryption, + header_password: extras.header_password, + archive_comment: match (extras.comment, extras.comment_password) { + (Some(data), Some(password)) => { + Some(engine::ArchiveCommentPlan::Encrypted { data, password }) + } + (Some(data), None) => Some(engine::ArchiveCommentPlan::Plain(data)), + (None, _) => None, + }, + archive_metadata: extras.metadata, + metadata_record: extras.metadata_record, + locked: extras.locked, + quick_open: options.features.quick_open, + progress, + }, + resources, + output, + ) +} + +/// How far one dictionary has to reach for these members. +/// +/// Solid members are coded as one chain through a single window, so a match can +/// point back into an earlier member and the whole archive is in reach. +/// Otherwise each member starts afresh and only the largest one matters. A +/// source that cannot say how long it is takes the largest window the writer +/// fits on its own, rather than the smallest. +fn dictionary_reach(entries: &[ArchiveEntry], solid: bool) -> u64 { + let sizes = entries + .iter() + .map(|entry| entry.source.len().unwrap_or(u64::MAX)); + if solid { + sizes.fold(0u64, u64::saturating_add) + } else { + sizes.max().unwrap_or(0) + } +} + +fn compression_plan( + options: WriterOptions, + filter_policy: FilterPolicy, + content: u64, + memory_limit: u64, + resources: &WriterResources, +) -> Result { + let method = compression_method_for_level(options.compression_level)?; + // A stored member has no window to reach across, and the size still lands in + // its header, so fitting one to the data would only inflate what the header + // claims. + let reach = if method == 0 { 0 } else { content }; + let dictionary_size = dictionary_size_for_options(options, reach, memory_limit)?; + let candidates = + encode_option_candidates_for_level(options.compression_level, dictionary_size)?; + let encode_options = *candidates.first().ok_or(Error::WriterFailure( + "RAR 5 compression level has no encoder options", + ))?; + Ok(compress::CompressPlan { + algorithm_version: rar50_algorithm_version(options, dictionary_size)?, + encode_options, + dictionary_size, + block_size: crate::codec::rar50::LZ_BLOCK_SIZE, + solid: options.features.solid, + method, + filter_policy, + candidates: crate::streaming::preparation::Records::collect( + candidates.into_iter().map(Ok), + resources, + )?, + }) +} + +fn source_integrity( + source: &EntrySource, + expected_size: u64, + block_size: usize, + progress: &dyn compress::CompressionProgress, + _resources: &WriterResources, +) -> Result<(u32, [u8; 32])> { + if let Some(allowance) = &_resources.execution { + return source_integrity_with_allowance( + source, + expected_size, + block_size, + progress, + allowance, + ); + } + source_integrity_with_allowance( + source, + expected_size, + block_size, + progress, + &crate::codec::workspace::Allowance::default(), + ) +} + +fn source_integrity_with_allowance( + source: &EntrySource, + expected_size: u64, + block_size: usize, + progress: &dyn compress::CompressionProgress, + allowance: &B, +) -> Result<(u32, [u8; 32])> { + if progress.is_cancelled() { + return Err(Error::Cancelled); + } + let mut crc = Crc32::new(); + let mut hash = blake2sp::Hasher::new(); + let mut buffer = crate::codec::workspace::Buffer::filled(block_size, 0u8, allowance)?; + let mut reader = source.open()?; + let mut observed = 0u64; + let mut limited = reader.by_ref().take(expected_size); + loop { + if progress.is_cancelled() { + return Err(Error::Cancelled); + } + let read = limited.read(&mut buffer)?; + if read == 0 { + break; + } + observed += read as u64; + crc.update(&buffer[..read]); + hash.update(&buffer[..read]); + } + crate::write_stream::check_source_length( + &mut *reader, + observed, + expected_size, + "entry source size changed while reading", + )?; + Ok((crc.finish(), hash.finalize())) +} + +/// What one compression job holds, for admitting jobs against the memory limit +/// and for shrinking a fitted dictionary until it fits. +/// +/// Nearly all of it is the match finder's chain links: four bytes per byte of +/// window, and the window is a power of two, so a dictionary just over one costs +/// almost double the one just under. That rounding is why a flat multiple of the +/// dictionary never described this well. It used to charge twelve times the +/// dictionary, which is the middle of the eight-to-sixteen the rounding then +/// produced, so it over-charged a dictionary that was already a power of two and +/// came close to under-charging one that had just crossed. +/// +/// Peak resident bytes for one member encode, over the member itself, taken on a +/// 16 MiB member so every window is filled: +/// +/// ```text +/// dictionary measured links alone +/// 256 KiB 6,197,248 1,048,576 +/// 1 MiB 9,170,944 4,194,304 +/// 3 MiB 20,152,320 16,777,216 +/// 4 MiB 20,234,240 16,777,216 +/// 8 MiB 36,941,824 33,554,432 +/// ``` +/// +/// The links account for the whole shape; what is left is the hash heads, the +/// token buffer for one block, and the packed output. Three megabytes covers +/// those with room to spare, and the fifth byte per window position absorbs the +/// rest rather than pretending to predict it. +/// +/// The optimal parse searches a binary tree instead of hash chains, and a tree +/// holds two links per window position where a chain holds one. So it is eight +/// bytes per byte of window rather than four, and the charge doubles the links +/// and keeps the one byte of slack. Measured on the same 16 MiB member, over +/// the member, against a 512-candidate parse: +/// +/// ```text +/// dictionary measured tree links alone +/// 256 KiB 19,668,992 2,097,152 +/// 1 MiB 21,794,816 8,388,608 +/// 4 MiB 47,030,272 33,554,432 +/// 8 MiB 75,390,976 67,108,864 +/// 16 MiB 109,412,352 134,217,728 +/// ``` +/// +/// The last row measures under its links because a 16 MiB member cannot fill a +/// 16 MiB window: the pages behind the untouched slots never become resident. +/// The charge covers the allocation anyway, since data that does fill the +/// window will touch them. +/// +/// `block_size` is the largest block the parse can be holding, not the size the +/// reader asks for: a block grows past that when the data it covers is not +/// moving, and the parse's arrays grow with it. See +/// [`crate::codec::rar50::MAX_LZ_BLOCK_SIZE`]. +fn streaming_lz_workspace(dictionary_size: u64, block_size: usize, optimal_parse: bool) -> u64 { + // Runs retain up to a dictionary of new input as well as history and + // combined search bytes. Charge those buffers alongside finder links. + let per_window_byte = if optimal_parse { 17 } else { 13 }; + dictionary_size + .checked_next_power_of_two() + .unwrap_or(dictionary_size) + .saturating_mul(per_window_byte) + // Repricing can retain an earlier winner beside the current and next + // token streams, in addition to the reused DP arrays and match cache. + .saturating_add((block_size as u64).saturating_mul(if optimal_parse { 160 } else { 64 })) + .saturating_add(3 * 1024 * 1024) +} + +#[allow(clippy::too_many_arguments)] +fn encrypt_reader_to( + reader: &mut dyn Read, + input_size: u64, + output: &mut dyn Write, + keys: &Rar50Keys, + iv: [u8; 16], + block_size: usize, + progress: Option>, + _resources: &WriterResources, +) -> Result<()> { + if let Some(allowance) = &_resources.execution { + return encrypt_reader_with_allowance( + reader, input_size, output, keys, iv, block_size, progress, allowance, + ); + } + encrypt_reader_with_allowance( + reader, + input_size, + output, + keys, + iv, + block_size, + progress, + &crate::codec::workspace::Allowance::default(), + ) +} + +#[allow(clippy::too_many_arguments)] +fn encrypt_reader_with_allowance( + reader: &mut dyn Read, + input_size: u64, + output: &mut dyn Write, + keys: &Rar50Keys, + iv: [u8; 16], + block_size: usize, + progress: Option>, + allowance: &B, +) -> Result<()> { + #[cfg(not(feature = "encryption"))] + { + let _ = ( + reader, input_size, output, keys, iv, block_size, progress, allowance, + ); + Err(Error::FeatureDisabled { + feature: "encryption", + }) + } + #[cfg(feature = "encryption")] + { + crate::write_progress::check_cancelled(progress)?; + if input_size == 0 { + return Ok(()); + } + let mut cipher = Rar50Cipher::new(keys.key, iv); + let chunk_size = block_size.max(16) & !15; + let mut buffer = crate::codec::workspace::Buffer::filled(chunk_size, 0u8, allowance)?; + let mut remaining = input_size; + while remaining >= chunk_size as u64 { + crate::write_progress::check_cancelled(progress)?; + reader.read_exact(&mut buffer)?; + cipher + .encrypt_in_place(&mut buffer) + .map_err(super::map_rar50_crypto_error)?; + output.write_all(&buffer)?; + remaining -= chunk_size as u64; + } + let final_plain = usize::try_from(remaining) + .map_err(|_| Error::InvalidArgument("RAR 5 encrypted data size overflows"))?; + let final_padded = final_plain.checked_add(15).ok_or(Error::InvalidArgument( + "RAR 5 encrypted data size overflows", + ))? & !15; + if final_padded != 0 { + crate::write_progress::check_cancelled(progress)?; + buffer[..final_padded].fill(0); + reader.read_exact(&mut buffer[..final_plain])?; + cipher + .encrypt_in_place(&mut buffer[..final_padded]) + .map_err(super::map_rar50_crypto_error)?; + output.write_all(&buffer[..final_padded])?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ArchiveMetadataEntry<'a> { + pub name: Option<&'a [u8]>, + pub creation_time: Option, +} + +/// Builds a RAR 5 or RAR 7 archive from members held in memory. +/// +/// [`write_streaming_archive_to`] is the same writer without the buffer; this +/// exists for callers that want the bytes back. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Rar50Writer<'a> { + options: WriterOptions, + entries: Vec, + archive_comment: Option<&'a [u8]>, + archive_comment_password: Option<&'a [u8]>, + archive_metadata: Option>, + filter_policy: FilterPolicy, + recovery_percent: Option, + progress: Option>, +} + +impl<'a> Rar50Writer<'a> { + pub fn new(options: WriterOptions) -> Self { + Self { + options, + entries: Vec::new(), + archive_comment: None, + archive_comment_password: None, + archive_metadata: None, + filter_policy: FilterPolicy::None, + recovery_percent: None, + progress: None, + } + } + + pub fn entry(mut self, entry: ArchiveEntry) -> Self { + self.entries.push(entry); + self + } + + pub fn entries(mut self, entries: impl IntoIterator) -> Self { + self.entries.extend(entries); + self + } + + pub fn archive_comment(mut self, comment: Option<&'a [u8]>) -> Self { + self.archive_comment = comment; + self.archive_comment_password = None; + self + } + + pub fn encrypted_archive_comment(mut self, comment: &'a [u8], password: &'a [u8]) -> Self { + self.archive_comment = Some(comment); + self.archive_comment_password = Some(password); + self + } + + pub fn archive_metadata(mut self, metadata: Option>) -> Self { + self.archive_metadata = metadata; + self + } + + pub fn filter_policy(mut self, policy: FilterPolicy) -> Self { + self.filter_policy = policy; + self + } + + pub fn recovery_percent(mut self, percent: Option) -> Self { + self.recovery_percent = percent; + self + } + + pub fn progress(mut self, progress: &'a dyn WriteProgress) -> Self { + self.progress = Some(ProgressReporter(progress)); + self + } + + /// Builds the archive in memory. Prefer [`Rar50Writer::write_to`], which + /// streams it instead. + pub fn finish(self) -> Result> { + self.finish_with_resources(&WriterResources::default()) + } + + pub fn finish_with_resources(self, resources: &WriterResources) -> Result> { + let mut out = crate::WriterOutput::new(resources); + self.write_to(&mut out, resources)?; + Ok(out.into_vec()) + } + + /// Writes the archive straight to `output` without ever holding it. + pub fn write_to(self, output: &mut dyn Write, resources: &WriterResources) -> Result<()> { + let mut extras = ArchiveExtras::default() + .with_recovery_percent(self.recovery_percent) + .with_filter_policy(self.filter_policy); + if let Some(comment) = self.archive_comment { + extras = match self.archive_comment_password { + Some(password) => extras.with_encrypted_comment(comment, password), + None => extras.with_comment(comment), + }; + } + if let Some(metadata) = self.archive_metadata { + extras = extras.with_metadata(metadata); + } + + write_streaming_archive_reporting( + &self.entries, + self.options, + extras, + resources, + self.progress, + output, + ) + } +} + +/// Everything this writer refuses, in one place, before anything is written. +/// +/// This replaced a six-by-two matrix of near-identical checks, one per shape of +/// member the builder accepts. Each built a set of allowed features and compared +/// it with the request, and several built it by assigning a flag to itself, +/// which accepted the flag whatever it said. +fn validate_plan(options: WriterOptions, shape: PlanShape) -> Result<()> { + if options.target.family() != crate::version::ArchiveFamily::Rar50Plus { + return Err(Error::UnsupportedVersion(options.target)); + } + crate::write_plan::validate_features(options.target, options.features, shape)?; + crate::write_plan::validate_compression_level(options.target, options.compression_level)?; + let _ = dictionary_size_for_options(options, 0, u64::MAX)?; + // Quick-open is an index of the headers, so it has nothing to index once + // they are encrypted. + if options.features.quick_open && options.features.header_encryption { + return Err(Error::UnsupportedWriterOption { + target: options.target, + option: WriterOption::Feature(crate::Feature::QuickOpen), + because: Some("with header encryption"), + }); + } + // Solid members share one dictionary and are coded as one chain, so the + // filter search never runs for them. Relations between two requests cannot + // be expressed in the capability table, which is why this lives here rather + // than in `supports`. + if shape.filtered && options.features.solid { + return Err(Error::UnsupportedWriterOption { + target: options.target, + option: WriterOption::Filter, + because: Some("in a solid archive"), + }); + } + Ok(()) +} + +fn validate_recovery_percent(percent: u64) -> Result<()> { + if !cfg!(feature = "recovery") { + return Err(Error::FeatureDisabled { + feature: "recovery", + }); + } + if !(1..=100).contains(&percent) { + return Err(Error::InvalidArgument( + "RAR 5 recovery percent must be in 1..=100", + )); + } + Ok(()) +} + +fn validate_nonempty_password(password: &[u8]) -> Result<()> { + crate::crypto::require_encryption()?; + + if password.is_empty() { + return Err(Error::InvalidArgument( + "RAR 5 encrypted writer needs a non-empty password", + )); + } + Ok(()) +} + +fn validate_file_entry(name: &[u8]) -> Result<()> { + if name.is_empty() { + return Err(Error::InvalidArgument("RAR 5 file name is empty")); + } + Ok(()) +} + +/// Checks one member and everything hanging off it. +/// +/// Both entry points run this now. It used to be six near-identical validators, +/// one per shape of member the builder took, and the streaming path ran none of +/// them beyond the name. +fn validate_entry(entry: &ArchiveEntry) -> Result<()> { + validate_file_entry(&entry.name)?; + if let Some(link) = &entry.redirection { + if !link.supports_header(entry.host_os, entry.attributes, entry.is_directory) + || (link.redirection_type >= 4 && entry.redirection_size.is_none()) + || entry.source.len()? != 0 + { + return Err(Error::InvalidArgument( + "redirections require supported target metadata and empty contents", + )); + } + } + if entry.redirection.is_none() && entry.redirection_size.is_some() { + return Err(Error::InvalidArgument( + "redirection size requires a redirection record", + )); + } + if entry.is_directory && entry.source.len()? != 0 { + return Err(Error::InvalidArgument( + "directory entries cannot carry file contents", + )); + } + if let Some(times) = entry.file_times { + times.encode()?; + if entry.mtime_nanoseconds.is_some() || (entry.mtime.is_some() && times.modified.is_some()) + { + return Err(Error::InvalidArgument( + "conflicting file modification timestamps", + )); + } + } + if let Some(nanos) = entry.mtime_nanoseconds { + if entry.mtime.is_none() || nanos >= 1_000_000_000 { + return Err(Error::InvalidArgument( + "fractional modification time requires seconds and nanoseconds below one second", + )); + } + } + if let Some(password) = &entry.password { + validate_nonempty_password(password)?; + } + entry.services.iter().try_for_each(validate_service) +} + +fn validate_service(service: &ServiceEntry) -> Result<()> { + // Encrypting a service record is only wired up for comments, so an + // encrypted ACL or STM would be written in the clear. + let allowed: &[&[u8]] = match service.password { + Some(_) => &[b"CMT"], + None => &[b"ACL", b"STM", b"CMT"], + }; + if !allowed.contains(&service.name.as_slice()) { + return Err(Error::UnsupportedFeature { + version: crate::ArchiveVersion::Rar50, + feature: "RAR 5 file service name", + }); + } + // An explicit empty comment is distinct from having no comment record. + if service.data.is_empty() && service.name != b"CMT" { + return Err(Error::InvalidArgument("RAR 5 file service data is empty")); + } + match &service.password { + Some(password) => validate_nonempty_password(password), + None => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::VolumeSink; + + #[test] + fn source_integrity_cancellation_and_short_read_fail_before_hashing() { + struct Cancelled; + impl super::compress::CompressionProgress for Cancelled { + fn advance(&self, _: u64) -> bool { + false + } + fn is_cancelled(&self) -> bool { + true + } + } + let unopened = + crate::EntrySource::from_opener(1, || panic!("cancelled scan opened the source")); + let allowance = crate::codec::workspace::Allowance::default(); + assert!(matches!( + super::source_integrity_with_allowance(&unopened, 1, 16, &Cancelled, &allowance), + Err(crate::Error::Cancelled) + )); + + let short = crate::EntrySource::from_opener(2, || { + Ok(Box::new(std::io::Cursor::new(b"x".to_vec()))) + }); + let progress = |_: u64| true; + assert_eq!( + super::source_integrity_with_allowance(&short, 2, 16, &progress, &allowance) + .unwrap_err() + .kind(), + crate::ErrorKind::SourceChanged + ); + } + + #[test] + fn volume_compression_plan_refusal_does_not_open_sources_or_start_a_volume() { + let entries = [super::ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_opener(1, || panic!("plan refusal opened source")), + )]; + let resources = crate::WriterResources::default().with_max_preparation_bytes(0); + let mut volumes = super::CollectedVolumes::new(); + let error = super::write_streaming_volumes_to( + &entries, + super::WriterOptions::new( + crate::ArchiveVersion::Rar50, + crate::FeatureSet::store_only(), + ), + super::ArchiveExtras::default(), + 64, + &mut volumes, + &resources, + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert!(volumes.take().is_empty()); + } + + #[test] + fn collected_volumes_enforce_order_and_report_length_mismatch() { + let mut volumes = super::CollectedVolumes::new(); + assert!(matches!( + volumes.start_volume(1), + Err(crate::Error::WriterFailure( + "RAR 5 volumes arrived out of order" + )) + )); + let mut output = volumes.start_volume(0).unwrap(); + output.write_all(b"abc").unwrap(); + output.flush().unwrap(); + assert!(matches!( + volumes.finish_volume(0, 2), + Err(crate::Error::WriterFailure( + "RAR 5 volume length does not match what was written" + )) + )); + volumes.finish_volume(0, 3).unwrap(); + drop(output); + assert_eq!(volumes.take(), [b"abc".to_vec()]); + } + + #[test] + fn writer_plan_rejects_wrong_family_and_solid_filters_before_opening_source() { + let entry = || { + super::ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_opener(1, || panic!("invalid plan opened source")), + ) + }; + let wrong_family = super::Rar50Writer::new(super::WriterOptions::new( + crate::ArchiveVersion::Rar29, + crate::FeatureSet::store_only(), + )) + .entry(entry()) + .finish() + .unwrap_err(); + assert!(matches!( + wrong_family, + crate::Error::UnsupportedVersion(crate::ArchiveVersion::Rar29) + )); + + let mut features = crate::FeatureSet::store_only(); + features.solid = true; + let solid_filter = super::Rar50Writer::new(super::WriterOptions::new( + crate::ArchiveVersion::Rar50, + features, + )) + .entry(entry()) + .filter_policy(super::FilterPolicy::explicit(super::FilterKind::E8)) + .finish() + .unwrap_err(); + assert!(matches!( + solid_filter, + crate::Error::UnsupportedWriterOption { + option: crate::WriterOption::Filter, + .. + } + )); + } + + fn simple_entry(data: &[u8]) -> super::ArchiveEntry { + super::ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_bytes(data.to_vec()), + ) + } + + #[test] + fn malformed_member_metadata_is_rejected_before_serialization() { + let mut file = simple_entry(b"payload"); + file.redirection_size = Some(0); + assert_eq!( + super::validate_entry(&file).unwrap_err(), + crate::Error::InvalidArgument("redirection size requires a redirection record") + ); + + let link = super::super::FileRedirection { + redirection_type: 4, + flags: 0, + target_name: b"target".to_vec(), + }; + file.redirection_size = None; + file.redirection = Some(link.clone()); + assert_eq!( + super::validate_entry(&file).unwrap_err(), + crate::Error::InvalidArgument( + "redirections require supported target metadata and empty contents" + ) + ); + file.redirection_size = Some(0); + assert_eq!( + super::validate_entry(&file).unwrap_err(), + crate::Error::InvalidArgument( + "redirections require supported target metadata and empty contents" + ) + ); + file.source = crate::EntrySource::from_bytes(Vec::new()); + file.host_os = 99; + assert_eq!( + super::validate_entry(&file).unwrap_err(), + crate::Error::InvalidArgument( + "redirections require supported target metadata and empty contents" + ) + ); + file.host_os = 0; + super::validate_entry(&file).unwrap(); + + let mut directory = simple_entry(b"payload").with_directory(true); + assert_eq!( + super::validate_entry(&directory).unwrap_err(), + crate::Error::InvalidArgument("directory entries cannot carry file contents") + ); + directory.source = crate::EntrySource::from_bytes(Vec::new()); + super::validate_entry(&directory).unwrap(); + + let mut timestamped = simple_entry(b""); + timestamped.mtime_nanoseconds = Some(1); + assert!(super::validate_entry(×tamped).is_err()); + timestamped.mtime = Some(0); + timestamped.mtime_nanoseconds = Some(1_000_000_000); + assert!(super::validate_entry(×tamped).is_err()); + timestamped.mtime_nanoseconds = None; + timestamped.file_times = + Some(crate::FileTimes::from_unix_nanoseconds(Some(0), None, None).unwrap()); + assert_eq!( + super::validate_entry(×tamped).unwrap_err(), + crate::Error::InvalidArgument("conflicting file modification timestamps") + ); + timestamped.mtime = None; + timestamped.mtime_nanoseconds = Some(1); + assert_eq!( + super::validate_entry(×tamped).unwrap_err(), + crate::Error::InvalidArgument("conflicting file modification timestamps") + ); + } + + #[test] + fn unsupported_or_empty_member_services_are_rejected() { + for service in [ + super::ServiceEntry::new(b"BAD", b"data"), + super::ServiceEntry::new(b"ACL", b""), + super::ServiceEntry::new(b"CMT", b"data").with_password(b""), + ] { + assert!(super::validate_entry(&simple_entry(b"").with_service(service)).is_err()); + } + super::validate_entry( + &simple_entry(b"").with_service(super::ServiceEntry::new(b"CMT", b"")), + ) + .unwrap(); + } + + #[test] + fn volume_preflight_rejects_unsupported_member_and_archive_shapes() { + fn reject( + entries: &[super::ArchiveEntry], + options: super::WriterOptions, + extras: super::ArchiveExtras<'_>, + ) -> crate::Error { + let mut sink = super::CollectedVolumes::new(); + let error = super::write_streaming_volumes_to( + entries, + options, + extras, + 4096, + &mut sink, + &crate::WriterResources::default(), + ) + .unwrap_err(); + assert!(sink.take().is_empty()); + error + } + + let plain = simple_entry(b"payload"); + let encrypted = simple_entry(b"payload").with_password(b"secret"); + assert_eq!( + reject( + &[encrypted, plain.clone()], + super::WriterOptions::default(), + super::ArchiveExtras::default(), + ) + .kind(), + crate::ErrorKind::UnsupportedFeature + ); + + let locked = super::ArchiveExtras { + locked: true, + ..Default::default() + }; + assert_eq!( + reject( + std::slice::from_ref(&plain), + super::WriterOptions::default(), + locked + ), + crate::Error::InvalidArgument("archive lock flag is not supported in volume output") + ); + + let metadata = super::ArchiveExtras::default().with_metadata(super::ArchiveMetadataEntry { + name: None, + creation_time: Some(1), + }); + assert_eq!( + reject( + std::slice::from_ref(&plain), + super::WriterOptions::default(), + metadata + ) + .kind(), + crate::ErrorKind::UnsupportedFeature + ); + + let link = simple_entry(b"").with_redirection(Some(super::super::FileRedirection { + redirection_type: 4, + flags: 0, + target_name: b"target".to_vec(), + })); + assert_eq!( + reject( + &[link], + super::WriterOptions::default(), + super::ArchiveExtras::default() + ), + crate::Error::InvalidArgument("symbolic links are not supported in volume output") + ); + + let commented = + simple_entry(b"").with_service(super::ServiceEntry::new(b"CMT", b"file comment")); + assert_eq!( + reject( + &[commented], + super::WriterOptions::default(), + super::ArchiveExtras::default(), + ) + .kind(), + crate::ErrorKind::UnsupportedFeature + ); + + let mut features = crate::FeatureSet::store_only(); + features.header_encryption = true; + assert_eq!( + reject( + &[plain], + super::WriterOptions::new(crate::ArchiveVersion::Rar50, features), + super::ArchiveExtras::default(), + ), + if cfg!(feature = "encryption") { + crate::Error::NeedPassword + } else { + crate::Error::FeatureDisabled { + feature: "encryption", + } + } + ); + } + + #[test] + fn archive_preflight_rejects_duplicate_metadata_before_emission() { + let record = super::super::ArchiveMetadataRecord { + flags: 2, + name: None, + creation_time: Some(1), + }; + let mut extras = + super::ArchiveExtras::default().with_metadata(super::ArchiveMetadataEntry { + name: None, + creation_time: Some(1), + }); + extras.metadata_record = Some(&record); + let mut output = Vec::new(); + let error = super::write_streaming_archive_to( + &[simple_entry(b"payload")], + super::WriterOptions::default(), + extras, + &crate::WriterResources::default(), + &mut output, + ) + .unwrap_err(); + assert_eq!( + error, + crate::Error::InvalidArgument("duplicate archive metadata settings") + ); + assert!(output.is_empty()); + } + + #[test] + fn volume_preflight_rejects_retained_metadata_before_opening_source_or_sink() { + let entry = super::ArchiveEntry::new( + b"file".to_vec(), + crate::EntrySource::from_opener(1, || panic!("metadata refusal opened source")), + ); + let record = super::super::ArchiveMetadataRecord { + flags: 2, + name: None, + creation_time: Some(1), + }; + let extras = super::ArchiveExtras { + metadata_record: Some(&record), + ..Default::default() + }; + let mut sink = super::CollectedVolumes::new(); + let error = super::write_streaming_volumes_to( + &[entry], + super::WriterOptions::default(), + extras, + 64, + &mut sink, + &crate::WriterResources::default(), + ) + .unwrap_err(); + assert!(matches!( + error, + crate::Error::UnsupportedWriterOption { + option: crate::WriterOption::ArchiveMetadata, + .. + } + )); + assert!(sink.take().is_empty()); + } + + #[test] + #[cfg(feature = "encryption")] + fn volume_header_password_can_encrypt_headers_without_encrypting_member_data() { + let mut features = crate::FeatureSet::store_only(); + features.header_encryption = true; + let extras = super::ArchiveExtras { + header_password: Some(b"password"), + ..Default::default() + }; + let mut sink = super::CollectedVolumes::new(); + super::write_streaming_volumes_to( + &[simple_entry(b"payload")], + super::WriterOptions::new(crate::ArchiveVersion::Rar50, features), + extras, + 64, + &mut sink, + &crate::WriterResources::default(), + ) + .unwrap(); + let parts = sink.take(); + assert!(!parts.is_empty()); + for part in parts { + assert!(matches!( + super::super::Archive::parse(&part), + Err(crate::Error::NeedPassword) + )); + let archive = + super::super::Archive::parse_with_password(&part, Some(b"password")).unwrap(); + assert!(archive.main.encrypted_headers); + assert!(archive.files().all(|file| !file.encrypted)); + } + } + + #[test] + #[cfg(feature = "encryption")] + fn header_encryption_rejects_conflicting_member_password_before_output() { + let mut features = crate::FeatureSet::store_only(); + features.header_encryption = true; + let options = super::WriterOptions::new(crate::ArchiveVersion::Rar50, features); + let extras = super::ArchiveExtras { + header_password: Some(b"header password"), + ..Default::default() + }; + let mut entry = simple_entry(b"payload"); + entry.password = Some(b"member password".to_vec()); + + let mut archive = Vec::new(); + let error = super::write_streaming_archive_to( + &[entry.clone()], + options, + extras.clone(), + &crate::WriterResources::default(), + &mut archive, + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument); + assert!(archive.is_empty()); + + let mut volumes = super::CollectedVolumes::new(); + let error = super::write_streaming_volumes_to( + &[entry], + options, + extras, + 64, + &mut volumes, + &crate::WriterResources::default(), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument); + assert!(volumes.take().is_empty()); + } + + #[test] + #[cfg(feature = "encryption")] + fn quick_open_with_encrypted_headers_is_rejected_by_plan_validation() { + let mut features = crate::FeatureSet::store_only(); + features.quick_open = true; + features.header_encryption = true; + let options = super::WriterOptions::new(crate::ArchiveVersion::Rar50, features); + let error = super::validate_plan( + options, + crate::write_plan::PlanShape::new().compressed(true), + ) + .unwrap_err(); + assert!(matches!( + error, + crate::Error::UnsupportedWriterOption { + option: crate::write_plan::WriterOption::Feature(crate::Feature::QuickOpen), + .. + } + )); + } + + #[test] + fn empty_volume_set_is_rejected_before_opening_a_sink() { + let mut sink = super::CollectedVolumes::new(); + let error = super::write_streaming_volumes_to( + &[], + super::WriterOptions::default(), + super::ArchiveExtras::default(), + 4096, + &mut sink, + &crate::WriterResources::default(), + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument); + assert!(sink.take().is_empty()); + } + + #[test] + fn invalid_writer_inputs_and_internal_failures_have_distinct_categories() { + for error in [ + super::validate_file_entry(b"").unwrap_err(), + #[cfg(feature = "encryption")] + super::validate_nonempty_password(b"").unwrap_err(), + #[cfg(feature = "recovery")] + super::validate_recovery_percent(101).unwrap_err(), + super::filter_policy::dictionary_size_fields(0, 0).unwrap_err(), + super::filter_policy::dictionary_size_fields(1, 131073).unwrap_err(), + super::headers::header_encryption_password([&b"one"[..], &b"two"[..]].into_iter()) + .unwrap_err(), + ] { + assert_eq!(error.kind(), crate::ErrorKind::InvalidArgument, "{error}"); + } + #[cfg(not(feature = "encryption"))] + assert_eq!( + super::validate_nonempty_password(b"").unwrap_err(), + Error::FeatureDisabled { + feature: "encryption" + } + ); + #[cfg(not(feature = "recovery"))] + assert_eq!( + super::validate_recovery_percent(101).unwrap_err(), + Error::FeatureDisabled { + feature: "recovery" + } + ); + let mut volumes = super::CollectedVolumes::default(); + let error = super::VolumeSink::start_volume(&mut volumes, 1) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::WriterFailure); + assert_eq!(error.kind().code(), "WRITE_FAILED"); + } + + #[test] + #[cfg(feature = "encryption")] + fn encryption_allowance_admits_before_reading_and_preserves_cbc_tails() { + use crate::codec::workspace::Allowance; + let keys = Rar50Keys::derive(b"test password", [7; 16], 0).unwrap(); + for len in [0usize, 1, 15, 16, 17, 31, 32, 95, 128, 129] { + let input: Vec = (0..len).map(|n| n as u8).collect(); + let mut expected = input.clone(); + expected.resize(len.next_multiple_of(16), 0); + Rar50Cipher::new(keys.key, [9; 16]) + .encrypt_in_place(&mut expected) + .unwrap(); + for limit in [0, 31, 32] { + let allowance = Allowance::limited(limit); + let mut reader = std::io::Cursor::new(&input); + let mut output = Vec::new(); + let result = encrypt_reader_with_allowance( + &mut reader, + len as u64, + &mut output, + &keys, + [9; 16], + 32, + None, + &allowance, + ); + if len == 0 || limit == 32 { + result.unwrap(); + assert_eq!(output, expected); + assert_eq!(reader.position(), len as u64); + } else { + assert_eq!(result.unwrap_err().kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(reader.position(), 0); + assert!(output.is_empty()); + } + assert_eq!(allowance.used(), 0); + } + } + } + + #[test] + #[cfg(feature = "encryption")] + fn encryption_allowance_releases_buffers_on_io_failure_and_cancellation() { + use crate::codec::workspace::Allowance; + use std::sync::atomic::{AtomicBool, Ordering}; + struct Progress(AtomicBool); + impl crate::WriteProgress for Progress { + fn report(&self, _: crate::WriteProgressEvent<'_>) {} + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + struct Sink<'a>(&'a Progress, bool, Vec); + impl Write for Sink<'_> { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.1 { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + self.2.extend_from_slice(data); + self.0 .0.store(true, Ordering::Relaxed); + Ok(data.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let keys = Rar50Keys::derive(b"test password", [7; 16], 0).unwrap(); + let allowance = Allowance::limited(32); + for fail_sink in [false, true] { + let progress = Progress(AtomicBool::new(false)); + let mut sink = Sink(&progress, fail_sink, Vec::new()); + let mut reader = std::io::Cursor::new([42u8; 96]); + let error = encrypt_reader_with_allowance( + &mut reader, + 96, + &mut sink, + &keys, + [9; 16], + 32, + Some(ProgressReporter(&progress)), + &allowance, + ) + .unwrap_err(); + if fail_sink { + assert!(matches!(error, Error::Io(_))); + } else { + assert!(matches!(error, Error::Cancelled)); + assert_eq!(sink.2.len(), 32); + } + assert_eq!(reader.position(), 32); + assert_eq!(allowance.used(), 0); + } + let result = encrypt_reader_with_allowance( + &mut std::io::Cursor::new([42u8; 7]), + 96, + &mut Vec::new(), + &keys, + [9; 16], + 32, + None, + &allowance, + ); + assert!(matches!(result, Err(Error::Io(_)))); + assert_eq!(allowance.used(), 0); + } + + use super::filter_policy::{ + encode_member_with_auto_size_filter_progress, encode_member_with_filter_policy_and_progress, + }; + use super::headers::{ + file_specific, write_block, write_end_header, write_hash_record_with_value, + write_main_header, + }; + use super::*; + use crate::codec::rar50::{encode_literal_only, encode_lz_member}; + use crate::codec::rar50::{encode_lz_member_with_options, EncodeOptions, Unpack50Encoder}; + use crate::filter_search::{ + auto_delta_filter_range, disjoint_filter_ranges, AUTO_DELTA_EDGE_SKIP, + }; + use crate::x86_filter_scan::auto_x86_filter_ranges; + use crate::{ArchiveVersion, FeatureSet}; + use crate::{WriteOperation, WriteProgressEvent}; + use std::cell::RefCell; + use std::fs; + use std::io::{Cursor, Result as IoResult, Write}; + use std::process::Command; + use std::rc::Rc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CollectWriter(Rc>>); + + #[test] + fn stored_writer_does_not_require_lz_workspace() { + let data = b"stored payload".repeat(1024); + let write = |limit| -> Result> { + let mut out = Vec::new(); + Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ) + .entries(vec![entry(b"payload", &data)]) + .write_to(&mut out, &WriterResources::new(limit))?; + Ok(out) + }; + let bytes = write(1024 * 1024).unwrap(); + assert_eq!(bytes, write(256 * 1024 * 1024).unwrap()); + let archive = crate::ArchiveReader::read_owned(bytes).unwrap(); + assert_eq!( + archive.read_member(b"payload", None).unwrap().unwrap(), + data + ); + } + + #[test] + fn the_writer_fits_the_dictionary_to_what_one_window_has_to_reach() { + let options = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(3); + let fitted = |content: u64| { + dictionary_size_for_options(options, content, u64::MAX).expect("a legal dictionary") + }; + + // The smallest window that still reaches past the data, and never below + // the format's floor or above the cap the writer picks on its own. + assert_eq!(fitted(0), 128 * 1024); + assert_eq!(fitted(130_000), 128 * 1024); + assert_eq!(fitted(200_000), 256 * 1024); + assert_eq!(fitted(700_000), 1024 * 1024); + assert_eq!(fitted(64 * 1024 * 1024), 4 * 1024 * 1024); + + // What the caller asked for stands, in both directions. + assert_eq!( + dictionary_size_for_options(options.with_dictionary_size(4 * 1024 * 1024), 0, u64::MAX) + .unwrap(), + 4 * 1024 * 1024 + ); + } + + #[test] + fn the_cap_on_a_fitted_dictionary_climbs_with_the_level() { + let plenty = 64 * 1024 * 1024; + let capped = |level: u8| { + let options = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(level); + dictionary_size_for_options(options, plenty, u64::MAX).expect("a legal dictionary") + }; + + assert_eq!(capped(1), 1024 * 1024); + assert_eq!(capped(2), 4 * 1024 * 1024); + assert_eq!(capped(3), 4 * 1024 * 1024); + assert_eq!(capped(4), 8 * 1024 * 1024); + assert_eq!(capped(5), 16 * 1024 * 1024); + + // An absent level is the default level, and picks the same window it + // would if the default had been spelled out. + let absent = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()); + assert_eq!( + dictionary_size_for_options(absent, plenty, u64::MAX).unwrap(), + capped(3), + ); + } + + #[test] + fn the_workspace_charge_covers_what_a_member_measured() { + let block = crate::codec::rar50::LZ_BLOCK_SIZE; + // Peak resident bytes over the member, one encode per row, taken with + // `/proc/self/status` VmHWM so the allocator could not be involved. + // Anything that raises what the encoder holds has to move these too. + let measured: [(u64, u64); 5] = [ + (256 * 1024, 6_197_248), + (1024 * 1024, 9_170_944), + (3 * 1024 * 1024, 20_152_320), + (4 * 1024 * 1024, 20_234_240), + (8 * 1024 * 1024, 36_941_824), + ]; + for (dictionary, peak) in measured { + let charged = streaming_lz_workspace(dictionary, block, false); + assert!( + charged >= peak, + "a {dictionary}-byte dictionary measured {peak} against a charge of {charged}", + ); + // Streaming runs also retain input/history buffers not present in + // these single-member measurements. + assert!( + charged < peak * 4, + "a {dictionary}-byte dictionary is charged {charged} for a measured {peak}", + ); + } + } + + /// The same measurement for a parse that searches a tree, which holds two + /// links per window position where a chain holds one. Charging the chain's + /// four bytes for it let a fitted dictionary pick a window the budget could + /// not hold: at a 32 MiB limit a 16 MiB member peaked at 43 MB. + #[test] + fn the_workspace_charge_covers_a_tree_search_too() { + let block = crate::codec::rar50::LZ_BLOCK_SIZE; + // Peak resident bytes over the member, 16 MiB of manpage text at + // level 5, which is the only level that parses optimally. + let measured: [(u64, u64); 4] = [ + (256 * 1024, 19_668_992), + (1024 * 1024, 21_794_816), + (4 * 1024 * 1024, 47_030_272), + (8 * 1024 * 1024, 75_390_976), + ]; + for (dictionary, peak) in measured { + let charged = streaming_lz_workspace(dictionary, block, true); + let chain = streaming_lz_workspace(dictionary, block, false); + assert!( + charged > chain, + "a tree is charged {charged} against a chain's {chain}", + ); + // The two smallest windows measure above their charge: what is over + // is the member copies the whole-member path holds, which + // `whole_member_workspace` charges for separately and which no + // window size changes. + let copies = 17 * 1024 * 1024; + assert!( + charged + copies >= peak, + "a {dictionary}-byte dictionary measured {peak} against a charge of {charged}", + ); + } + } + + #[test] + fn the_workspace_charge_never_falls_as_the_dictionary_grows() { + let block = crate::codec::rar50::LZ_BLOCK_SIZE; + let mut previous = 0; + let mut dictionary = 128 * 1024u64; + while dictionary <= 64 * 1024 * 1024 { + for optimal_parse in [false, true] { + let charged = streaming_lz_workspace(dictionary, block, optimal_parse); + assert!( + charged >= previous, + "{dictionary} charged less than the size below it" + ); + if !optimal_parse { + previous = charged; + } + } + // Every size the format allows, not just the powers of two, because + // the rounding inside is the whole reason this function exists. + dictionary += 128 * 1024; + } + } + + #[test] + fn a_fitted_dictionary_shrinks_to_the_workspace_budget() { + let options = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(3); + // Derived exactly as the writer derives it, so the budget below is the + // one that admits a 256 KiB window and nothing larger. Spelling the + // parameters out here instead would make this test pass or fail on + // whether the ladder had moved since. + let block = crate::codec::rar50::MAX_LZ_BLOCK_SIZE; + let parse = encode_options_for_level(Some(3), DEFAULT_RAR50_DICTIONARY_SIZE) + .unwrap() + .optimal_parse; + let room_for_256k = streaming_lz_workspace(256 * 1024, block, parse); + + // A budget that cannot hold the fitted window takes the largest one it + // can, rather than failing a write the old 128 KiB default would have + // finished. + let fitted = dictionary_size_for_options(options, 8 * 1024 * 1024, room_for_256k).unwrap(); + + assert_eq!(fitted, 256 * 1024); + assert!(streaming_lz_workspace(fitted, block, parse) <= room_for_256k); + // A dictionary the caller named is used as asked, so the write fails + // later saying what it needed instead of quietly writing something else. + assert_eq!( + dictionary_size_for_options( + options.with_dictionary_size(4 * 1024 * 1024), + 8 * 1024 * 1024, + room_for_256k + ) + .unwrap(), + 4 * 1024 * 1024 + ); + } + + fn encode_member_with_filter_policy( + data: &[u8], + algorithm_version: u8, + policy: &FilterPolicy, + options: EncodeOptions, + ) -> Result> { + encode_member_with_filter_policy_and_progress( + data, + algorithm_version, + policy, + options, + None, + ) + } + + fn encode_member_with_auto_size_filter( + data: &[u8], + algorithm_version: u8, + options: EncodeOptions, + ) -> Result> { + encode_member_with_auto_size_filter_progress(data, algorithm_version, options, None) + } + + fn encode_member_with_filter_spec( + data: &[u8], + algorithm_version: u8, + filter: FilterSpec, + options: EncodeOptions, + ) -> crate::codec::Result> { + Unpack50Encoder::with_options(options).encode_member_with_filter( + data, + algorithm_version, + filter, + ) + } + + fn encode_member_with_filter_specs( + data: &[u8], + algorithm_version: u8, + filters: &[FilterSpec], + options: EncodeOptions, + ) -> crate::codec::Result> { + Unpack50Encoder::with_options(options).encode_member_with_filters( + data, + algorithm_version, + filters, + ) + } + + /// Builds a member from bytes the test already holds. + fn entry(name: &[u8], data: &[u8]) -> ArchiveEntry { + ArchiveEntry::new( + name.to_vec(), + EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data.to_vec())), + ) + } + + #[test] + fn compressed_writer_reports_determinate_progress() { + let data: Vec = (0usize..128 * 1024) + .map(|i| (i.wrapping_mul(37) % 251) as u8) + .collect(); + let entry = entry(b"payload.bin", &data) + .with_attributes(0x20) + .with_host_os(1); + let last = std::sync::atomic::AtomicU64::new(0); + let advances = AtomicUsize::new(0); + let intermediate = std::sync::atomic::AtomicBool::new(false); + let reporter = |event: WriteProgressEvent<'_>| { + if let WriteProgressEvent::Advanced { + operation: WriteOperation::Compression, + completed_bytes, + total_bytes, + .. + } = event + { + assert!(completed_bytes >= last.swap(completed_bytes, Ordering::Relaxed)); + assert!(completed_bytes <= total_bytes); + if completed_bytes < total_bytes { + intermediate.store(true, Ordering::Relaxed); + } + advances.fetch_add(1, Ordering::Relaxed); + } + }; + + Rar50Writer::new(WriterOptions::new( + ArchiveVersion::Rar70, + FeatureSet::default(), + )) + .entries([entry].to_vec()) + .filter_policy(FilterPolicy::Auto) + .progress(&reporter) + .finish() + .unwrap(); + + assert!(advances.load(Ordering::Relaxed) >= 1); + assert!(intermediate.load(Ordering::Relaxed)); + // Progress is now counted in input bytes, however many passes the + // filter search makes over them. + assert_eq!(last.load(Ordering::Relaxed), data.len() as u64); + } + + #[test] + #[cfg(feature = "recovery")] + fn recovery_writer_reports_determinate_pass_progress() { + let entry = entry(b"payload.bin", b"recovery progress payload") + .with_attributes(0x20) + .with_host_os(1); + let starts = AtomicUsize::new(0); + let advances = AtomicUsize::new(0); + let finishes = AtomicUsize::new(0); + let reporter = |event: WriteProgressEvent<'_>| match event { + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Recovery, + total_bytes: Some(total), + .. + } => { + assert!(total > 0); + starts.fetch_add(1, Ordering::Relaxed); + } + WriteProgressEvent::Advanced { + operation: WriteOperation::Recovery, + completed_bytes, + total_bytes, + .. + } => { + assert!(completed_bytes <= total_bytes); + advances.fetch_add(1, Ordering::Relaxed); + } + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Recovery, + .. + } => { + finishes.fetch_add(1, Ordering::Relaxed); + } + _ => {} + }; + let features = FeatureSet::store_only(); + + Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(0), + ) + .entries([entry].to_vec()) + .recovery_percent(Some(10)) + .progress(&reporter) + .finish() + .unwrap(); + + assert!(starts.load(Ordering::Relaxed) >= 1); + assert!(advances.load(Ordering::Relaxed) >= 1); + assert_eq!( + starts.load(Ordering::Relaxed), + finishes.load(Ordering::Relaxed) + ); + } + + impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.0.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } + } + + #[derive(Debug, Clone, PartialEq, Eq)] + // Historical numeric snapshots; timestamp presence has dedicated regression tests. + struct CollectedEntry { + name: Vec, + data: Vec, + file_time: u32, + attr: u64, + host_os: u64, + is_directory: bool, + } + + fn collect_extract(archive: &Archive) -> Result> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(crate::ArchiveReadOptions::default(), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter(data))) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time.unwrap_or(0), + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) + .collect()) + } + + #[test] + fn streaming_writer_round_trips_across_input_blocks() { + let data = b"bounded streaming member data\n".repeat(80_000); + let entry = ArchiveEntry::new( + b"large.txt".to_vec(), + EntrySource::from_bytes(Arc::<[u8]>::from(data.clone())), + ) + .with_mtime(None) + .with_attributes(0x20) + .with_host_os(1); + let mut bytes = Vec::new(); + write_streaming_archive_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ArchiveExtras::default(), + &WriterResources::default(), + &mut bytes, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"large.txt"); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn streaming_writer_rejects_workspace_larger_than_budget() { + let entry = ArchiveEntry::new( + b"small.txt".to_vec(), + EntrySource::from_bytes(Arc::<[u8]>::from(&b"small"[..])), + ) + .with_mtime(None) + .with_attributes(0x20) + .with_host_os(1); + let result = write_streaming_archive_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ArchiveExtras::default(), + &WriterResources::new(1024), + &mut Vec::new(), + ); + assert!(matches!(result, Err(Error::MemoryLimitExceeded { .. }))); + } + + #[test] + fn streaming_writer_propagates_partial_output_failure_and_cleans_spools() { + struct FailingOutput { + remaining: usize, + written: usize, + } + impl Write for FailingOutput { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.remaining == 0 { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + let count = bytes.len().min(self.remaining); + self.remaining -= count; + self.written += count; + Ok(count) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let scratch = crate::scratch::case("rar50-partial-output-failure"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let data = vec![42u8; 8192]; + let entry = ArchiveEntry::new(b"file".to_vec(), EntrySource::from_bytes(data)); + let mut output = FailingOutput { + remaining: 1024, + written: 0, + }; + let error = write_streaming_archive_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ArchiveExtras::default(), + &resources, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!(output.written, 1024); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[test] + fn parallel_streaming_blocks_are_byte_identical_to_serial_blocks() { + let mut data = Vec::new(); + for index in 0..52_000u32 { + data.extend_from_slice(b"parallel block identity payload "); + data.extend_from_slice(&index.to_le_bytes()); + } + // A stretch that does not move, so the two loops have to agree about + // growing a block and not only about cutting one every block_size. + data.extend(std::iter::repeat_n(0u8, 3 * 1024 * 1024)); + data.extend((0..400_000u32).map(|index| (index as u8).wrapping_mul(37))); + let options = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()); + let dictionary_size = + dictionary_size_for_options(options, data.len() as u64, u64::MAX).unwrap(); + let algorithm_version = rar50_algorithm_version(options, dictionary_size).unwrap(); + let encode_options = + encode_options_for_level(options.compression_level, dictionary_size).unwrap(); + let block_size = crate::codec::rar50::LZ_BLOCK_SIZE; + let mut serial = Vec::new(); + crate::codec::rar50::encode_lz_reader_to( + &mut Cursor::new(&data), + data.len() as u64, + &mut serial, + algorithm_version, + encode_options, + block_size, + None, + ) + .unwrap(); + let original = data.clone(); + + let source = EntrySource::from_bytes(Arc::<[u8]>::from(data)); + let required = streaming_lz_workspace( + dictionary_size, + crate::codec::rar50::MAX_LZ_BLOCK_SIZE, + encode_options.optimal_parse, + ); + let mut prepared = compress::compress_members_reporting( + &[source], + compress::CompressPlan { + algorithm_version, + encode_options, + dictionary_size, + block_size, + solid: false, + method: 1, + filter_policy: FilterPolicy::None, + candidates: vec![encode_options].into(), + }, + &WriterResources::new(required.saturating_mul(4)), + &|_| true, + ) + .unwrap(); + let mut parallel = Vec::new(); + prepared[0].packed.copy_to(&mut parallel).unwrap(); + // Arca: the streaming writer keeps the tree finder across a member's + // blocks where this per-block reference starts each block afresh, so + // the two no longer match byte for byte. What has to hold is that the + // writer's stream decodes to the input and packs no worse. + let decoded = + crate::codec::rar50::decode_lz(¶llel, algorithm_version, original.len()).unwrap(); + assert!(decoded == original); + assert!(parallel.len() <= serial.len(), "{} > {}", parallel.len(), serial.len()); + } + + #[test] + fn streaming_entry_order_is_identical_at_one_and_many_job_budgets() { + let entries: Vec<_> = (0..4u8) + .map(|index| { + let mut data = vec![index; 1_300_000]; + data.extend((0..300_000).map(|offset| (offset as u8).wrapping_add(index))); + ArchiveEntry::new( + format!("entry-{index}.bin").into_bytes(), + EntrySource::from_bytes(Arc::<[u8]>::from(data)), + ) + .with_mtime(None) + .with_attributes(0x20) + .with_host_os(1) + }) + .collect(); + let options = WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()); + let dictionary_size = + dictionary_size_for_options(options, dictionary_reach(&entries, false), u64::MAX) + .unwrap(); + // Exactly what the writer charges itself, so the budget below admits one + // job and no more. Derived rather than written down, because a charge + // larger than the writer's would quietly make this a two-job test. + let required = streaming_lz_workspace( + dictionary_size, + 1024 * 1024, + encode_options_for_level(options.compression_level, dictionary_size) + .unwrap() + .optimal_parse, + ); + let mut one_job = Vec::new(); + write_streaming_archive_to( + &entries, + options, + ArchiveExtras::default(), + &WriterResources::new(required), + &mut one_job, + ) + .unwrap(); + let mut many_jobs = Vec::new(); + write_streaming_archive_to( + &entries, + options, + ArchiveExtras::default(), + &WriterResources::new(required.saturating_mul(4)), + &mut many_jobs, + ) + .unwrap(); + assert_eq!(many_jobs, one_job); + } + + #[test] + fn streaming_writer_rejects_a_source_larger_than_its_declared_size() { + let source = EntrySource::from_opener(4, || Ok(Box::new(Cursor::new(b"data!".to_vec())))); + let entry = ArchiveEntry::new(b"changing.bin".to_vec(), source) + .with_attributes(0x20) + .with_host_os(1); + let result = write_streaming_archive_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ArchiveExtras::default(), + &WriterResources::default(), + &mut Vec::new(), + ); + let error = result.unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::SourceChanged); + assert_eq!( + error.entry_context(), + Some((b"changing.bin".as_slice(), "compressing")) + ); + } + + #[test] + #[cfg(feature = "encryption")] + fn encrypted_streaming_writer_round_trips_across_input_blocks() { + let data = b"encrypted bounded streaming member data\n".repeat(40_000); + let entry = ArchiveEntry::new( + b"secret.txt".to_vec(), + EntrySource::from_bytes(Arc::<[u8]>::from(data.clone())), + ) + .with_mtime(None) + .with_attributes(0x20) + .with_host_os(1) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let mut bytes = Vec::new(); + write_streaming_archive_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, features), + ArchiveExtras::default(), + &WriterResources::default(), + &mut bytes, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let entries = RefCell::new(Vec::new()); + archive + .extract_to( + crate::ArchiveReadOptions::with_password(b"password"), + |meta| { + let output = Rc::new(RefCell::new(Vec::new())); + entries + .borrow_mut() + .push((meta.name.clone(), Rc::clone(&output))); + Ok(Box::new(CollectWriter(output))) + }, + ) + .unwrap(); + let entries = entries.into_inner(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, b"secret.txt"); + assert_eq!(*entries[0].1.borrow(), data); + } + + #[test] + fn internal_literal_only_compressed_member_round_trips_through_rar50_reader() { + let data = b"RAR5 literal-only compressed format-layer experiment\n"; + let packed = encode_literal_only(data, 0).unwrap(); + let name = b"compressed.txt"; + + let mut archive = Vec::new(); + archive.extend_from_slice(RAR50_SIGNATURE); + write_main_header( + &mut archive, + 0, + None, + &[], + &crate::WriterResources::default(), + ) + .unwrap(); + + let mut extra = Vec::new(); + write_hash_record_with_value(&mut extra, blake2sp::hash(data)).unwrap(); + let compression_info = 1 << 7; // RAR5 v0, non-solid, method m1, 128 KiB dictionary. + let specific = file_specific( + name, + data.len() as u64, + crc32(data), + 0x20, + None, + compression_info, + 0, + false, + &crate::WriterResources::default(), + ) + .unwrap(); + write_block( + &mut archive, + HEAD_FILE, + HFL_EXTRA | HFL_DATA, + Some(packed.len() as u64), + &specific, + &extra, + &packed, + &crate::WriterResources::default(), + ) + .unwrap(); + write_end_header(&mut archive, 0, &crate::WriterResources::default()).unwrap(); + + let parsed = Archive::parse(&archive).unwrap(); + let file = parsed.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + assert_eq!(info.method, 1); + assert_eq!(info.dictionary_size, 128 * 1024); + + let extracted = collect_extract(&parsed).unwrap(); + assert_eq!(extracted[0].name, name); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn writer_stamps_requested_rar50_dictionary_size() { + let data = b"RAR5 dictionary-size writer option fixture".repeat(64); + let options = + WriterOptions::new(crate::ArchiveVersion::Rar50, crate::FeatureSet::default()) + .with_dictionary_size(512 * 1024); + let entries = [entry(b"dict.bin", &data) + .with_attributes(0x20) + .with_host_os(3)]; + let archive = Rar50Writer::new(options) + .entries(entries.to_vec()) + .finish() + .unwrap(); + + let parsed = Archive::parse(&archive).unwrap(); + let info = parsed + .files() + .next() + .unwrap() + .decoded_compression_info() + .unwrap(); + let extracted = collect_extract(&parsed).unwrap(); + + assert_eq!(info.algorithm_version, 0); + assert_eq!(info.dictionary_size, 512 * 1024); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn writer_uses_rar7_dictionary_fields_when_size_needs_v1_encoding() { + let data = b"RAR7 dictionary-size writer option fixture".repeat(64); + let options = + WriterOptions::new(crate::ArchiveVersion::Rar70, crate::FeatureSet::default()) + .with_dictionary_size(192 * 1024); + let entries = [entry(b"dict7.bin", &data) + .with_attributes(0x20) + .with_host_os(3)]; + let archive = Rar50Writer::new(options) + .entries(entries.to_vec()) + .finish() + .unwrap(); + + let parsed = Archive::parse(&archive).unwrap(); + let info = parsed + .files() + .next() + .unwrap() + .decoded_compression_info() + .unwrap(); + let extracted = collect_extract(&parsed).unwrap(); + + assert_eq!(info.algorithm_version, 1); + assert_eq!(info.dictionary_size, 192 * 1024); + assert_eq!(extracted[0].data, data); + } + + #[test] + fn writer_rejects_unencodable_rar50_dictionary_size() { + let options = + WriterOptions::new(crate::ArchiveVersion::Rar50, crate::FeatureSet::default()) + .with_dictionary_size(192 * 1024); + let entries = [entry(b"bad.bin", b"data data data data") + .with_attributes(0x20) + .with_host_os(3)]; + + assert!(matches!( + Rar50Writer::new(options).entries(entries.to_vec()).finish(), + Err(Error::InvalidArgument( + "RAR 5 v0 dictionary size must be a power-of-two multiple of 128 KiB" + )) + )); + } + + #[test] + fn only_the_bottom_of_the_ladder_settles_for_a_greedy_parse() { + // Level 1 is the rung that exists to be quick. Everything above it + // parses by shortest path, because on this corpus nothing else closes + // the distance to WinRAR at any rung: the parse is worth several + // percent and the search depth around it is worth tenths. + for level in 0..=1u8 { + let options = + encode_options_for_level(Some(level), DEFAULT_RAR50_DICTIONARY_SIZE).unwrap(); + assert!(!options.optimal_parse, "level {level} priced every path"); + } + for level in 2..=5u8 { + let options = + encode_options_for_level(Some(level), DEFAULT_RAR50_DICTIONARY_SIZE).unwrap(); + assert!(options.optimal_parse, "level {level} settled for greedy"); + } + assert!( + encode_options_for_level(None, DEFAULT_RAR50_DICTIONARY_SIZE) + .unwrap() + .optimal_parse, + "an absent level is the default level, which parses by shortest path", + ); + } + + #[test] + fn every_level_encodes_the_member_once() { + // Level 5 used to try levels four down to one as well and keep the + // smallest. Over the whole bench corpus that caught one byte, on one + // member, for four extra whole-member encodes. + for level in 0..=5u8 { + let candidates = + encode_option_candidates_for_level(Some(level), DEFAULT_RAR50_DICTIONARY_SIZE) + .unwrap(); + assert_eq!(candidates.len(), 1, "level {level} encodes more than once"); + } + } + + #[test] + fn a_filter_screen_does_not_pay_for_the_optimal_parse() { + use crate::filter_search::FilterSearch; + + let search = crate::rar50::write::filter_policy::Rar50Search { + algorithm_version: 0, + }; + let five = encode_options_for_level(Some(5), DEFAULT_RAR50_DICTIONARY_SIZE).unwrap(); + let screen = search.screen_options(five); + + assert!(!screen.optimal_parse); + assert_eq!(screen.max_match_candidates, five.max_match_candidates); + assert_eq!(screen.max_match_distance, five.max_match_distance); + } + + #[test] + fn the_ladder_never_packs_larger_as_it_climbs() { + // A rung that costs more time and returns more bytes is a bug, and it + // was one: level 4 used to return level 3's output for half again the + // time, because the shortest-path parse only started at level 5. + let long_tail = b"stable long match payload for RAR5 best-level search ".repeat(10); + let mut data = Vec::new(); + data.extend_from_slice(b"abc"); + data.extend_from_slice(&long_tail); + for index in 0..320usize { + data.extend_from_slice(b"abc"); + data.push((index as u8).wrapping_mul(37)); + data.extend_from_slice(b" near same-hash decoy "); + data.extend_from_slice(&(index as u32).to_le_bytes()); + } + data.extend_from_slice(b"abc"); + data.extend_from_slice(&long_tail); + + let mut previous: Option<(u8, usize)> = None; + for level in 1..=5u8 { + let options = + encode_options_for_level(Some(level), DEFAULT_RAR50_DICTIONARY_SIZE).unwrap(); + let packed = + encode_member_with_filter_policy(&data, 0, &FilterPolicy::None, options).unwrap(); + + if let Some((lower, size)) = previous { + assert!( + packed.len() <= size, + "level {level} packed larger than level {lower}: {} against {size}", + packed.len(), + ); + } + previous = Some((level, packed.len())); + + let mut decoder = crate::codec::rar50::Unpack50Decoder::new(); + let output = decoder + .decode_member( + &packed, + 0, + data.len(), + false, + crate::codec::rar50::DecodeMode::Lz, + ) + .unwrap(); + assert_eq!(output, data, "level {level} did not round trip"); + } + } + + #[test] + fn auto_x86_filter_ranges_select_dense_opcode_clusters() { + let mut data = vec![0u8; 100_000]; + data[1_000] = 0xe8; + data[7_000] = 0xe9; + for pos in [50_000, 50_064, 50_128, 50_192] { + data[pos] = 0xe8; + } + for pos in [70_000, 70_064, 70_128, 70_192] { + data[pos] = 0xe9; + } + + let e8_ranges = auto_x86_filter_ranges(&data, false); + assert!(e8_ranges + .iter() + .any(|range| range.start <= 50_000 && range.end >= 50_197)); + assert!(!e8_ranges + .iter() + .any(|range| range.start <= 1_000 && range.end >= 1_005)); + + let e8e9_ranges = auto_x86_filter_ranges(&data, true); + assert!(e8e9_ranges + .iter() + .any(|range| range.start <= 70_000 && range.end >= 70_197)); + } + + #[test] + fn auto_x86_filter_policy_can_emit_multiple_disjoint_ranges() { + let mut data = vec![0x41u8; 80_000]; + for cluster_start in [8_000, 60_000] { + for index in 0..8 { + let pos = cluster_start + index * 64; + data[pos] = 0xe8; + data[pos + 1..pos + 5].copy_from_slice(&(0x2000u32 + index as u32).to_le_bytes()); + } + } + let ranges = disjoint_filter_ranges(auto_x86_filter_ranges(&data, false)); + let filters: Vec<_> = ranges + .into_iter() + .map(|range| FilterSpec::range(FilterKind::E8, range)) + .collect(); + + let packed = + encode_member_with_filter_specs(&data, 0, &filters, EncodeOptions::default()).unwrap(); + let mut decoder = crate::codec::rar50::Unpack50Decoder::new(); + let output = decoder + .decode_member( + &packed, + 0, + data.len(), + false, + crate::codec::rar50::DecodeMode::Lz, + ) + .unwrap(); + + assert_eq!(filters.len(), 2); + assert_eq!(output, data); + } + + #[test] + fn auto_filter_policy_considers_ranged_delta_candidates() { + let mut data = vec![0x55u8; AUTO_DELTA_EDGE_SKIP]; + for sample in 0..256u16 { + let left = sample as u8; + let right = left.wrapping_add(1); + data.extend_from_slice(&[left, right]); + } + data.extend(std::iter::repeat_n(0xaa, AUTO_DELTA_EDGE_SKIP)); + let options = EncodeOptions::default(); + + let plain = encode_lz_member_with_options(&data, 0, options).unwrap(); + let ranged = encode_member_with_filter_spec( + &data, + 0, + FilterSpec::range( + FilterKind::Delta { channels: 2 }, + auto_delta_filter_range(&data, 2).unwrap(), + ), + options, + ) + .unwrap(); + let auto = encode_member_with_auto_size_filter(&data, 0, options).unwrap(); + + assert!(ranged.len() < plain.len()); + assert!(auto.len() <= ranged.len()); + let mut decoder = crate::codec::rar50::Unpack50Decoder::new(); + let output = decoder + .decode_member( + &auto, + 0, + data.len(), + false, + crate::codec::rar50::DecodeMode::Lz, + ) + .unwrap(); + assert_eq!(output, data); + } + + #[test] + fn explicit_filters_accept_large_members_after_filter_ranges_are_split() { + let data = vec![0u8; 4 * 1024 * 1024 + 1]; + let packed = encode_member_with_filter_policy( + &data, + 0, + &FilterPolicy::explicit(FilterKind::Delta { channels: 1 }), + EncodeOptions::new(0), + ) + .unwrap(); + let mut decoder = crate::codec::rar50::Unpack50Decoder::new(); + + assert_eq!( + decoder + .decode_member( + &packed, + 0, + data.len(), + false, + crate::codec::rar50::DecodeMode::Lz + ) + .unwrap(), + data + ); + } + + #[test] + #[ignore = "requires local rar command; used for reference-validating experimental RAR5 compressed output"] + fn reference_rar_accepts_internal_literal_only_compressed_member() { + let data = b"RAR5 literal-only compressed reference experiment\n"; + let packed = encode_literal_only(data, 0).unwrap(); + let name = b"compressed.txt"; + + let mut archive = Vec::new(); + archive.extend_from_slice(RAR50_SIGNATURE); + write_main_header( + &mut archive, + 0, + None, + &[], + &crate::WriterResources::default(), + ) + .unwrap(); + + let mut extra = Vec::new(); + write_hash_record_with_value(&mut extra, blake2sp::hash(data)).unwrap(); + let specific = file_specific( + name, + data.len() as u64, + crc32(data), + 0x20, + None, + 1 << 7, + 0, + false, + &crate::WriterResources::default(), + ) + .unwrap(); + write_block( + &mut archive, + HEAD_FILE, + HFL_EXTRA | HFL_DATA, + Some(packed.len() as u64), + &specific, + &extra, + &packed, + &crate::WriterResources::default(), + ) + .unwrap(); + write_end_header(&mut archive, 0, &crate::WriterResources::default()).unwrap(); + + let dir = crate::scratch::case("rars-rar50-literal-only"); + let path = dir.join("archive.rar"); + fs::write(&path, archive).unwrap(); + let output = match Command::new("rar").arg("t").arg(&path).output() { + Ok(output) => output, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + eprintln!("skipping reference test: local `rar` command is not installed"); + return; + } + Err(error) => panic!("failed to run rar: {error}"), + }; + if std::env::var_os("RARS_KEEP_REFERENCE_ARCHIVE").is_some() { + eprintln!("kept reference archive: {}", path.display()); + std::mem::forget(dir); + } + + assert!( + output.status.success(), + "rar rejected experimental RAR5 compressed output\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + #[ignore = "requires local rar command; used for reference-validating experimental RAR5 match output"] + fn reference_rar_accepts_internal_match_compressed_member() { + let data = b"RAR5 match compressed reference experiment\n".repeat(8); + let packed = encode_lz_member(&data, 0).unwrap(); + let name = b"compressed.txt"; + + let mut archive = Vec::new(); + archive.extend_from_slice(RAR50_SIGNATURE); + write_main_header( + &mut archive, + 0, + None, + &[], + &crate::WriterResources::default(), + ) + .unwrap(); + + let mut extra = Vec::new(); + write_hash_record_with_value(&mut extra, blake2sp::hash(&data)).unwrap(); + let specific = file_specific( + name, + data.len() as u64, + crc32(&data), + 0x20, + None, + 1 << 7, + 0, + false, + &crate::WriterResources::default(), + ) + .unwrap(); + write_block( + &mut archive, + HEAD_FILE, + HFL_EXTRA | HFL_DATA, + Some(packed.len() as u64), + &specific, + &extra, + &packed, + &crate::WriterResources::default(), + ) + .unwrap(); + write_end_header(&mut archive, 0, &crate::WriterResources::default()).unwrap(); + + let dir = crate::scratch::case("rars-rar50-match"); + let path = dir.join("archive.rar"); + fs::write(&path, archive).unwrap(); + let output = match Command::new("rar").arg("t").arg(&path).output() { + Ok(output) => output, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + eprintln!("skipping reference test: local `rar` command is not installed"); + return; + } + Err(error) => panic!("failed to run rar: {error}"), + }; + if std::env::var_os("RARS_KEEP_REFERENCE_ARCHIVE").is_some() { + eprintln!("kept reference archive: {}", path.display()); + std::mem::forget(dir); + } + + assert!( + output.status.success(), + "rar rejected experimental RAR5 match output\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn writer_options_default_targets_rar50_with_store_only_features() { + let options = WriterOptions::default(); + assert_eq!(options.target, crate::ArchiveVersion::Rar50); + assert_eq!(options.features, crate::FeatureSet::store_only()); + } +} diff --git a/arca-rar/vendor/rars/src/rar50/write/plan.rs b/arca-rar/vendor/rars/src/rar50/write/plan.rs new file mode 100644 index 00000000..2f4509b4 --- /dev/null +++ b/arca-rar/vendor/rars/src/rar50/write/plan.rs @@ -0,0 +1,236 @@ +//! Codec settings and execution decisions, resolved before scheduling work. + +use super::FilterPolicy; +use crate::codec::rar50::EncodeOptions; +use crate::streaming::preparation::Records; +use crate::{Result, WriterResources}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum FallbackReason { + AutomaticFilterWorkspace { required: u64, limit: u64 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Execution { + WholeMember, + Blocks { fallback: FallbackReason }, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct MemberPlan { + pub(super) execution: Execution, + /// Whole-member workspace or one streaming job's workspace, never both. + pub(super) workspace: u64, +} + +pub(super) enum ExecutionPlan { + Stored, + Blocks { workspace: u64 }, + IndependentMembers(Records), +} + +impl ExecutionPlan { + #[cfg(test)] + pub(super) fn new( + settings: &CompressPlan, + sizes: impl ExactSizeIterator, + limit: u64, + ) -> Self { + Self::with_resources(settings, sizes, &WriterResources::new(limit)).unwrap() + } + + pub(super) fn with_resources( + settings: &CompressPlan, + sizes: impl ExactSizeIterator, + resources: &WriterResources, + ) -> Result { + let limit = resources.memory_limit(); + // Select the requested mode before estimating its active workspace. + if settings.method == 0 { + return Ok(Self::Stored); + } + let whole_members = !settings.solid + && (settings.filter_policy != FilterPolicy::None || settings.candidates.len() > 1); + if !whole_members { + return Ok(Self::Blocks { + workspace: streaming_workspace(settings, true), + }); + } + Ok(Self::IndependentMembers(Records::collect( + sizes.map(|size| { + let required = if size == 0 { + 0 + } else { + whole_member_workspace(size, settings) + }; + Ok( + if required <= limit || settings.filter_policy != FilterPolicy::Auto { + MemberPlan { + execution: Execution::WholeMember, + workspace: required, + } + } else { + MemberPlan { + execution: Execution::Blocks { + fallback: FallbackReason::AutomaticFilterWorkspace { + required, + limit, + }, + }, + // Automatic fallback uses only the base encoder candidate. + workspace: streaming_workspace(settings, false), + } + }, + ) + }), + resources, + )?)) + } + + #[cfg(test)] + fn members(&self) -> &[MemberPlan] { + match self { + Self::IndependentMembers(members) => members, + _ => panic!("expected independent member plan"), + } + } +} + +fn streaming_workspace(settings: &CompressPlan, include_candidates: bool) -> u64 { + let optimal = settings.encode_options.optimal_parse + || (include_candidates + && settings + .candidates + .iter() + .any(|options| options.optimal_parse)); + super::streaming_lz_workspace( + settings.dictionary_size, + crate::codec::rar50::MAX_LZ_BLOCK_SIZE, + optimal, + ) +} + +#[derive(Debug)] +#[cfg_attr(test, derive(Clone))] +pub(super) struct CompressPlan { + pub(super) algorithm_version: u8, + pub(super) encode_options: EncodeOptions, + pub(super) dictionary_size: u64, + pub(super) block_size: usize, + pub(super) solid: bool, + /// The RAR 5 compression method. Method zero means the members are stored + /// verbatim, so nothing is compressed at all. + pub(super) method: u8, + /// Filters and multi-candidate encoding both need the whole member at + /// once, so they only run for members that fit the memory budget. + pub(super) filter_policy: FilterPolicy, + pub(super) candidates: Records, +} + +/// Working memory a member needs to be filtered as a whole: the member, the +/// filtered copy, and the candidate packed outputs being compared. +pub(super) fn whole_member_workspace(input_size: u64, plan: &CompressPlan) -> u64 { + let optimal = plan.encode_options.optimal_parse + || plan.candidates.iter().any(|options| options.optimal_parse); + let reach = plan + .candidates + .iter() + .map(|options| options.max_match_distance as u64) + .chain(std::iter::once( + plan.encode_options.max_match_distance as u64, + )) + .max() + .unwrap_or(0) + .min(input_size) + .max(crate::codec::rar50::LZ_BLOCK_SIZE as u64); + let block = input_size.min(crate::codec::rar50::MAX_LZ_BLOCK_SIZE as u64) as usize; + // Input, transformed input and competing packed outputs stay live alongside + // the finder and the per-block token/parse workspace, not instead of them. + input_size + .saturating_mul(4) + .saturating_add(super::streaming_lz_workspace(reach, block, optimal)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn settings() -> CompressPlan { + let encode_options = EncodeOptions::new(8).with_max_match_distance(128 * 1024); + CompressPlan { + algorithm_version: 0, + encode_options, + dictionary_size: 128 * 1024, + block_size: crate::codec::rar50::LZ_BLOCK_SIZE, + solid: false, + method: 1, + filter_policy: FilterPolicy::Auto, + candidates: vec![encode_options].into(), + } + } + + #[test] + fn automatic_fallback_records_the_admission_boundary_and_reason() { + let settings = settings(); + let size = 2 * 1024 * 1024; + let required = whole_member_workspace(size, &settings); + let at_limit = ExecutionPlan::new(&settings, [size].into_iter(), required); + assert_eq!(at_limit.members()[0].execution, Execution::WholeMember); + let below = ExecutionPlan::new(&settings, [size].into_iter(), required - 1); + assert_eq!( + below.members()[0].execution, + Execution::Blocks { + fallback: FallbackReason::AutomaticFilterWorkspace { + required, + limit: required - 1 + }, + } + ); + assert!(below.members()[0].workspace < required); + } + + #[test] + fn explicit_filters_and_candidate_trials_cannot_silently_fall_back() { + let mut settings = settings(); + settings.filter_policy = + FilterPolicy::Explicit(crate::FilterSpec::whole(crate::FilterKind::Delta { + channels: 1, + })); + for filter in [settings.filter_policy.clone(), FilterPolicy::None] { + settings.filter_policy = filter; + settings + .candidates + .push_growing(settings.encode_options) + .unwrap(); + let execution = ExecutionPlan::new(&settings, [16].into_iter(), 1); + assert_eq!(execution.members()[0].execution, Execution::WholeMember); + assert!(execution.members()[0].workspace > 1); + } + } + + #[test] + fn stored_empty_and_solid_modes_do_not_acquire_whole_member_workspace() { + let mut settings = settings(); + let empty = ExecutionPlan::new(&settings, [0].into_iter(), 0); + assert_eq!(empty.members()[0].workspace, 0); + assert_eq!(empty.members()[0].execution, Execution::WholeMember); + settings.method = 0; + let stored = ExecutionPlan::new(&settings, [u64::MAX].into_iter(), 0); + assert!(matches!(stored, ExecutionPlan::Stored)); + settings.method = 1; + settings.solid = true; + let solid = ExecutionPlan::new(&settings, [16, u64::MAX].into_iter(), u64::MAX); + assert!(matches!(solid, ExecutionPlan::Blocks { workspace } if workspace > 0)); + } + + #[test] + fn independent_member_planning_respects_the_execution_ledger() { + let ledger = crate::codec::workspace::Allowance::limited(0); + let resources = WriterResources::default().with_execution_allowance(ledger.clone()); + let error = ExecutionPlan::with_resources(&settings(), [1].into_iter(), &resources) + .err() + .unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(ledger.used(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/read_control.rs b/arca-rar/vendor/rars/src/read_control.rs new file mode 100644 index 00000000..f36c0f75 --- /dev/null +++ b/arca-rar/vendor/rars/src/read_control.rs @@ -0,0 +1,367 @@ +//! Cooperative reader cancellation, with observation local to an operation. +use crate::{Error, Result}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +/// A shared one-way cancellation signal for archive readers. +/// Clone the token to cancel from another thread or an input/output callback. +/// Supply it through [`crate::ArchiveReadOptions::with_cancellation`]. A token +/// cannot be reset; create a new one for another operation after cancellation. +/// Parsing does not retain the token or apply it to later extraction. +/// +/// Readers check before starting work, between headers and members, around +/// output callbacks, during bounded reads/writes, and periodically in decoder, +/// filter and VM loops. Parallel extraction joins admitted workers before +/// returning; workers check the signal before decoding and during codec work. +/// +/// Cancellation is cooperative, without a wall-clock deadline. Blocked caller +/// I/O, key derivation, allocation, copies and individual checksum/crypto library +/// operations cannot be preempted. Once observed, cancellation returns +/// [`Error::Cancelled`] (possibly with entry/volume context). An unrelated error +/// is not replaced merely because the signal was set concurrently. +/// +/// Partial output and earlier extracted members can remain. The call aborts; +/// it does not resume solid state, roll back sinks or refund output budgets. +#[derive(Clone, Debug, Default)] +pub struct ReadCancellation(Arc, Option>); + +impl ReadCancellation { + /// Creates an uncancelled signal. + pub fn new() -> Self { + Self::default() + } + /// Create a signal which observes this token as well as its own cancellation. + /// Cancelling the child never cancels its parent or siblings. This lets a + /// scoped operation stop its workers without poisoning a caller-owned token. + pub fn child(&self) -> Self { + Self( + Arc::new(AtomicBool::new(false)), + Some(Arc::new(self.clone())), + ) + } + /// Signals cancellation to every clone. Calling this repeatedly is harmless. + pub fn cancel(&self) { + self.0.store(true, Ordering::Relaxed); + } + /// Returns whether any clone has signalled cancellation. + pub fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + || self.1.as_ref().is_some_and(|parent| parent.is_cancelled()) + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct ReadControl(Option>); + +#[derive(Debug)] +struct Observation { + token: ReadCancellation, + observed: AtomicBool, + #[cfg(test)] + checks_remaining: std::sync::atomic::AtomicUsize, +} + +impl ReadControl { + #[cfg(feature = "recovery")] + pub(crate) fn cancellation(&self) -> Option<&ReadCancellation> { + self.0.as_ref().map(|state| &state.token) + } + pub(crate) fn is_enabled(&self) -> bool { + self.0.is_some() + } + pub(crate) fn new(token: Option<&ReadCancellation>) -> Self { + Self(token.map(|token| { + Arc::new(Observation { + token: token.clone(), + observed: AtomicBool::new(false), + #[cfg(test)] + checks_remaining: std::sync::atomic::AtomicUsize::new(usize::MAX), + }) + })) + } + pub(crate) fn check(&self) -> Result<()> { + if let Some(state) = &self.0 { + #[cfg(test)] + { + let mut remaining = state.checks_remaining.load(Ordering::Relaxed); + while remaining != usize::MAX { + match state.checks_remaining.compare_exchange_weak( + remaining, + remaining.saturating_sub(1), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(0) => { + state.token.cancel(); + break; + } + Ok(_) => break, + Err(actual) => remaining = actual, + } + } + } + if state.token.is_cancelled() { + state.observed.store(true, Ordering::Relaxed); + return Err(Error::Cancelled); + } + } + Ok(()) + } + /// Test-only checkpoint scheduling avoids races and wall-clock sleeps. + #[cfg(test)] + pub(crate) fn cancel_after_checks(&self, successful_checks: usize) { + self.0 + .as_ref() + .expect("configured token") + .checks_remaining + .store(successful_checks, Ordering::Relaxed); + } + pub(crate) fn finish(&self, result: Result) -> Result { + // Do not turn a real error into cancellation just because the token was + // signalled later. Restore only cancellation actually observed by this work. + if self + .0 + .as_ref() + .is_some_and(|s| s.observed.load(Ordering::Relaxed)) + { + match result { + Err(e) if e.kind() == crate::ErrorKind::Cancelled => Err(e), + _ => Err(Error::Cancelled), + } + } else { + result + } + } + pub(crate) fn check_codec(&self) -> crate::codec::Result<()> { + self.check().map_err(|_| crate::codec::Error::Cancelled) + } + pub(crate) fn reader(&self, reader: R) -> ControlledReader { + ControlledReader { + reader, + control: self.clone(), + } + } + pub(crate) fn write_all( + &self, + writer: &mut (impl Write + ?Sized), + bytes: &[u8], + ) -> io::Result<()> { + if !self.is_enabled() { + return writer.write_all(bytes); + } + let mut remaining = bytes; + while !remaining.is_empty() { + self.check().map_err(io::Error::other)?; + match writer.write(&remaining[..remaining.len().min(64 * 1024)]) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), + Ok(n) => remaining = &remaining[n..], + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } + self.check().map_err(io::Error::other) + } + pub(crate) fn poller(&self) -> Poller { + Poller { + control: self.clone(), + steps: 0, + next_position: 0, + } + } +} + +pub(crate) struct ControlledReader { + reader: R, + control: ReadControl, +} +impl Read for ControlledReader { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.control.0.is_none() { + return self.reader.read(bytes); + } + self.control.check().map_err(io::Error::other)?; + let len = bytes.len().min(64 * 1024); + self.reader.read(&mut bytes[..len]) + } +} +impl Seek for ControlledReader { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.control.check().map_err(io::Error::other)?; + self.reader.seek(pos) + } +} + +pub(crate) struct Poller { + control: ReadControl, + steps: usize, + next_position: usize, +} +impl Poller { + // Bound both no-output symbol work and runs that expand large matches. + pub(crate) fn check(&mut self, position: usize) -> Result<()> { + if self.control.0.is_none() { + return Ok(()); + } + if self.steps == 0 || position >= self.next_position { + self.control.check()?; + self.steps = 4096; + self.next_position = position.saturating_add(64 * 1024); + } + self.steps -= 1; + Ok(()) + } + pub(crate) fn check_codec(&mut self, position: usize) -> crate::codec::Result<()> { + self.check(position) + .map_err(|_| crate::codec::Error::Cancelled) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn child_cancellation_is_local_but_parent_cancellation_propagates() { + let parent = ReadCancellation::new(); + let first = parent.child(); + let second = parent.child(); + let nested = second.child(); + first.cancel(); + assert!(!parent.is_cancelled()); + assert!(!second.is_cancelled()); + assert!(!nested.is_cancelled()); + parent.cancel(); + assert!(second.is_cancelled()); + assert!(nested.is_cancelled()); + assert!(parent.child().is_cancelled()); + } + + #[test] + fn cancellation_observation_is_local_and_survives_error_adapters() { + let token = ReadCancellation::new(); + let observed = ReadControl::new(Some(&token)); + let unrelated = ReadControl::new(Some(&token)); + token.clone().cancel(); + assert!(observed.check().is_err()); + assert_eq!( + observed + .finish::<()>(Err(Error::WrongPasswordOrCorruptData)) + .unwrap_err() + .kind(), + crate::ErrorKind::Cancelled + ); + assert!(matches!( + unrelated.finish::<()>(Err(Error::WrongPasswordOrCorruptData)), + Err(Error::WrongPasswordOrCorruptData) + )); + } + + #[test] + fn reader_bounds_reads_and_preserves_the_cancellation_sentinel() { + struct Source { + token: ReadCancellation, + calls: usize, + } + impl Read for Source { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + self.calls += 1; + assert!(bytes.len() <= 64 * 1024); + self.token.cancel(); + bytes.fill(42); + Ok(bytes.len()) + } + } + let token = ReadCancellation::new(); + let control = ReadControl::new(Some(&token)); + let mut source = Source { token, calls: 0 }; + let mut reader = control.reader(&mut source); + let mut bytes = vec![0; 128 * 1024]; + assert_eq!(reader.read(&mut bytes).unwrap(), 64 * 1024); + let err = Error::from(reader.read(&mut bytes).unwrap_err()); + assert_eq!(err.kind(), crate::ErrorKind::Cancelled); + assert_eq!(source.calls, 1); + } + + #[test] + fn polling_bounds_both_output_and_no_output_work() { + for output in [false, true] { + let token = ReadCancellation::new(); + let control = ReadControl::new(Some(&token)); + control.cancel_after_checks(1); + let mut poller = control.poller(); + poller.check(0).unwrap(); + if output { + assert!(poller.check(64 * 1024).is_err()); + } else { + for _ in 1..4096 { + poller.check(0).unwrap(); + } + assert!(poller.check(0).is_err()); + } + } + } + + #[test] + fn controlled_write_retries_interrupts_and_preserves_failures() { + struct ScriptedWriter { + replies: std::collections::VecDeque>, + written: Vec, + } + impl Write for ScriptedWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let count = self.replies.pop_front().expect("unexpected write")?; + self.written.extend_from_slice(&bytes[..count]); + Ok(count) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let token = ReadCancellation::new(); + let control = ReadControl::new(Some(&token)); + let mut writer = ScriptedWriter { + replies: [Err(io::ErrorKind::Interrupted.into()), Ok(1), Ok(2)].into(), + written: Vec::new(), + }; + control.write_all(&mut writer, b"abc").unwrap(); + assert_eq!(writer.written, b"abc"); + assert!(writer.replies.is_empty()); + + for (reply, kind) in [ + (Ok(0), io::ErrorKind::WriteZero), + ( + Err(io::ErrorKind::BrokenPipe.into()), + io::ErrorKind::BrokenPipe, + ), + ] { + let mut writer = ScriptedWriter { + replies: [reply].into(), + written: Vec::new(), + }; + assert_eq!( + control.write_all(&mut writer, b"x").unwrap_err().kind(), + kind + ); + assert!(writer.written.is_empty()); + } + + struct CancelAfterWrite(ReadCancellation); + impl Write for CancelAfterWrite { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.cancel(); + Ok(bytes.len().min(1)) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let error = control + .write_all(&mut CancelAfterWrite(token), b"abc") + .unwrap_err(); + assert_eq!(Error::from(error).kind(), crate::ErrorKind::Cancelled); + } +} diff --git a/arca-rar/vendor/rars/src/reader_scratch.rs b/arca-rar/vendor/rars/src/reader_scratch.rs new file mode 100644 index 00000000..a67ada11 --- /dev/null +++ b/arca-rar/vendor/rars/src/reader_scratch.rs @@ -0,0 +1,49 @@ +use std::path::PathBuf; + +/// Opt-in disk scratch for large RAR5/7 compressed members. +/// +/// Files are privately created in an existing caller-selected directory and +/// removed on drop (cleanup errors cannot be reported from Drop). Scratch may +/// contain decrypted data. The directory must remain trusted and available. +/// Bare WebAssembly rejects this disk policy; it never substitutes RAM storage. +/// +/// ```no_run +/// # fn extract(archive: &rars::Archive) -> rars::Result<()> { +/// let scratch = rars::Rar50Scratch::new("/path/to/private/scratch", 4 * 1024 * 1024 * 1024) +/// .with_filter_memory_limit(8 * 1024 * 1024); +/// let options = rars::ArchiveReadOptions::new() +/// .with_rar50_buffered_decode_limit(1024 * 1024) +/// .with_rar50_scratch(&scratch); +/// archive.extract_to_with_options(options, |_| Ok(Box::new(std::io::sink())))?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct Rar50Scratch { + pub(crate) directory: PathBuf, + pub(crate) max_bytes: u64, + pub(crate) filter_memory_limit: u64, +} + +impl Rar50Scratch { + /// Limits combined logical scratch-file lengths for one member, including + /// raw output, transformed output and filter records. Files are released + /// between members; filesystem allocation overhead is outside this quota. + /// Admission reserves room for twice the declared member size. Filter + /// records require an additional 24 bytes each as they are encountered. + pub fn new(directory: impl Into, max_bytes: u64) -> Self { + Self { + directory: directory.into(), + max_bytes, + filter_memory_limit: 8 * 1024 * 1024, + } + } + + /// Bounds temporary filter data (default 8 MiB). Each filter range requires + /// at most twice its length; a larger range is refused before allocation. + /// Dictionary/history, compressed blocks and fixed I/O buffers are separate. + pub fn with_filter_memory_limit(mut self, bytes: u64) -> Self { + self.filter_memory_limit = bytes; + self + } +} diff --git a/arca-rar/vendor/rars/src/reader_signature_tests.rs b/arca-rar/vendor/rars/src/reader_signature_tests.rs new file mode 100644 index 00000000..10ef9b66 --- /dev/null +++ b/arca-rar/vendor/rars/src/reader_signature_tests.rs @@ -0,0 +1,151 @@ +use super::*; +use std::io::{self, Cursor, Seek, SeekFrom}; + +struct Observed { + input: Cursor>, + bytes: usize, + refuse_after: Option, +} + +impl Read for Observed { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if self.refuse_after.is_some_and(|end| self.bytes >= end) { + return Err(io::ErrorKind::PermissionDenied.into()); + } + let count = output.len().min(3); + let count = self.input.read(&mut output[..count])?; + self.bytes += count; + Ok(count) + } +} + +#[test] +fn strong_standard_signatures_read_only_the_fixed_prefix() { + for signature in [detect::RAR15_SIGNATURE.as_slice(), detect::RAR50_SIGNATURE] { + let mut data = vec![42; 2 * 1024 * 1024]; + data[..signature.len()].copy_from_slice(signature); + let mut source = Observed { + input: Cursor::new(data), + bytes: 0, + refuse_after: Some(8), + }; + let actual = read_archive_signature(&mut source, 2 * 1024 * 1024) + .unwrap() + .unwrap(); + assert_eq!(actual, detect_archive_family(signature).unwrap()); + assert_eq!(source.bytes, 8); + } +} + +#[test] +fn signature_prefix_retains_full_scan_precedence_and_short_inputs() { + for bytes in [ + b"".as_slice(), + b"Rar!\x1a\x07", + b"Rar!\x1a\x07\x00", + b"Rar!\x1a\x07\x01\x00", + b"RE~^", + b"RE~^payload Rar!\x1a\x07\x01\x00later", + b"RE~^payload Rar!\x1a\x07\x00later", + b"stub Rar!\x1a\x07\x01\x00later", + b"stub RE~^payload", + b"stub RE~^ Rar!\x1a\x07\x00payload", + b"not an archive", + ] { + for len in 0..=bytes.len() { + let expected = find_archive_start(&bytes[..len], SFX_SCAN_LIMIT); + let actual = read_archive_signature(&mut &bytes[..len], len); + assert_eq!(actual.unwrap(), expected); + } + } +} + +#[test] +fn sfx_prefix_fallback_keeps_existing_scan_length_and_io_errors() { + let mut bytes = vec![0; 300 * 1024]; + bytes.extend_from_slice(detect::RAR50_SIGNATURE); + let mut source = Observed { + input: Cursor::new(bytes.clone()), + bytes: 0, + refuse_after: None, + }; + assert_eq!( + read_archive_signature(&mut source, bytes.len()).unwrap(), + find_archive_start(&bytes, SFX_SCAN_LIMIT) + ); + assert_eq!(source.bytes, bytes.len()); + source.input.set_position(0); + source.bytes = 0; + source.refuse_after = Some(8); + assert!(matches!( + read_archive_signature(&mut source, bytes.len()), + Err(Error::Io(_)) + )); +} + +#[test] +fn reader_prefix_retains_cancellation_io_error_and_eof_ordering() { + struct Callback { + input: Cursor>, + token: ReadCancellation, + fail: bool, + advertised_len: u64, + } + impl Read for Callback { + fn read(&mut self, output: &mut [u8]) -> io::Result { + let count = output.len().min(3); + let count = self.input.read(&mut output[..count])?; + if self.input.position() == 8 { + self.token.cancel(); + if self.fail { + return Err(io::ErrorKind::PermissionDenied.into()); + } + } + Ok(count) + } + } + impl Seek for Callback { + fn seek(&mut self, position: SeekFrom) -> io::Result { + match position { + SeekFrom::End(0) => self.input.seek(SeekFrom::Start(self.advertised_len)), + position => self.input.seek(position), + } + } + } + for fail in [false, true] { + let token = ReadCancellation::new(); + let reader = Callback { + input: Cursor::new(detect::RAR50_SIGNATURE.to_vec()), + token: token.clone(), + fail, + advertised_len: 32, + }; + let error = ArchiveReader::read_reader_with_options( + reader, + ArchiveReadOptions::new().with_cancellation(&token), + ) + .unwrap_err(); + assert!(token.is_cancelled()); + assert_eq!( + error.kind(), + if fail { + ErrorKind::Io + } else { + ErrorKind::Cancelled + } + ); + if fail { + assert!( + matches!(error, Error::Io(ref source) if source.kind == io::ErrorKind::PermissionDenied) + ); + } + } + let reader = Callback { + input: Cursor::new(b"Rar!".to_vec()), + token: ReadCancellation::new(), + fail: false, + advertised_len: 32, + }; + let error = ArchiveReader::read_reader(reader).unwrap_err(); + assert!(matches!(error, Error::Io(ref source) if source.kind == io::ErrorKind::UnexpectedEof)); +} diff --git a/arca-rar/vendor/rars/src/recovery/mod.rs b/arca-rar/vendor/rars/src/recovery/mod.rs new file mode 100644 index 00000000..1db76fcd --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/mod.rs @@ -0,0 +1,8 @@ +//! Recovery-record primitives shared by RAR writer and repair code. +//! +//! RAR 5 recovery data uses GF(2^16) with reduction polynomial `0x1100b` +//! and a Cauchy encoder matrix. This crate intentionally exposes the field +//! and matrix building blocks before wiring them into archive serialization. + +pub mod rar3; +pub mod rar5; diff --git a/arca-rar/vendor/rars/src/recovery/rar3.rs b/arca-rar/vendor/rars/src/recovery/rar3.rs new file mode 100644 index 00000000..08ab7325 --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar3.rs @@ -0,0 +1,9 @@ +//! Recovery diagnostics remain available without recovery execution. + +mod errors; +pub use errors::{Error, Result}; + +#[cfg(feature = "recovery")] +mod implementation; +#[cfg(feature = "recovery")] +pub use implementation::*; diff --git a/arca-rar/vendor/rars/src/recovery/rar3/errors.rs b/arca-rar/vendor/rars/src/recovery/rar3/errors.rs new file mode 100644 index 00000000..61c28def --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar3/errors.rs @@ -0,0 +1,25 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + InvalidParitySize, + InvalidCodewordSize, + TooManyErasures, + DecodeFailed, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidParitySize => f.write_str("RAR 3 recovery parity size is invalid"), + Self::InvalidCodewordSize => f.write_str("RAR 3 recovery codeword size is invalid"), + Self::TooManyErasures => { + f.write_str("RAR 3 recovery data cannot repair this many erasures") + } + Self::DecodeFailed => f.write_str("RAR 3 recovery decode failed"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; diff --git a/arca-rar/vendor/rars/src/recovery/rar3/implementation.rs b/arca-rar/vendor/rars/src/recovery/rar3/implementation.rs new file mode 100644 index 00000000..033f1993 --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar3/implementation.rs @@ -0,0 +1,482 @@ +use super::{Error, Result}; + +const MAX_PARITY: usize = 255; +const MAX_POLYNOMIAL: usize = 512; +const PRIMITIVE_POLYNOMIAL: u16 = 0x11d; + +#[derive(Debug, Clone)] +pub(crate) struct RSCoder8 { + parity_size: usize, + gf_exp: [u8; MAX_POLYNOMIAL], + gf_log: [u16; MAX_PARITY + 1], + generator: Vec, +} + +impl RSCoder8 { + pub(crate) fn new(parity_size: usize) -> Result { + if parity_size == 0 || parity_size > MAX_PARITY { + return Err(Error::InvalidParitySize); + } + let mut coder = Self { + parity_size, + gf_exp: [0; MAX_POLYNOMIAL], + gf_log: [0; MAX_PARITY + 1], + generator: vec![0; parity_size], + }; + coder.init_field(); + coder.init_generator(); + Ok(coder) + } + + #[cfg(test)] + fn encode(&self, data: &[u8]) -> Vec { + let mut shift = vec![0u8; self.parity_size + 1]; + for &byte in data { + let feedback = byte ^ shift[self.parity_size - 1]; + for index in (1..self.parity_size).rev() { + shift[index] = shift[index - 1] ^ self.mul(self.generator[index], feedback); + } + shift[0] = self.mul(self.generator[0], feedback); + } + (0..self.parity_size) + .map(|index| shift[self.parity_size - index - 1]) + .collect() + } + + pub(crate) fn correct_erasures(&self, codeword: &mut [u8], erasures: &[usize]) -> Result<()> { + if codeword.is_empty() || codeword.len() > MAX_PARITY { + return Err(Error::InvalidCodewordSize); + } + if erasures.len() > self.parity_size { + return Err(Error::TooManyErasures); + } + if erasures.iter().any(|&index| index >= codeword.len()) { + return Err(Error::InvalidCodewordSize); + } + + let mut syndromes = vec![0u8; self.parity_size]; + let mut all_zero = true; + for (index, syndrome) in syndromes.iter_mut().enumerate() { + let factor = self.gf_exp[index + 1]; + let mut sum = 0; + for &byte in codeword.iter() { + sum = byte ^ self.mul(factor, sum); + } + *syndrome = sum; + all_zero &= sum == 0; + } + if all_zero { + return Ok(()); + } + if erasures.is_empty() { + return Err(Error::DecodeFailed); + } + + let mut locator = vec![0u8; self.parity_size + 1]; + locator[0] = 1; + for &erasure in erasures { + let multiplier = self.gf_exp[codeword.len() - erasure - 1]; + for index in (1..=self.parity_size).rev() { + locator[index] ^= self.mul(multiplier, locator[index - 1]); + } + } + + let mut error_locs = Vec::new(); + let mut denominators = Vec::new(); + for root in (MAX_PARITY - codeword.len())..=MAX_PARITY { + let mut sum = 0; + for (power, &coefficient) in locator.iter().enumerate() { + sum ^= self.mul(self.gf_exp[(power * root) % MAX_PARITY], coefficient); + } + if sum == 0 { + let loc = MAX_PARITY - root; + error_locs.push(loc); + let mut denominator = 0; + for index in (1..=self.parity_size).step_by(2) { + denominator ^= self.mul( + locator[index], + self.gf_exp[(root * (index - 1)) % MAX_PARITY], + ); + } + denominators.push(denominator); + } + } + // Each validated erasure supplies a root in this range. The locator's + // constant term is one, so it has at most parity_size distinct roots. + let evaluator = self.multiply_polynomials(&locator, &syndromes); + for (&loc, &denominator) in error_locs.iter().zip(&denominators) { + if denominator == 0 { + return Err(Error::DecodeFailed); + } + let data_pos = codeword + .len() + .checked_sub(loc + 1) + .ok_or(Error::DecodeFailed)?; + let dloc = MAX_PARITY - loc; + let mut numerator = 0; + for (index, &coefficient) in evaluator.iter().enumerate() { + numerator ^= self.mul(coefficient, self.gf_exp[(dloc * index) % MAX_PARITY]); + } + let correction = self.mul( + numerator, + self.gf_exp[MAX_PARITY - usize::from(self.gf_log[denominator as usize])], + ); + codeword[data_pos] ^= correction; + } + Ok(()) + } + + fn init_field(&mut self) { + let mut value = 1u16; + for index in 0..MAX_PARITY { + self.gf_log[value as usize] = index as u16; + self.gf_exp[index] = value as u8; + value <<= 1; + if value > 0xff { + value ^= PRIMITIVE_POLYNOMIAL; + } + } + for index in MAX_PARITY..MAX_POLYNOMIAL { + self.gf_exp[index] = self.gf_exp[index - MAX_PARITY]; + } + } + + fn init_generator(&mut self) { + let mut current = vec![0u8; self.parity_size]; + current[0] = 1; + for index in 1..=self.parity_size { + let mut factor = vec![0u8; self.parity_size]; + factor[0] = self.gf_exp[index]; + if self.parity_size > 1 { + factor[1] = 1; + } + self.generator = self.multiply_polynomials(&factor, ¤t); + current.clone_from(&self.generator); + } + } + + fn multiply_polynomials(&self, left: &[u8], right: &[u8]) -> Vec { + let mut out = vec![0u8; self.parity_size]; + for left_index in 0..self.parity_size { + if left.get(left_index).copied().unwrap_or(0) == 0 { + continue; + } + for right_index in 0..(self.parity_size - left_index) { + out[left_index + right_index] ^= self.mul( + left[left_index], + right.get(right_index).copied().unwrap_or(0), + ); + } + } + out + } + + fn mul(&self, left: u8, right: u8) -> u8 { + if left == 0 || right == 0 { + 0 + } else { + self.gf_exp[usize::from(self.gf_log[left as usize] + self.gf_log[right as usize])] + } + } +} + +pub fn reconstruct_data_volumes( + data_volumes: &[Option<&[u8]>], + recovery_count: usize, + recovery_volumes: &[(usize, &[u8])], +) -> Result>> { + if data_volumes.is_empty() || data_volumes.len() + recovery_count > MAX_PARITY { + return Err(Error::InvalidCodewordSize); + } + if recovery_volumes.is_empty() || recovery_count == 0 { + return Err(Error::InvalidParitySize); + } + let shard_len = recovery_volumes[0].1.len(); + if recovery_volumes + .iter() + .any(|&(index, data)| index >= recovery_count || data.len() != shard_len) + { + return Err(Error::InvalidCodewordSize); + } + if data_volumes + .iter() + .flatten() + .any(|data| data.len() > shard_len) + { + return Err(Error::InvalidCodewordSize); + } + + let mut recovery_by_index = vec![None; recovery_count]; + for &(index, data) in recovery_volumes { + if recovery_by_index[index].replace(data).is_some() { + return Err(Error::InvalidCodewordSize); + } + } + + let missing_data: Vec<_> = data_volumes + .iter() + .enumerate() + .filter_map(|(index, data)| data.is_none().then_some(index)) + .collect(); + if missing_data.is_empty() { + return Ok(data_volumes + .iter() + .map(|data| { + // No missing data volumes were found above. + let data = data.expect("every data volume is present"); + let mut out = vec![0; shard_len]; + out[..data.len()].copy_from_slice(data); + out + }) + .collect()); + } + + let missing_recovery: Vec<_> = recovery_by_index + .iter() + .enumerate() + .filter_map(|(index, data)| data.is_none().then_some(data_volumes.len() + index)) + .collect(); + let mut erasures = missing_data.clone(); + erasures.extend(missing_recovery); + if erasures.len() > recovery_count { + return Err(Error::TooManyErasures); + } + + let coder = RSCoder8::new(recovery_count)?; + let mut out: Vec> = data_volumes + .iter() + .map(|data| { + let mut shard = vec![0; shard_len]; + if let Some(data) = data { + shard[..data.len()].copy_from_slice(data); + } + shard + }) + .collect(); + + for offset in 0..shard_len { + let mut codeword = vec![0; data_volumes.len() + recovery_count]; + for (index, data) in data_volumes.iter().enumerate() { + if let Some(data) = data { + codeword[index] = data.get(offset).copied().unwrap_or(0); + } + } + for (index, data) in recovery_by_index.iter().enumerate() { + if let Some(data) = data { + codeword[data_volumes.len() + index] = data[offset]; + } + } + coder.correct_erasures(&mut codeword, &erasures)?; + for &index in &missing_data { + out[index][offset] = codeword[index]; + } + } + + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::{reconstruct_data_volumes, Error, RSCoder8}; + + #[test] + fn rs8_encoder_matches_unrar_generator_shape() { + let coder = RSCoder8::new(11).unwrap(); + assert_eq!( + coder.generator, + vec![97, 180, 203, 151, 195, 196, 219, 7, 113, 50, 69] + ); + } + + #[test] + fn rs8_reconstructs_single_erased_data_symbol() { + let coder = RSCoder8::new(4).unwrap(); + let data = b"rar recovery data"; + let parity = coder.encode(data); + let mut codeword = [data.as_slice(), parity.as_slice()].concat(); + let original = codeword.clone(); + codeword[3] ^= 0xa5; + + coder.correct_erasures(&mut codeword, &[3]).unwrap(); + + assert_eq!(codeword, original); + } + + #[test] + fn rs8_reconstructs_multiple_erased_symbols_including_parity() { + let coder = RSCoder8::new(5).unwrap(); + let data = b"rar3-rs8"; + let parity = coder.encode(data); + let mut codeword = [data.as_slice(), parity.as_slice()].concat(); + let original = codeword.clone(); + codeword[1] = 0; + codeword[7] = 0; + codeword[10] = 0; + + coder.correct_erasures(&mut codeword, &[1, 7, 10]).unwrap(); + + assert_eq!(codeword, original); + } + + #[test] + fn rs8_rejects_more_erasures_than_parity_symbols() { + let coder = RSCoder8::new(2).unwrap(); + let mut codeword = b"abcde".to_vec(); + + assert_eq!( + coder.correct_erasures(&mut codeword, &[0, 1, 2]), + Err(Error::TooManyErasures) + ); + } + + #[test] + fn rs8_rejects_invalid_parameters_and_corrupt_unmarked_codewords() { + assert_eq!(RSCoder8::new(0).unwrap_err(), Error::InvalidParitySize); + assert_eq!(RSCoder8::new(256).unwrap_err(), Error::InvalidParitySize); + + let coder = RSCoder8::new(1).unwrap(); + assert_eq!( + coder.correct_erasures(&mut [], &[]), + Err(Error::InvalidCodewordSize) + ); + assert_eq!( + coder.correct_erasures(&mut [0; 256], &[]), + Err(Error::InvalidCodewordSize) + ); + assert_eq!( + coder.correct_erasures(&mut [0; 2], &[2]), + Err(Error::InvalidCodewordSize) + ); + assert_eq!(coder.correct_erasures(&mut [0; 2], &[]), Ok(())); + assert_eq!( + coder.correct_erasures(&mut [1, 0], &[]), + Err(Error::DecodeFailed) + ); + } + + #[test] + fn rs8_reports_each_failure_category() { + for (error, message) in [ + ( + Error::InvalidParitySize, + "RAR 3 recovery parity size is invalid", + ), + ( + Error::InvalidCodewordSize, + "RAR 3 recovery codeword size is invalid", + ), + ( + Error::TooManyErasures, + "RAR 3 recovery data cannot repair this many erasures", + ), + (Error::DecodeFailed, "RAR 3 recovery decode failed"), + ] { + assert_eq!(error.to_string(), message); + } + } + + #[test] + fn rs8_rejects_duplicate_erasure_positions() { + for parity in 2..=4 { + let coder = RSCoder8::new(parity).unwrap(); + for codeword_len in 2..=8 { + for index in 0..codeword_len { + let mut codeword = vec![0; codeword_len]; + codeword[0] = 1; + assert!( + coder + .correct_erasures(&mut codeword, &[index, index]) + .is_err(), + "parity={parity}, length={codeword_len}, index={index}" + ); + } + } + } + } + + #[test] + fn rev3_rejects_invalid_shard_geometry_and_duplicate_recovery_indices() { + let present = [1u8, 2]; + let recovery = [3u8, 4]; + type Case<'a> = ( + &'a [Option<&'a [u8]>], + usize, + &'a [(usize, &'a [u8])], + Error, + ); + let cases: &[Case<'_>] = &[ + (&[], 1, &[(0, &recovery)], Error::InvalidCodewordSize), + (&[None], 255, &[(0, &recovery)], Error::InvalidCodewordSize), + (&[None], 0, &[(0, &recovery)], Error::InvalidParitySize), + (&[None], 1, &[], Error::InvalidParitySize), + (&[None], 1, &[(1, &recovery)], Error::InvalidCodewordSize), + ( + &[None], + 1, + &[(0, &recovery), (0, &present[..1])], + Error::InvalidCodewordSize, + ), + ( + &[Some(&present)], + 1, + &[(0, &recovery[..1])], + Error::InvalidCodewordSize, + ), + ( + &[None], + 1, + &[(0, &recovery), (0, &recovery)], + Error::InvalidCodewordSize, + ), + (&[None, None], 1, &[(0, &recovery)], Error::TooManyErasures), + ]; + for &(data, count, shards, ref expected) in cases { + assert_eq!( + reconstruct_data_volumes(data, count, shards), + Err(expected.clone()) + ); + } + } + + #[test] + fn rev3_returns_healthy_padded_data_without_decoding() { + let actual = b"a"; + let recovery = b"rs"; + assert_eq!( + reconstruct_data_volumes(&[Some(actual)], 1, &[(0, recovery)]), + Ok(vec![b"a\0".to_vec()]) + ); + } + + #[test] + fn rev3_reconstructs_missing_data_volume_from_recovery_volume() { + let data = [ + b"volume-one".as_slice(), + b"volume-two".as_slice(), + b"volume-three".as_slice(), + ]; + let recovery_count = 2; + let coder = RSCoder8::new(recovery_count).unwrap(); + let shard_len = data.iter().map(|shard| shard.len()).max().unwrap(); + let mut recovery = vec![vec![0; shard_len]; recovery_count]; + for offset in 0..shard_len { + let column: Vec<_> = data + .iter() + .map(|shard| shard.get(offset).copied().unwrap_or(0)) + .collect(); + let encoded = coder.encode(&column); + for (row, byte) in recovery.iter_mut().zip(encoded) { + row[offset] = byte; + } + } + + let repaired = reconstruct_data_volumes( + &[Some(data[0]), None, Some(data[2])], + recovery_count, + &[(0, recovery[0].as_slice())], + ) + .unwrap(); + + assert_eq!(&repaired[1][..data[1].len()], data[1]); + } +} diff --git a/arca-rar/vendor/rars/src/recovery/rar5.rs b/arca-rar/vendor/rars/src/recovery/rar5.rs new file mode 100644 index 00000000..5c3ff0e1 --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar5.rs @@ -0,0 +1,12 @@ +//! Recovery diagnostics remain available without recovery execution. + +mod checksum; +pub use checksum::{crc64_rar_state, crc64_xz}; + +mod errors; +pub use errors::{Error, Result}; + +#[cfg(feature = "recovery")] +mod implementation; +#[cfg(feature = "recovery")] +pub use implementation::*; diff --git a/arca-rar/vendor/rars/src/recovery/rar5/checksum.rs b/arca-rar/vendor/rars/src/recovery/rar5/checksum.rs new file mode 100644 index 00000000..947b1968 --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar5/checksum.rs @@ -0,0 +1,24 @@ +//! Checksum utilities independent of recovery generation and repair. + +const CRC64_XZ_POLY: u64 = 0xc96c_5795_d787_0f42; +pub(super) const CRC64_XZ_INIT: u64 = 0xffff_ffff_ffff_ffff; + +pub fn crc64_xz(data: &[u8]) -> u64 { + crc64_update(data, CRC64_XZ_INIT) ^ CRC64_XZ_INIT +} + +pub(super) fn crc64_update(data: &[u8], initial: u64) -> u64 { + let mut crc = initial; + for &byte in data { + crc ^= byte as u64; + for _ in 0..8 { + let mask = 0u64.wrapping_sub(crc & 1); + crc = (crc >> 1) ^ (CRC64_XZ_POLY & mask); + } + } + crc +} + +pub fn crc64_rar_state(data: &[u8]) -> u64 { + crc64_update(data, 0) +} diff --git a/arca-rar/vendor/rars/src/recovery/rar5/errors.rs b/arca-rar/vendor/rars/src/recovery/rar5/errors.rs new file mode 100644 index 00000000..79ddcf3b --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar5/errors.rs @@ -0,0 +1,79 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + Cancelled, + BadRecoveryChunk, + OddShardSize, + PlanOverflow, + PrefixExceedsPlan, + TooManyDamagedShards, + ShardSizeMismatch, + TooManyShards, + SingularElement, + RebuildTooLarge, + /// A repair solve, padded reconstruction or record rebuild exceeds its + /// workspace admission ceiling, which scales with the bytes supplied. + /// This is not a process-RAM quota. + RepairWorkspaceLimitExceeded { + limit: u64, + required: u64, + }, + Io(std::io::ErrorKind), + /// Preserve a structured archive error from workspace admission, a reader or a sink. + TypedIo(Box), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Cancelled => f.write_str("RAR 5 recovery cancelled"), + Self::BadRecoveryChunk => f.write_str("RAR 5 recovery chunk is invalid"), + Self::OddShardSize => f.write_str("RAR 5 recovery shard size is odd"), + Self::PlanOverflow => f.write_str("RAR 5 recovery plan overflows"), + Self::PrefixExceedsPlan => { + f.write_str("RAR 5 recovery prefix exceeds planned shard capacity") + } + Self::TooManyDamagedShards => { + f.write_str("RAR 5 recovery data cannot repair this many damaged shards") + } + Self::ShardSizeMismatch => f.write_str("RAR 5 recovery shard sizes differ"), + Self::TooManyShards => f.write_str("RAR 5 recovery shard count is invalid"), + Self::SingularElement => f.write_str("RAR 5 recovery matrix is singular"), + Self::RebuildTooLarge => { + f.write_str("RAR 5 recovery record is too large to rebuild in memory") + } + Self::RepairWorkspaceLimitExceeded { limit, required } => write!( + f, + "RAR 5 recovery repair workspace limit {limit} bytes exceeded (requires {required})" + ), + Self::Io(kind) => write!(f, "RAR 5 recovery I/O failed: {kind}"), + Self::TypedIo(error) => std::fmt::Display::fmt(error, f), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::TypedIo(error) => Some(error.as_ref()), + _ => None, + } + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + match error.downcast::() { + Ok(error) => Self::TypedIo(Box::new(error)), + Err(error) => Self::Io(error.kind()), + } + } +} + +impl From for Error { + fn from(error: crate::codec::Error) -> Self { + Self::TypedIo(Box::new(error.into())) + } +} + +pub type Result = std::result::Result; diff --git a/arca-rar/vendor/rars/src/recovery/rar5/implementation.rs b/arca-rar/vendor/rars/src/recovery/rar5/implementation.rs new file mode 100644 index 00000000..7e73f50e --- /dev/null +++ b/arca-rar/vendor/rars/src/recovery/rar5/implementation.rs @@ -0,0 +1,4789 @@ +use super::checksum::{crc64_update, CRC64_XZ_INIT}; +#[cfg(test)] +use super::{crc64_rar_state, crc64_xz}; +use super::{Error, Result}; + +use crate::codec::workspace::{Allowance, Budget, Buffer}; +const FIELD_SIZE: usize = 65_535; +const FIELD_MASK: u32 = 0xffff; +const PRIMITIVE_POLYNOMIAL: u32 = 0x1100b; +const ZERO_LOG_SENTINEL: u32 = (FIELD_SIZE * 2) as u32; +const MAX_WINRAR602_DATA_SHARDS: u64 = 200; +const KIB: u64 = 1024; +const RAR5_RECOVERY_CHUNK_FIXED_HEADER_SIZE: u64 = 0x48; +/// Resident parity a record rebuild may allocate. The archive itself is +/// already in memory by the time a rebuild starts, so this bounds the extra. +const MAX_REBUILD_PARITY_BYTES: u64 = 256 * 1024 * 1024; +/// Solve/reconstruction allocations and resident rebuild workspace a repair +/// may admit however small its input. Input and parsed record storage, shared +/// GF tables, final collected archive output, allocator overhead and control +/// storage are outside this bound. +const REPAIR_WORKSPACE_FLOOR_BYTES: u64 = 256 * 1024 * 1024; +/// Genuine geometry needs about twice the bytes it was given: damaged shards +/// are bounded by the parity supplied for them, and padding by the data. Only +/// a record declaring far more than it carries can exceed this multiple. +const REPAIR_WORKSPACE_INPUT_FACTOR: u64 = 4; + +fn repair_workspace_limit(input_bytes: u64) -> u64 { + REPAIR_WORKSPACE_FLOOR_BYTES.max(input_bytes.saturating_mul(REPAIR_WORKSPACE_INPUT_FACTOR)) +} + +fn repair_workspace_sum(parts: &[u64]) -> Result { + parts.iter().try_fold(0u64, |total, &bytes| { + total.checked_add(bytes).ok_or(Error::PlanOverflow) + }) +} + +fn repair_workspace_product(count: u64, width: u64) -> Result { + count.checked_mul(width).ok_or(Error::PlanOverflow) +} + +fn admit_repair_workspace(required: u64, input_bytes: u64) -> Result<()> { + admit_repair_workspace_with_limit(required, repair_workspace_limit(input_bytes)) +} + +fn admit_repair_workspace_with_limit(required: u64, limit: u64) -> Result<()> { + if required > limit { + return Err(Error::RepairWorkspaceLimitExceeded { limit, required }); + } + Ok(()) +} + +/// Count a conservative sum, rather than requiring individual vectors to fit. +/// Inversion holds the original equations, a working copy and the inverse at +/// once. RHS rows, pivot copies and per-word scratch are counted too, together +/// with returned shard buffers and their descriptors. Metadata is small but +/// counted using the maximum growth of the damaged-index vector. +fn repair_solve_workspace_required( + data_count: usize, + damaged_count: usize, + shard_len: usize, + output_bytes: u64, + read_scratch: usize, +) -> Result { + let data_count = data_count as u64; + let damaged_count = damaged_count as u64; + let matrix_cells = repair_workspace_product(damaged_count, damaged_count)?; + let row_descriptors = (6 * std::mem::size_of::>() + + std::mem::size_of::<(std::ops::Range, Vec)>() + + 4 * std::mem::size_of::()) as u64; + let metadata = (std::mem::size_of::>() + + 2 * std::mem::size_of::() + + std::mem::size_of::()) as u64; + repair_workspace_sum(&[ + repair_workspace_product(matrix_cells, 3 * std::mem::size_of::() as u64)?, + // Per-row coefficients against every data column. + repair_workspace_product( + repair_workspace_product(damaged_count, data_count)?, + std::mem::size_of::() as u64, + )?, + repair_workspace_product(damaged_count, shard_len as u64)?, + repair_workspace_product(damaged_count, row_descriptors)?, + repair_workspace_product(data_count, metadata)?, + output_bytes, + read_scratch as u64, + ]) +} + +use crate::progress::ProgressReporter; +use crate::{WriteOperation, WriteProgressEvent}; + +fn shared_gf16() -> &'static Gf16 { + static GF16: std::sync::OnceLock = std::sync::OnceLock::new(); + GF16.get_or_init(Gf16::new) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct InlineRecoveryPlan { + pub data_shards: u64, + pub recovery_shards: u64, + pub group_count: u64, + pub header_size: u64, + pub shard_size: u64, +} + +impl InlineRecoveryPlan { + pub fn payload_size(self) -> Result { + self.recovery_shards + .checked_mul(self.shard_size) + .ok_or(Error::PlanOverflow) + } +} + +pub fn plan_inline_recovery( + archive_size: u64, + recovery_percent: u64, +) -> Result { + let pct = recovery_percent.min(100); + let data_shards = if archive_size >= 200 * KIB { + MAX_WINRAR602_DATA_SHARDS + } else { + archive_size.div_ceil(KIB).max(1) + }; + let mut recovery_shards = (2 * pct * data_shards) / 200; + recovery_shards = recovery_shards.min(data_shards); + if recovery_shards == 0 && archive_size < 200 * KIB { + recovery_shards = 1; + } + let mut group_count = archive_size.div_ceil(data_shards); + group_count += group_count & 1; + // data_shards is capped at 200 and group_count is at most + // ceil(size / 200) + 1 for large inputs, so both additions fit at u64::MAX. + let header_size = data_shards * 8 + RAR5_RECOVERY_CHUNK_FIXED_HEADER_SIZE; + let shard_size = header_size + group_count; + + Ok(InlineRecoveryPlan { + data_shards, + recovery_shards, + group_count, + header_size, + shard_size, + }) +} + +pub fn split_prefix_shard_ranges( + prefix_len: usize, + plan: InlineRecoveryPlan, +) -> Result>> { + let data_shards = usize::try_from(plan.data_shards).map_err(|_| Error::PlanOverflow)?; + let group_count = usize::try_from(plan.group_count).map_err(|_| Error::PlanOverflow)?; + let capacity = data_shards + .checked_mul(group_count) + .ok_or(Error::PlanOverflow)?; + if prefix_len > capacity { + return Err(Error::PrefixExceedsPlan); + } + + let mut ranges = Vec::with_capacity(data_shards); + for shard_index in 0..data_shards { + // Every shard index is below data_shards, whose full product with + // group_count was admitted above. + let start = (shard_index * group_count).min(prefix_len); + let end = start.saturating_add(group_count).min(prefix_len); + ranges.push(start..end); + } + Ok(ranges) +} + +pub fn split_prefix_shards(prefix: &[u8], plan: InlineRecoveryPlan) -> Result>> { + let group_count = usize::try_from(plan.group_count).map_err(|_| Error::PlanOverflow)?; + let ranges = split_prefix_shard_ranges(prefix.len(), plan)?; + let mut shards = Vec::with_capacity(ranges.len()); + for range in ranges { + let mut shard = vec![0u8; group_count]; + if range.start < range.end { + shard[..range.end - range.start].copy_from_slice(&prefix[range]); + } + shards.push(shard); + } + Ok(shards) +} + +pub fn encode_inline_recovery_parity( + archive_prefix: &[u8], + recovery_percent: u64, +) -> Result<(InlineRecoveryPlan, Vec>)> { + let plan = plan_inline_recovery(archive_prefix.len() as u64, recovery_percent)?; + let shards = split_prefix_shards(archive_prefix, plan)?; + let shard_refs: Vec<&[u8]> = shards.iter().map(Vec::as_slice).collect(); + // Planning caps recovery_shards at 200. + let parity = encode_parity_shards(&shard_refs, plan.recovery_shards as usize)?; + Ok((plan, parity)) +} + +pub fn build_structural_inline_recovery_data( + archive_prefix: &[u8], + recovery_percent: u64, +) -> Result> { + let mut out = Vec::new(); + build_streamed_inline_recovery( + &mut std::io::Cursor::new(archive_prefix), + archive_prefix.len() as u64, + recovery_percent, + RecoveryMemoryMode::Resident, + None, + &mut out, + None, + 1, + )?; + Ok(out) +} + +/// Builds recovery payload bytes with the geometry of `plan`, so a record +/// rebuilt during repair keeps the shape the archive was written with. +#[cfg(test)] +pub(crate) fn build_inline_recovery_data_for_plan( + archive_prefix: &[u8], + plan: InlineRecoveryPlan, +) -> Result> { + build_inline_recovery_data_for_plan_with_control( + archive_prefix, + plan, + &crate::read_control::ReadControl::default(), + ) +} + +pub(crate) fn build_inline_recovery_data_for_plan_with_control( + archive_prefix: &[u8], + plan: InlineRecoveryPlan, + control: &crate::read_control::ReadControl, +) -> Result> { + check_repair(control)?; + let mut out = Vec::new(); + build_streamed_inline_recovery_for_plan( + &mut control.reader(std::io::Cursor::new(archive_prefix)), + archive_prefix.len() as u64, + plan, + RecoveryMemoryMode::Resident, + None, + &mut out, + None, + 1, + ) + .map_err(|error| { + if matches!( + control.finish::<()>(Err(error.clone().into())), + Err(crate::Error::Cancelled) + ) { + Error::Cancelled + } else { + error + } + })?; + check_repair(control)?; + Ok(out) +} + +/// Read buffer size for streaming recovery passes. Even, so words never +/// straddle a chunk boundary. +pub(crate) const RECOVERY_IO_BLOCK: usize = 256 * 1024; +/// Striped mode never claims more than this, however large the budget is. +#[cfg(any(test, feature = "write"))] +const STRIPE_BUDGET_CAP: u64 = 64 * 1024 * 1024; +/// Below this a stripe pass seeks far more than it reads, so rather than +/// thrash we report what striping would actually cost and let the caller +/// refuse the job. +#[cfg(any(test, feature = "write"))] +const MIN_STRIPE_LEN: u64 = 4 * 1024; + +/// How a recovery pass holds parity while it works. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RecoveryMemoryMode { + /// Every parity row stays in memory and the body is read straight + /// through. Needs `recovery_shards * group_count` bytes. + Resident, + /// Parity is built one column stripe at a time and spilled to scratch + /// storage, so memory is bounded regardless of archive size. Costs one + /// seek per data shard per stripe. + // Reader repair uses resident parity; writers select bounded stripes. + #[cfg_attr(not(any(test, feature = "write")), expect(dead_code))] + Striped { stripe_len: usize }, +} + +/// Picks a memory mode for `plan` under `memory_limit`, returning the mode and +/// the number of bytes the caller should reserve for it. +#[cfg(any(test, feature = "write"))] +pub(crate) fn choose_recovery_memory_mode( + plan: InlineRecoveryPlan, + memory_limit: u64, +) -> Result<(RecoveryMemoryMode, u64)> { + let parity_bytes = plan + .recovery_shards + .checked_mul(plan.group_count) + .ok_or(Error::PlanOverflow)?; + let resident_bytes = parity_bytes.saturating_add(RECOVERY_IO_BLOCK as u64); + if resident_bytes <= memory_limit { + return Ok((RecoveryMemoryMode::Resident, resident_bytes)); + } + + // One buffer per parity row plus one for the data being read. + let rows = plan.recovery_shards.saturating_add(1).max(1); + let budget = memory_limit.min(STRIPE_BUDGET_CAP); + let mut stripe_len = (budget / rows).min(plan.group_count) & !1; + if stripe_len < MIN_STRIPE_LEN { + stripe_len = MIN_STRIPE_LEN.min(plan.group_count.max(2)); + } + stripe_len = stripe_len.max(2); + // The budget caps this width at 64 MiB on every supported host. + let stripe_len_usize = stripe_len as usize; + let required = rows.saturating_mul(stripe_len); + Ok(( + RecoveryMemoryMode::Striped { + stripe_len: stripe_len_usize, + }, + required, + )) +} + +/// Choose a phase geometry from actual managed capacity as well as the legacy +/// workspace policy. Both parity construction and later chunk framing must fit. +#[cfg(any(test, feature = "write"))] +pub(crate) fn choose_recovery_capacity_mode( + plan: InlineRecoveryPlan, + memory_limit: u64, + allowance: &crate::codec::workspace::Limited, +) -> Result<(RecoveryMemoryMode, u64)> { + use crate::codec::workspace::Limited; + let field = ((FIELD_SIZE * 4 + 1) * 2 + (FIELD_SIZE + 1) * 4) as u64; + let rows = plan.recovery_shards; + let descriptor = std::mem::size_of::>() as u64; + let fixed = field + .saturating_add(rows.saturating_mul(descriptor)) + .saturating_add(rows.saturating_mul(plan.data_shards).saturating_mul(2)) + .saturating_add(plan.data_shards.saturating_mul(8)); + let io = plan.group_count.max(1).min(RECOVERY_IO_BLOCK as u64); + let framing = fixed.saturating_add(io).saturating_add(plan.header_size); + let parity_descriptors = rows.saturating_mul(descriptor); + let resident = framing + .saturating_add(parity_descriptors) + .saturating_add(rows.saturating_mul(plan.group_count)); + let (legacy_mode, legacy_required) = choose_recovery_memory_mode(plan, memory_limit)?; + if matches!(legacy_mode, RecoveryMemoryMode::Resident) && resident <= allowance.available() { + return Ok((RecoveryMemoryMode::Resident, legacy_required)); + } + let stripe_fixed = fixed.saturating_add(parity_descriptors); + let minimum = + framing.max(stripe_fixed.saturating_add(rows.saturating_add(1).saturating_mul(2))); + allowance.check_capacity(minimum)?; + let width = allowance + .available() + .saturating_sub(stripe_fixed) + .min(memory_limit.min(STRIPE_BUDGET_CAP)) + / rows.saturating_add(1); + let stripe_len = width.min(plan.group_count).max(2) & !1; + Ok(( + RecoveryMemoryMode::Striped { + stripe_len: stripe_len as usize, + }, + rows.saturating_add(1).saturating_mul(stripe_len), + )) +} + +/// What a streaming recovery pass produced, so the caller can frame the +/// service block around the payload it just wrote. +#[derive(Debug, Clone, Copy)] +pub(crate) struct StreamedRecoveryOutput { + #[cfg_attr(not(any(test, feature = "write")), expect(dead_code))] + pub(crate) plan: InlineRecoveryPlan, + #[cfg_attr(not(any(test, feature = "write")), expect(dead_code))] + pub(crate) payload_len: u64, + #[cfg_attr(not(any(test, feature = "write")), expect(dead_code))] + pub(crate) payload_crc32: u32, +} + +pub(crate) trait ReadSeek: std::io::Read + std::io::Seek {} +impl ReadSeek for T {} + +pub(crate) trait ReadWriteSeek: std::io::Read + std::io::Write + std::io::Seek {} +impl ReadWriteSeek for T {} + +/// Builds the inline recovery payload for `body` and writes it to `sink`. +/// +/// The body is read exactly once to build parity, then each parity row is +/// framed into a `{RB}` chunk. Output is byte-identical to computing the whole +/// thing in memory; only the working set differs. +/// +/// `parity_scratch` must be supplied for [`RecoveryMemoryMode::Striped`] and +/// is where partial parity lives between stripes. +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_streamed_inline_recovery( + body: &mut dyn ReadSeek, + body_len: u64, + recovery_percent: u64, + mode: RecoveryMemoryMode, + parity_scratch: Option<&mut dyn ReadWriteSeek>, + sink: &mut dyn std::io::Write, + progress: Option>, + pass: usize, +) -> Result { + let plan = plan_inline_recovery(body_len, recovery_percent)?; + build_streamed_inline_recovery_for_plan( + body, + body_len, + plan, + mode, + parity_scratch, + sink, + progress, + pass, + ) +} + +/// Same as [`build_streamed_inline_recovery`] with the geometry supplied +/// rather than derived from a percentage. +/// +/// Repair needs this: WinRAR accepts recovery percentages up to 1000%, which +/// [`plan_inline_recovery`] never produces, so a record rebuilt from surviving +/// chunks has to take its geometry from those chunks. +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_streamed_inline_recovery_for_plan( + body: &mut dyn ReadSeek, + body_len: u64, + plan: InlineRecoveryPlan, + mode: RecoveryMemoryMode, + parity_scratch: Option<&mut dyn ReadWriteSeek>, + sink: &mut dyn std::io::Write, + progress: Option>, + pass: usize, +) -> Result { + streamed_recovery_with_allowance( + body, + body_len, + plan, + mode, + parity_scratch, + sink, + progress, + pass, + &Allowance::default(), + ) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn streamed_recovery_with_allowance( + body: &mut dyn ReadSeek, + body_len: u64, + plan: InlineRecoveryPlan, + mode: RecoveryMemoryMode, + parity_scratch: Option<&mut dyn ReadWriteSeek>, + sink: &mut dyn std::io::Write, + progress: Option>, + pass: usize, + allowance: &B, +) -> Result { + check_recovery_cancelled(progress)?; + let payload_len = plan.payload_size()?; + if plan.header_size + != RAR5_RECOVERY_CHUNK_FIXED_HEADER_SIZE + .checked_add(plan.data_shards.checked_mul(8).ok_or(Error::PlanOverflow)?) + .ok_or(Error::PlanOverflow)? + { + return Err(Error::PlanOverflow); + } + if plan.shard_size + != plan + .header_size + .checked_add(plan.group_count) + .ok_or(Error::PlanOverflow)? + { + return Err(Error::PlanOverflow); + } + + // Fail on unrepresentable geometry before doing any work. + let total_size = u32::try_from(plan.shard_size).map_err(|_| Error::PlanOverflow)?; + // shard_size >= header_size was checked above, and shard_size fits u32. + let header_size_u32 = plan.header_size as u32; + let data_shards_u16 = u16::try_from(plan.data_shards).map_err(|_| Error::PlanOverflow)?; + let recovery_shards_u16 = + u16::try_from(plan.recovery_shards).map_err(|_| Error::PlanOverflow)?; + // Both shard counts fit u16; group_count and header_size are bounded by + // the admitted u32 shard_size. All four fit native usize, including i386. + let data_shards = plan.data_shards as usize; + let recovery_shards = plan.recovery_shards as usize; + let group_count = plan.group_count as usize; + let header_size = plan.header_size as usize; + if body_len > plan.group_count.saturating_mul(plan.data_shards) { + return Err(Error::PrefixExceedsPlan); + } + + if let Some(progress) = progress { + progress.report(WriteProgressEvent::OperationStarted { + operation: WriteOperation::Recovery, + total_bytes: Some(payload_len), + total_entries: None, + pass, + }); + } + + // Phase A budget is the parity bytes; phase B adds the chunk headers. + // group_count <= shard_size and payload_size admitted the product of + // recovery_shards and shard_size above. + let encode_units = plan.recovery_shards * plan.group_count; + let report = |completed: u64| { + check_recovery_cancelled(progress)?; + if let Some(progress) = progress { + progress.report(WriteProgressEvent::Advanced { + operation: WriteOperation::Recovery, + completed_bytes: completed.min(payload_len), + total_bytes: payload_len, + pass, + }); + } + check_recovery_cancelled(progress) + }; + + check_recovery_cancelled(progress)?; + let field = RecoveryField::new(allowance)?; + let gf = field.view(); + let matrix = encoder_matrix_with_allowance(data_shards, recovery_shards, &gf, allowance)?; + let mut shard_states = Buffer::filled(data_shards, 0u64, allowance)?; + + let mut rows = match mode { + RecoveryMemoryMode::Resident => { + let parity = encode_parity_resident( + body, + body_len, + &plan, + &matrix, + &mut shard_states, + encode_units, + &report, + &gf, + allowance, + )?; + ParityRows::Resident(parity) + } + RecoveryMemoryMode::Striped { stripe_len } => { + let scratch = parity_scratch.ok_or(Error::PlanOverflow)?; + encode_parity_striped( + body, + body_len, + &plan, + &matrix, + &mut shard_states, + stripe_len, + scratch, + encode_units, + &report, + &gf, + allowance, + )?; + ParityRows::Spooled { + scratch, + group_count: plan.group_count, + } + } + }; + + // Every chunk repeats the CRC state of parity row 0, so it has to be + // known before the first chunk can be framed. + let mut buffer = Buffer::filled(RECOVERY_IO_BLOCK.min(group_count.max(1)), 0u8, allowance)?; + let mut final_state = 0u64; + rows.for_each_chunk(0, &mut buffer, |chunk| { + final_state = crc64_update(chunk, final_state); + })?; + + let chunk_data_extent = body_len + .saturating_sub( + plan.group_count + .saturating_mul(plan.data_shards.saturating_sub(1)), + ) + .min(plan.group_count); + // The extent cannot exceed group_count, already bounded by shard_size. + let chunk_data_extent_u32 = chunk_data_extent as u32; + + let mut payload_crc32 = crate::crc32::Crc32::new(); + let mut written = 0u64; + for shard_index in 0..recovery_shards { + check_recovery_cancelled(progress)?; + let mut header = Buffer::filled(header_size, 0u8, allowance)?; + // The validated header size is exactly 0x48 + 8 * data_shards. + // Writing its fields directly removes impossible short writes to a + // fixed-size in-memory Cursor while retaining the RAR wire layout. + header[0x00..0x04].copy_from_slice(b"{RB}"); + header[0x0c..0x10].copy_from_slice(&total_size.to_le_bytes()); + header[0x10..0x14].copy_from_slice(&header_size_u32.to_le_bytes()); + header[0x14] = 1; + header[0x15] = 1; + header[0x1e..0x22].copy_from_slice(&chunk_data_extent_u32.to_le_bytes()); + header[0x22..0x2a].copy_from_slice(&body_len.to_le_bytes()); + header[0x2a..0x32].copy_from_slice(&plan.group_count.to_le_bytes()); + header[0x32..0x3a].copy_from_slice(&plan.shard_size.to_le_bytes()); + header[0x3a..0x3c].copy_from_slice(&data_shards_u16.to_le_bytes()); + header[0x3c..0x3e].copy_from_slice(&recovery_shards_u16.to_le_bytes()); + header[0x3e..0x40].copy_from_slice(&(shard_index as u16).to_le_bytes()); + for (index, &state) in shard_states.iter().enumerate() { + header[0x40 + index * 8..0x48 + index * 8].copy_from_slice(&state.to_le_bytes()); + } + header[header_size - 8..].copy_from_slice(&final_state.to_le_bytes()); + // The chunk CRC covers everything from 0x0c onwards, so compute it + // over the header and the parity row before either is emitted. + let mut chunk_crc = crc64_update(&header[0x0c..], CRC64_XZ_INIT); + rows.for_each_chunk(shard_index, &mut buffer, |chunk| { + chunk_crc = crc64_update(chunk, chunk_crc); + })?; + header[0x04..0x0c].copy_from_slice(&(chunk_crc ^ CRC64_XZ_INIT).to_le_bytes()); + + sink.write_all(&header)?; + payload_crc32.update(&header); + written += header.len() as u64; + + let mut sink_error = None; + rows.for_each_chunk(shard_index, &mut buffer, |chunk| { + if sink_error.is_some() { + return; + } + if let Err(error) = sink.write_all(chunk) { + sink_error = Some(error); + return; + } + payload_crc32.update(chunk); + })?; + if let Some(error) = sink_error { + return Err(error.into()); + } + written += plan.group_count; + + report(encode_units + (shard_index as u64 + 1) * plan.header_size)?; + } + + debug_assert_eq!(written, payload_len); + if let Some(progress) = progress { + progress.report(WriteProgressEvent::OperationFinished { + operation: WriteOperation::Recovery, + total_bytes: Some(payload_len), + total_entries: None, + pass, + }); + } + + Ok(StreamedRecoveryOutput { + plan, + payload_len, + payload_crc32: payload_crc32.finish(), + }) +} + +fn check_recovery_cancelled(progress: Option>) -> Result<()> { + if progress.is_some_and(ProgressReporter::is_cancelled) { + return Err(Error::Cancelled); + } + Ok(()) +} + +fn rows_with_allowance( + rows: usize, + width: usize, + allowance: &B, +) -> Result, B>> { + let mut out = Buffer::with_capacity(rows, allowance)?; + for _ in 0..rows { + out.push_admitted(Buffer::filled(width, 0u8, allowance)?); + } + Ok(out) +} + +enum RecoveryField { + Shared(&'static Gf16), + Owned { + exp: Buffer, + log: Buffer, + }, +} +impl RecoveryField { + fn new(allowance: &B) -> Result { + if !B::LIMITED { + return Ok(Self::Shared(shared_gf16())); + } + let mut exp = Buffer::filled(FIELD_SIZE * 4 + 1, 0u16, allowance)?; + let mut log = Buffer::filled(FIELD_SIZE + 1, 0u32, allowance)?; + initialize_field(&mut exp, &mut log); + Ok(Self::Owned { exp, log }) + } + fn view(&self) -> GfView<'_> { + match self { + Self::Shared(gf) => gf.view(), + Self::Owned { exp, log } => GfView { exp, log }, + } + } +} + +fn encoder_matrix_with_allowance( + data_shards: usize, + recovery_shards: usize, + gf: &GfView<'_>, + allowance: &B, +) -> Result, B>> { + if data_shards == 0 + || recovery_shards == 0 + || data_shards + .checked_add(recovery_shards) + .is_none_or(|sum| sum > FIELD_SIZE) + { + return Err(Error::TooManyShards); + } + let mut matrix = Buffer::with_capacity(recovery_shards, allowance)?; + for i in 0..recovery_shards { + let mut row = Buffer::filled(data_shards, 0u16, allowance)?; + for (j, cell) in row.iter_mut().enumerate() { + *cell = gf.inv(((i + data_shards) ^ j) as u16)?; + } + matrix.push_admitted(row); + } + Ok(matrix) +} + +/// Parity rows, wherever they happen to live. +enum ParityRows<'a, B: Budget> { + Resident(Buffer, B>), + Spooled { + scratch: &'a mut dyn ReadWriteSeek, + group_count: u64, + }, +} + +impl ParityRows<'_, B> { + /// Feeds row `index` to `visit` in buffer-sized pieces. + fn for_each_chunk( + &mut self, + index: usize, + buffer: &mut [u8], + mut visit: impl FnMut(&[u8]), + ) -> Result<()> { + match self { + Self::Resident(rows) => { + let row = &rows[index]; + if !row.is_empty() { + visit(row); + } + Ok(()) + } + Self::Spooled { + scratch, + group_count, + } => { + // The admitted payload contains every recovery row, and the + // caller supplies only row zero or an in-range shard index. + let start = index as u64 * *group_count; + scratch.seek(std::io::SeekFrom::Start(start))?; + let mut remaining = *group_count; + while remaining != 0 { + let want = remaining.min(buffer.len() as u64) as usize; + scratch.read_exact(&mut buffer[..want])?; + visit(&buffer[..want]); + remaining -= want as u64; + } + Ok(()) + } + } + } +} + +/// Accumulates `coefficient * source` into `destination` over GF(2^16). +/// +/// A trailing odd byte is treated as the low half of a word whose high half is +/// the zero padding every shard carries. +fn accumulate_scaled(destination: &mut [u8], source: &[u8], coefficient: u16, gf: &GfView<'_>) { + let words = source.len() / 2; + for word in 0..words { + let offset = word * 2; + let value = u16::from_le_bytes([source[offset], source[offset + 1]]); + let scaled = gf.mul(coefficient, value); + if scaled == 0 { + continue; + } + let current = u16::from_le_bytes([destination[offset], destination[offset + 1]]); + destination[offset..offset + 2].copy_from_slice(&(current ^ scaled).to_le_bytes()); + } + if !source.len().is_multiple_of(2) { + let offset = words * 2; + let value = u16::from_le_bytes([source[offset], 0]); + let scaled = gf.mul(coefficient, value); + if scaled != 0 { + let current = u16::from_le_bytes([destination[offset], destination[offset + 1]]); + destination[offset..offset + 2].copy_from_slice(&(current ^ scaled).to_le_bytes()); + } + } +} + +/// Single forward pass over the body, holding every parity row in memory. +#[allow(clippy::too_many_arguments)] +fn encode_parity_resident( + body: &mut dyn ReadSeek, + body_len: u64, + plan: &InlineRecoveryPlan, + matrix: &[Buffer], + shard_states: &mut [u64], + encode_units: u64, + report: &dyn Fn(u64) -> Result<()>, + gf: &GfView<'_>, + allowance: &B, +) -> Result, B>> { + // The streaming entry point admitted these as native-size values. + let group_count = plan.group_count as usize; + let recovery_shards = plan.recovery_shards as usize; + let mut parity = rows_with_allowance(recovery_shards, group_count, allowance)?; + if group_count == 0 || body_len == 0 { + return Ok(parity); + } + + let mut buffer = Buffer::filled(RECOVERY_IO_BLOCK.min(group_count), 0u8, allowance)?; + body.seek(std::io::SeekFrom::Start(0))?; + + let mut consumed = 0u64; + for (shard_index, state) in shard_states.iter_mut().enumerate() { + // The wire limits admit at most u16::MAX data shards of u32-sized + // groups, so every data-shard offset fits u64. + let shard_start = shard_index as u64 * plan.group_count; + if shard_start >= body_len { + break; + } + let shard_bytes = plan.group_count.min(body_len - shard_start); + + let mut offset = 0u64; + while offset < shard_bytes { + let want = (shard_bytes - offset).min(buffer.len() as u64) as usize; + body.read_exact(&mut buffer[..want])?; + *state = crc64_update(&buffer[..want], *state); + + let destination_offset = offset as usize; + for (row, parity_row) in matrix.iter().zip(parity.iter_mut()) { + accumulate_scaled( + &mut parity_row[destination_offset..], + &buffer[..want], + row[shard_index], + gf, + ); + } + + offset += want as u64; + consumed += want as u64; + report(scaled_progress(consumed, body_len, encode_units))?; + } + } + + Ok(parity) +} + +/// Column-stripe pass: bounded memory, one seek per data shard per stripe, +/// and the body still read exactly once in total. +#[allow(clippy::too_many_arguments)] +fn encode_parity_striped( + body: &mut dyn ReadSeek, + body_len: u64, + plan: &InlineRecoveryPlan, + matrix: &[Buffer], + shard_states: &mut [u64], + stripe_len: usize, + scratch: &mut dyn ReadWriteSeek, + encode_units: u64, + report: &dyn Fn(u64) -> Result<()>, + gf: &GfView<'_>, + allowance: &B, +) -> Result<()> { + let recovery_shards = plan.recovery_shards as usize; + if plan.group_count == 0 { + return Ok(()); + } + let stripe_len = stripe_len.max(2); + let mut stripe = rows_with_allowance(recovery_shards, stripe_len, allowance)?; + let mut buffer = Buffer::filled(stripe_len, 0u8, allowance)?; + let mut consumed = 0u64; + + let mut column = 0u64; + while column < plan.group_count { + let width = (plan.group_count - column).min(stripe_len as u64); + let width_usize = width as usize; + for row in stripe.iter_mut() { + row[..width_usize].fill(0); + } + + for (shard_index, state) in shard_states.iter_mut().enumerate() { + let shard_start = shard_index as u64 * plan.group_count; + let shard_bytes = plan.group_count.min(body_len.saturating_sub(shard_start)); + if column >= shard_bytes { + continue; + } + let span = width.min(shard_bytes - column); + let span_usize = span as usize; + + body.seek(std::io::SeekFrom::Start(shard_start + column))?; + body.read_exact(&mut buffer[..span_usize])?; + *state = crc64_update(&buffer[..span_usize], *state); + + for (row, stripe_row) in matrix.iter().zip(stripe.iter_mut()) { + accumulate_scaled(stripe_row, &buffer[..span_usize], row[shard_index], gf); + } + + consumed += span; + report(scaled_progress(consumed, body_len, encode_units))?; + } + + for (shard_index, stripe_row) in stripe.iter().enumerate() { + // position is within an admitted row of the complete payload. + let position = shard_index as u64 * plan.group_count + column; + scratch.seek(std::io::SeekFrom::Start(position))?; + scratch.write_all(&stripe_row[..width_usize])?; + } + + column += width; + } + + Ok(()) +} + +fn scaled_progress(consumed: u64, total: u64, units: u64) -> u64 { + // Both callers report only after consuming body bytes, so total is nonzero. + ((consumed as u128 * units as u128) / total as u128) as u64 +} + +#[derive(Debug, Clone)] +struct InlineRecoveryChunk { + plan: InlineRecoveryPlan, + protected_size: u64, + shard_index: usize, + data_shard_states: Vec, + parity: Vec, +} + +#[derive(Debug, Clone)] +struct FoundInlineRecoveryChunk { + offset: usize, + chunk: InlineRecoveryChunk, +} + +pub fn repair_inline_recovery_prefix( + archive_prefix: &[u8], + recovery_data: &[u8], +) -> Result> { + repair_inline_recovery_prefix_with_control( + archive_prefix, + recovery_data, + &crate::read_control::ReadControl::default(), + ) +} + +pub(crate) fn repair_inline_recovery_prefix_with_control( + archive_prefix: &[u8], + recovery_data: &[u8], + control: &crate::read_control::ReadControl, +) -> Result> { + check_repair(control)?; + // The range path keeps empty shards and zero padding virtual. In particular, + // healthy prefixes must not allocate data_shards * group_count bytes merely + // because a recovery header describes that much capacity. + let repaired_shards = repair_inline_recovery_prefix_shards_with_control( + archive_prefix.len(), + recovery_data, + |range| Ok(archive_prefix[range].to_vec()), + control, + )?; + check_repair(control)?; + let mut repaired = archive_prefix.to_vec(); + for (range, shard) in repaired_shards { + check_repair(control)?; + repaired[range].copy_from_slice(&shard); + } + Ok(repaired) +} + +/// Repair damaged RAR5 inline-recovery data shards without materializing the +/// whole protected prefix. +/// +/// `read_range` receives byte ranges relative to the protected prefix and must +/// return the current bytes for each requested range. The returned pairs contain +/// only the damaged prefix ranges that need to be written back. +/// +/// An admission ceiling of four times `protected_size` plus the recovery data, +/// and at least 256 MiB, covers solve matrices and their working +/// copies, row descriptors, RHS/pivot/read scratch and returned repaired shard +/// buffers. Input and parsed recovery records, shared field tables, final +/// collected archive output, allocator overhead and control storage are excluded. +/// This is not a process-RAM limit. Oversized solves return +/// [`Error::RepairWorkspaceLimitExceeded`] before allocating their matrices. +/// Requested range scratch is admitted before discovery calls `read_range`; +/// additional allocations made by the caller's callback are outside the ceiling. +pub fn repair_inline_recovery_prefix_shards( + protected_size: usize, + recovery_data: &[u8], + read_range: F, +) -> Result, Vec)>> +where + F: FnMut(std::ops::Range) -> Result>, +{ + repair_inline_recovery_prefix_shards_with_control( + protected_size, + recovery_data, + read_range, + &crate::read_control::ReadControl::default(), + ) +} + +pub(crate) fn repair_inline_recovery_prefix_shards_with_control( + protected_size: usize, + recovery_data: &[u8], + read_range: F, + control: &crate::read_control::ReadControl, +) -> Result, Vec)>> +where + F: FnMut(std::ops::Range) -> Result>, +{ + repair_inline_recovery_prefix_shards_with_workspace_limit( + protected_size, + recovery_data, + read_range, + control, + repair_workspace_limit((protected_size as u64).saturating_add(recovery_data.len() as u64)), + ) +} + +fn repair_inline_recovery_prefix_shards_with_workspace_limit( + protected_size: usize, + recovery_data: &[u8], + mut read_range: F, + control: &crate::read_control::ReadControl, + workspace_limit: u64, +) -> Result, Vec)>> +where + F: FnMut(std::ops::Range) -> Result>, +{ + let mut poller = control.poller(); + check_repair(control)?; + let chunks = parse_available_inline_recovery_chunks_with_control(recovery_data, control)?; + let first = chunks.first().ok_or(Error::BadRecoveryChunk)?; + if first.protected_size != protected_size as u64 { + return Err(Error::BadRecoveryChunk); + } + if chunks.iter().any(|chunk| { + chunk.plan != first.plan + || chunk.protected_size != first.protected_size + || chunk.data_shard_states != first.data_shard_states + }) { + return Err(Error::BadRecoveryChunk); + } + + let plan = first.plan; + let shard_len = usize::try_from(plan.group_count).map_err(|_| Error::PlanOverflow)?; + let shard_ranges = split_prefix_shard_ranges(protected_size, plan)?; + // Admit the largest requested range before CRC discovery asks the caller + // to materialize it. Arbitrary additional callback allocations are outside + // our policy; the requested byte range is bounded here. + admit_repair_workspace_with_limit( + repair_solve_workspace_required( + shard_ranges.len(), + 0, + shard_len, + 0, + shard_len.min(protected_size), + )?, + workspace_limit, + )?; + let mut damaged = Vec::new(); + for (index, range) in shard_ranges.iter().enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + let shard = read_range(range.clone())?; + if shard.len() != range.len() { + return Err(Error::ShardSizeMismatch); + } + if repair_crc(&shard, 0, control)? != first.data_shard_states[index] { + damaged.push(index); + } + } + if damaged.is_empty() { + return Ok(Vec::new()); + } + if damaged.len() > chunks.len() { + return Err(Error::TooManyDamagedShards); + } + + let output_bytes = damaged.iter().try_fold(0u64, |bytes, &index| { + bytes + .checked_add(shard_ranges[index].len() as u64) + .ok_or(Error::PlanOverflow) + })?; + admit_repair_workspace_with_limit( + repair_solve_workspace_required( + shard_ranges.len(), + damaged.len(), + shard_len, + output_bytes, + shard_len.min(protected_size), + )?, + workspace_limit, + )?; + + let recovery_rows: Vec<_> = chunks[..damaged.len()] + .iter() + .map(|chunk| (chunk.shard_index, chunk.parity.as_slice())) + .collect(); + let gf = shared_gf16(); + // Only damaged columns and available rows participate in the solve. A + // high row index is an index, not a request to allocate preceding rows. + let equations = recovery_equations(gf, shard_ranges.len(), &damaged, &recovery_rows, control)?; + let inverse = invert_linear_system_matrix_with_control(gf, &equations, control)?; + let word_count = shard_len / 2; + let mut rhs_by_row = Vec::with_capacity(recovery_rows.len()); + for (_, parity) in &recovery_rows { + let mut rhs = Vec::with_capacity(word_count); + for word in parity.as_chunks::<2>().0 { + poller.check(0).map_err(|_| Error::Cancelled)?; + rhs.push(u16::from_le_bytes([word[0], word[1]])); + } + rhs_by_row.push(rhs); + } + let mut damaged_lookup = vec![false; shard_ranges.len()]; + for &index in &damaged { + damaged_lookup[index] = true; + } + + for (data_index, range) in shard_ranges.iter().enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + if damaged_lookup[data_index] || range.is_empty() { + continue; + } + let shard = read_range(range.clone())?; + if shard.len() != range.len() { + return Err(Error::ShardSizeMismatch); + } + for (row_index, rhs) in rhs_by_row.iter_mut().enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + let coeff = recovery_coefficient( + gf, + shard_ranges.len(), + recovery_rows[row_index].0, + data_index, + )?; + // Omitted bytes are zero padding, so they contribute no RHS terms. + for (word_index, word) in shard.chunks(2).enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + let data_symbol = u16::from_le_bytes([word[0], word.get(1).copied().unwrap_or(0)]); + rhs[word_index] ^= gf.mul(coeff, data_symbol); + } + } + } + + let mut repaired = damaged + .iter() + .map(|&index| vec![0; shard_ranges[index].len()]) + .collect::>(); + for word_index in 0..word_count { + poller.check(0).map_err(|_| Error::Cancelled)?; + let rhs = rhs_by_row + .iter() + .map(|row| row[word_index]) + .collect::>(); + let solved = apply_inverse_matrix_with_control(gf, &inverse, &rhs, control)?; + for (output, &symbol) in repaired.iter_mut().zip(&solved) { + poller.check(0).map_err(|_| Error::Cancelled)?; + let byte_offset = word_index * 2; + if byte_offset < output.len() { + let bytes = symbol.to_le_bytes(); + let take = (output.len() - byte_offset).min(2); + output[byte_offset..byte_offset + take].copy_from_slice(&bytes[..take]); + } + } + } + + Ok(damaged + .into_iter() + .zip(repaired) + .map(|(index, data)| (shard_ranges[index].clone(), data)) + .collect()) +} + +/// What a raw inline-recovery repair knows beyond the archive bytes. +#[derive(Debug, Clone, Default)] +pub(crate) struct InlineRepairOptions<'a> { + pub control: crate::read_control::ReadControl, + /// Password for header-encrypted archives, needed to frame a replacement + /// end-of-archive header. + pub password: Option<&'a [u8]>, + /// Byte range holding this archive's recovery record, when a parsed + /// archive already said where it lives. + pub record_range: Option>, +} + +/// Repair an archive's protected prefix and, when possible, incomplete inline +/// recovery records. Solves use the admission ceiling documented on +/// [`repair_inline_recovery_prefix_shards`]. Record rebuilding checks a separate +/// use of that ceiling, scaled by `input`, for its full encoder matrix, +/// resident parity, CRC/header and I/O scratch, row descriptors and growing +/// intermediate framed-record buffer. Input/parsed records, shared field +/// tables, final archive output and allocator/control overhead are excluded. +/// If rebuilding is too large, the repaired prefix is returned while the +/// incomplete recovery record is retained. +pub fn repair_inline_recovery_archive(input: &[u8]) -> Result> { + Ok(repair_inline_recovery_archive_with_report(input, &InlineRepairOptions::default())?.0) +} + +pub(crate) fn repair_inline_recovery_archive_with_report( + input: &[u8], + options: &InlineRepairOptions<'_>, +) -> Result<(Vec, crate::RecoveryRepairReport)> { + let control = &options.control; + check_repair(control)?; + let chunks = select_record_chunks( + find_inline_recovery_chunks_with_control(input, control)?, + options.record_range.clone(), + )?; + let first = chunks.first().ok_or(Error::BadRecoveryChunk)?; + let plan = first.chunk.plan; + let protected_size = + usize::try_from(first.chunk.protected_size).map_err(|_| Error::PlanOverflow)?; + if protected_size > input.len() { + return Err(Error::BadRecoveryChunk); + } + let mut recovery_data = Vec::with_capacity( + chunks + .iter() + .map(|found| found.chunk.plan.shard_size as usize) + .sum(), + ); + for found in &chunks { + check_repair(control)?; + append_inline_recovery_chunk(input, found, &mut recovery_data); + } + let original_prefix = &input[..protected_size]; + let repaired_prefix = + repair_inline_recovery_prefix_with_control(original_prefix, &recovery_data, control)?; + let data_repaired = repaired_prefix != original_prefix; + let mut indices = chunks + .iter() + .map(|found| found.chunk.shard_index) + .collect::>(); + indices.sort_unstable(); + indices.dedup(); + let available = indices.len() as u64; + let expected = plan.recovery_shards; + let record_start = chunks + .iter() + .filter_map(|found| { + let distance = found + .chunk + .shard_index + .checked_mul(plan.shard_size as usize)?; + found.offset.checked_sub(distance) + }) + .min() + .ok_or(Error::BadRecoveryChunk)?; + if record_start < protected_size { + return Err(Error::BadRecoveryChunk); + } + let payload_len = usize::try_from(plan.payload_size()?).map_err(|_| Error::PlanOverflow)?; + let record_end = record_start + .checked_add(payload_len) + .ok_or(Error::PlanOverflow)?; + + // Splicing the repaired prefix back in is the floor. Whatever happens to + // the record itself, the caller still gets the data repair. + let prefix_only = || { + let mut out = input.to_vec(); + out[..protected_size].copy_from_slice(&repaired_prefix); + out + }; + let record_complete = available == expected && record_end <= input.len(); + let mut recovery_record_rebuilt = false; + let mut repaired = if record_complete { + prefix_only() + } else { + match rebuild_inline_recovery_record_with_control( + input, + &repaired_prefix, + record_start..record_end, + plan, + control, + ) { + Ok(rebuilt) => { + recovery_record_rebuilt = true; + rebuilt + } + Err(Error::Cancelled) => return Err(Error::Cancelled), + Err(_) => prefix_only(), + } + }; + + // An archive that stops where its recovery record does lost its + // end-of-archive header along with whatever followed. + let mut end_record_rebuilt = false; + if (record_complete || recovery_record_rebuilt) && repaired.len() <= record_end { + let mut read_options = crate::ArchiveReadOptions::with_optional_password(options.password); + read_options.cancellation = control.cancellation(); + let end = crate::rar50::recovery_end_header(&repaired, read_options) + .map_err(map_recovery_end_error)?; + repaired.extend_from_slice(&end); + end_record_rebuilt = true; + } + let report = crate::RecoveryRepairReport { + changed: repaired != input, + data_repaired, + recovery_record_rebuilt, + end_record_rebuilt, + available_recovery_shards: Some(available), + expected_recovery_shards: Some(expected), + }; + check_repair(control)?; + Ok((repaired, report)) +} + +fn map_recovery_end_error(error: crate::Error) -> Error { + if error.kind() == crate::ErrorKind::Cancelled { + Error::Cancelled + } else { + Error::BadRecoveryChunk + } +} + +/// Rebuilds the whole recovery record from the repaired prefix, replacing +/// `record` in the archive. +/// +/// Admission counts the full encoder matrix, resident parity and the framed +/// record, including a growing Vec's old and replacement buffers. Limiting +/// parity alone leaves repeated headers and coefficients unbounded when a +/// small available chunk declares thousands of missing recovery rows. +/// Refusal costs the caller the rebuild and keeps the data repair. +fn rebuild_inline_recovery_record_with_control( + input: &[u8], + repaired_prefix: &[u8], + record: std::ops::Range, + plan: InlineRecoveryPlan, + control: &crate::read_control::ReadControl, +) -> Result> { + check_repair(control)?; + let parity_bytes = plan + .recovery_shards + .checked_mul(plan.group_count) + .ok_or(Error::PlanOverflow)?; + if parity_bytes > MAX_REBUILD_PARITY_BYTES { + return Err(Error::RebuildTooLarge); + } + let matrix_cells = repair_workspace_product(plan.data_shards, plan.recovery_shards)?; + let row_descriptors = 2 * std::mem::size_of::>() as u64; + admit_repair_workspace( + repair_workspace_sum(&[ + repair_workspace_product(matrix_cells, std::mem::size_of::() as u64)?, + parity_bytes, + repair_workspace_product(plan.recovery_shards, row_descriptors)?, + repair_workspace_product(plan.data_shards, std::mem::size_of::() as u64)?, + // At most twice the framed record capacity plus its previous buffer + // while Vec growth replaces it. This is deliberately conservative. + repair_workspace_product(plan.payload_size()?, 3)?, + plan.group_count.max(1).min(RECOVERY_IO_BLOCK as u64), + plan.header_size, + ])?, + input.len() as u64, + )?; + let rebuilt = build_inline_recovery_data_for_plan_with_control(repaired_prefix, plan, control)?; + if rebuilt.len() != record.len() { + return Err(Error::BadRecoveryChunk); + } + // record.start came from a parsed chunk within input, and the prefix + // boundary was checked above. + let gap = &input[repaired_prefix.len()..record.start]; + let mut out = Vec::with_capacity(input.len().max(record.end)); + out.extend_from_slice(repaired_prefix); + out.extend_from_slice(gap); + out.extend_from_slice(&rebuilt); + if let Some(tail) = input.get(record.end..) { + out.extend_from_slice(tail); + } + Ok(out) +} + +/// Narrows found chunks down to the ones belonging to this archive's own +/// recovery record. +/// +/// A stored archive nested inside this one carries `{RB}` chunks that parse +/// just as cleanly as ours, and taking the first one found picks the nested +/// record whenever it sits earlier in the file. This archive's record always +/// protects more bytes than anything stored inside it, so the largest +/// protected size wins. `record_range` settles it outright when a parsed +/// archive already said where the record is. +fn select_record_chunks( + chunks: Vec, + record_range: Option>, +) -> Result> { + let mut chunks = match record_range { + Some(range) => chunks + .into_iter() + .filter(|found| range.contains(&found.offset)) + .collect(), + None => chunks, + }; + let protected_size = chunks + .iter() + .map(|found| found.chunk.protected_size) + .max() + .ok_or(Error::BadRecoveryChunk)?; + chunks.retain(|found| found.chunk.protected_size == protected_size); + Ok(chunks) +} + +fn find_inline_recovery_chunks_with_control( + input: &[u8], + control: &crate::read_control::ReadControl, +) -> Result> { + check_repair(control)?; + let mut chunks = Vec::new(); + let mut offset = 0usize; + while let Some(relative) = repair_marker(&input[offset..], control)? { + let start = offset + relative; + if let Ok(chunk) = parse_inline_recovery_chunk_with_control(&input[start..], control) { + // Parsing already checked that this entire declared shard fits. + let shard_size = chunk.plan.shard_size as usize; + chunks.push(FoundInlineRecoveryChunk { + offset: start, + chunk, + }); + offset = start + shard_size; + continue; + } + check_repair(control)?; + offset = start + 1; + } + if chunks.is_empty() { + return Err(Error::BadRecoveryChunk); + } + Ok(chunks) +} + +fn find_recovery_marker(input: &[u8]) -> Option { + let mut offset = 0usize; + while offset + 4 <= input.len() { + let relative = input[offset..].iter().position(|&byte| byte == b'{')?; + offset += relative; + if input.get(offset..offset + 4) == Some(b"{RB}") { + return Some(offset); + } + offset += 1; + } + None +} + +fn append_inline_recovery_chunk(input: &[u8], found: &FoundInlineRecoveryChunk, out: &mut Vec) { + // The scanner parsed this complete u32-sized shard from input[start..]. + let shard_size = found.chunk.plan.shard_size as usize; + let start = found.offset; + let end = start + shard_size; + out.extend_from_slice(&input[start..end]); +} + +/// Reconstruct equally padded data shards from available recovery rows. +/// +/// Known shard lengths, missing-row availability and field dimensions are +/// validated before output allocation. An admission ceiling of four times the +/// supplied shard bytes, and at least 256 MiB, covers padded output, solve +/// matrices/copies and row/vector scratch. The caller's input, shared field +/// tables, allocator and control overhead are excluded; this is not a +/// process-RAM limit. Oversized reconstruction returns +/// [`Error::RepairWorkspaceLimitExceeded`]. +pub fn reconstruct_data_shards( + data_shards: &[Option<&[u8]>], + recovery_shards: &[(usize, &[u8])], +) -> Result>> { + if data_shards.is_empty() { + return Err(Error::TooManyShards); + } + let shard_len = recovery_shards + .first() + .map(|(_, shard)| shard.len()) + .or_else(|| data_shards.iter().flatten().map(|shard| shard.len()).max()) + .ok_or(Error::TooManyDamagedShards)?; + if !shard_len.is_multiple_of(2) { + return Err(Error::OddShardSize); + } + if recovery_shards + .iter() + .any(|(_, shard)| shard.len() != shard_len) + { + return Err(Error::ShardSizeMismatch); + } + if data_shards + .iter() + .flatten() + .any(|shard| shard.len() > shard_len) + { + return Err(Error::ShardSizeMismatch); + } + let missing_count = data_shards.iter().filter(|shard| shard.is_none()).count(); + if missing_count > recovery_shards.len() { + return Err(Error::TooManyDamagedShards); + } + if data_shards.len() > FIELD_SIZE { + return Err(Error::TooManyShards); + } + // Unused recovery rows need no field coefficients. Validate each selected + // row without initializing the field or allocating any padded output. + if recovery_shards[..missing_count].iter().any(|&(row, _)| { + data_shards + .len() + .checked_add(row) + .is_none_or(|position| position >= FIELD_SIZE) + }) { + return Err(Error::TooManyShards); + } + let padded_bytes = repair_workspace_product(data_shards.len() as u64, shard_len as u64)?; + let input_bytes = data_shards + .iter() + .flatten() + .map(|shard| shard.len()) + .chain(recovery_shards.iter().map(|(_, shard)| shard.len())) + .fold(0u64, |total, len| total.saturating_add(len as u64)); + admit_repair_workspace( + repair_solve_workspace_required( + data_shards.len(), + missing_count, + shard_len, + padded_bytes, + 0, + )?, + input_bytes, + )?; + + let missing = data_shards + .iter() + .enumerate() + .filter_map(|(index, shard)| shard.is_none().then_some(index)) + .collect::>(); + let mut out = Vec::with_capacity(data_shards.len()); + for shard in data_shards { + let mut padded = vec![0; shard_len]; + if let Some(shard) = shard { + padded[..shard.len()].copy_from_slice(shard); + } + out.push(padded); + } + if missing.is_empty() { + return Ok(out); + } + recover_damaged_shards(&mut out, &missing, &recovery_shards[..missing.len()])?; + Ok(out) +} + +fn parse_available_inline_recovery_chunks_with_control( + recovery_data: &[u8], + control: &crate::read_control::ReadControl, +) -> Result> { + check_repair(control)?; + Ok( + find_inline_recovery_chunks_with_control(recovery_data, control)? + .into_iter() + .map(|found| found.chunk) + .collect(), + ) +} + +pub(crate) fn inline_recovery_chunk_counts_with_control( + recovery_data: &[u8], + control: &crate::read_control::ReadControl, +) -> Result<(u64, u64)> { + check_repair(control)?; + let chunks = parse_available_inline_recovery_chunks_with_control(recovery_data, control)?; + let first = chunks.first().ok_or(Error::BadRecoveryChunk)?; + let expected = first.plan.recovery_shards; + let mut indices = chunks + .iter() + .map(|chunk| chunk.shard_index) + .collect::>(); + indices.sort_unstable(); + indices.dedup(); + Ok((indices.len() as u64, expected)) +} + +fn parse_inline_recovery_chunk_with_control( + input: &[u8], + control: &crate::read_control::ReadControl, +) -> Result { + check_repair(control)?; + if input.len() < 0x48 || &input[..4] != b"{RB}" { + return Err(Error::BadRecoveryChunk); + } + // The fixed 0x48-byte prefix is present, so all fixed fields fit. + let total_size = u32::from_le_bytes(input[0x0c..0x10].try_into().unwrap()) as u64; + let header_size = u32::from_le_bytes(input[0x10..0x14].try_into().unwrap()) as u64; + if header_size < RAR5_RECOVERY_CHUNK_FIXED_HEADER_SIZE || header_size > total_size { + return Err(Error::BadRecoveryChunk); + } + // Both values came from u32 fields, which fit usize on supported targets. + let total_size_usize = total_size as usize; + let header_size_usize = header_size as usize; + if input.len() < total_size_usize { + return Err(Error::BadRecoveryChunk); + } + let expected_crc = u64::from_le_bytes(input[0x04..0x0c].try_into().unwrap()); + let actual_crc = !repair_crc(&input[0x0c..total_size_usize], CRC64_XZ_INIT, control)?; + if actual_crc != expected_crc { + return Err(Error::BadRecoveryChunk); + } + if input[0x14] != 1 || input[0x15] != 1 { + return Err(Error::BadRecoveryChunk); + } + + let protected_size = u64::from_le_bytes(input[0x22..0x2a].try_into().unwrap()); + let group_count = u64::from_le_bytes(input[0x2a..0x32].try_into().unwrap()); + if !group_count.is_multiple_of(2) { + return Err(Error::BadRecoveryChunk); + } + let shard_size = u64::from_le_bytes(input[0x32..0x3a].try_into().unwrap()); + let data_shards = u16::from_le_bytes(input[0x3a..0x3c].try_into().unwrap()) as u64; + let recovery_shards = u16::from_le_bytes(input[0x3c..0x3e].try_into().unwrap()) as u64; + if data_shards == 0 || recovery_shards == 0 || data_shards + recovery_shards > FIELD_SIZE as u64 + { + return Err(Error::BadRecoveryChunk); + } + let shard_index = u16::from_le_bytes(input[0x3e..0x40].try_into().unwrap()) as usize; + let plan = InlineRecoveryPlan { + data_shards, + recovery_shards, + group_count, + header_size, + shard_size, + }; + if shard_size != total_size + || shard_index >= recovery_shards as usize + || header_size_usize != 0x48 + data_shards as usize * 8 + { + return Err(Error::BadRecoveryChunk); + } + + let mut data_shard_states = Vec::with_capacity(data_shards as usize); + let mut pos = 0x40; + for _ in 0..data_shards { + check_repair(control)?; + data_shard_states.push(u64::from_le_bytes(input[pos..pos + 8].try_into().unwrap())); + pos += 8; + } + let _final_state = u64::from_le_bytes(input[pos..pos + 8].try_into().unwrap()); + let parity = input[header_size_usize..total_size_usize].to_vec(); + if parity.len() as u64 != group_count { + return Err(Error::BadRecoveryChunk); + } + Ok(InlineRecoveryChunk { + plan, + protected_size, + shard_index, + data_shard_states, + parity, + }) +} + +fn recover_damaged_shards( + data_shards: &mut [Vec], + damaged: &[usize], + recovery_shards: &[(usize, &[u8])], +) -> Result<()> { + recover_damaged_shards_with_control( + data_shards, + damaged, + recovery_shards, + &crate::read_control::ReadControl::default(), + ) +} + +fn recover_damaged_shards_with_control( + data_shards: &mut [Vec], + damaged: &[usize], + recovery_shards: &[(usize, &[u8])], + control: &crate::read_control::ReadControl, +) -> Result<()> { + let mut poller = control.poller(); + check_repair(control)?; + let data_count = data_shards.len(); + let mut damaged_lookup = vec![false; data_count]; + for &data_index in damaged { + poller.check(0).map_err(|_| Error::Cancelled)?; + damaged_lookup[data_index] = true; + } + + if recovery_shards.is_empty() { + return Err(Error::TooManyDamagedShards); + } + let gf = shared_gf16(); + let equations = recovery_equations(gf, data_count, damaged, recovery_shards, control)?; + let inverse = invert_linear_system_matrix_with_control(gf, &equations, control)?; + + // Coefficients for each supplied row against every column, hoisted out of + // the per-word loop. Damaged columns are skipped there, so they stay zero. + let mut coefficients = Vec::with_capacity(recovery_shards.len()); + for &(row_index, _) in recovery_shards { + let mut row = vec![0; data_count]; + for (data_index, coefficient) in row.iter_mut().enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + if !damaged_lookup[data_index] { + *coefficient = recovery_coefficient(gf, data_count, row_index, data_index)?; + } + } + coefficients.push(row); + } + + let shard_len = data_shards.first().ok_or(Error::TooManyShards)?.len(); + for word_offset in (0..shard_len).step_by(2) { + poller.check(0).map_err(|_| Error::Cancelled)?; + let mut rhs = Vec::with_capacity(recovery_shards.len()); + for (&(_, parity), coefficients) in recovery_shards.iter().zip(&coefficients) { + poller.check(0).map_err(|_| Error::Cancelled)?; + let mut value = u16::from_le_bytes([parity[word_offset], parity[word_offset + 1]]); + for (data_index, shard) in data_shards.iter().enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + if damaged_lookup[data_index] { + continue; + } + let data_symbol = u16::from_le_bytes([shard[word_offset], shard[word_offset + 1]]); + value ^= gf.mul(coefficients[data_index], data_symbol); + } + rhs.push(value); + } + let solved = apply_inverse_matrix_with_control(gf, &inverse, &rhs, control)?; + for (&data_index, &symbol) in damaged.iter().zip(&solved) { + poller.check(0).map_err(|_| Error::Cancelled)?; + data_shards[data_index][word_offset..word_offset + 2] + .copy_from_slice(&symbol.to_le_bytes()); + } + } + Ok(()) +} + +fn recovery_coefficient(gf: &Gf16, data_count: usize, row: usize, column: usize) -> Result { + let position = data_count + .checked_add(row) + .filter(|&position| position < FIELD_SIZE) + .ok_or(Error::TooManyShards)?; + if data_count == 0 || column >= data_count { + return Err(Error::TooManyShards); + } + gf.inv((position ^ column) as u16) +} + +fn recovery_equations( + gf: &Gf16, + data_count: usize, + damaged: &[usize], + recovery_shards: &[(usize, &[u8])], + control: &crate::read_control::ReadControl, +) -> Result>> { + let mut equations = Vec::with_capacity(recovery_shards.len()); + let mut poller = control.poller(); + check_repair(control)?; + for &(row_index, _) in recovery_shards { + // Validate the row even when no damaged columns were supplied. + recovery_coefficient(gf, data_count, row_index, 0)?; + let mut row = Vec::with_capacity(damaged.len()); + for &column in damaged { + poller.check(0).map_err(|_| Error::Cancelled)?; + row.push(recovery_coefficient(gf, data_count, row_index, column)?); + } + equations.push(row); + } + Ok(equations) +} + +#[cfg(test)] +fn invert_linear_system_matrix(gf: &Gf16, matrix: &[Vec]) -> Result>> { + invert_linear_system_matrix_with_control( + gf, + matrix, + &crate::read_control::ReadControl::default(), + ) +} + +fn invert_linear_system_matrix_with_control( + gf: &Gf16, + matrix: &[Vec], + control: &crate::read_control::ReadControl, +) -> Result>> { + let mut poller = control.poller(); + check_repair(control)?; + let n = matrix.len(); + if matrix.iter().any(|row| row.len() != n) { + return Err(Error::BadRecoveryChunk); + } + let mut matrix = matrix.to_vec(); + let mut inverse = vec![vec![0u16; n]; n]; + for (row, inverse_row) in inverse.iter_mut().enumerate() { + poller.check(0).map_err(|_| Error::Cancelled)?; + inverse_row[row] = 1; + } + + for col in 0..n { + poller.check(0).map_err(|_| Error::Cancelled)?; + let pivot = (col..n) + .find(|&row| matrix[row][col] != 0) + .ok_or(Error::SingularElement)?; + matrix.swap(col, pivot); + inverse.swap(col, pivot); + let inv = gf.inv(matrix[col][col])?; + for value in &mut matrix[col] { + poller.check(0).map_err(|_| Error::Cancelled)?; + *value = gf.mul(*value, inv); + } + for value in &mut inverse[col] { + poller.check(0).map_err(|_| Error::Cancelled)?; + *value = gf.mul(*value, inv); + } + + let pivot_matrix_row = matrix[col].clone(); + let pivot_inverse_row = inverse[col].clone(); + for row in 0..n { + poller.check(0).map_err(|_| Error::Cancelled)?; + if row == col { + continue; + } + let factor = matrix[row][col]; + if factor == 0 { + continue; + } + for (value, pivot) in matrix[row] + .iter_mut() + .zip(pivot_matrix_row.iter().copied()) + .skip(col) + { + poller.check(0).map_err(|_| Error::Cancelled)?; + *value ^= gf.mul(factor, pivot); + } + for (value, pivot) in inverse[row] + .iter_mut() + .zip(pivot_inverse_row.iter().copied()) + { + poller.check(0).map_err(|_| Error::Cancelled)?; + *value ^= gf.mul(factor, pivot); + } + } + } + Ok(inverse) +} + +#[cfg(test)] +fn apply_inverse_matrix(gf: &Gf16, inverse: &[Vec], rhs: &[u16]) -> Result> { + apply_inverse_matrix_with_control( + gf, + inverse, + rhs, + &crate::read_control::ReadControl::default(), + ) +} + +fn apply_inverse_matrix_with_control( + gf: &Gf16, + inverse: &[Vec], + rhs: &[u16], + control: &crate::read_control::ReadControl, +) -> Result> { + check_repair(control)?; + if inverse.len() != rhs.len() || inverse.iter().any(|row| row.len() != rhs.len()) { + return Err(Error::BadRecoveryChunk); + } + let mut output = Vec::with_capacity(rhs.len()); + let mut poller = control.poller(); + for row in inverse { + let mut sum = 0; + for (&coefficient, &value) in row.iter().zip(rhs) { + poller.check(0).map_err(|_| Error::Cancelled)?; + sum ^= gf.mul(coefficient, value); + } + output.push(sum); + } + Ok(output) +} + +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Gf16 { + exp: Box<[u16]>, + log: Box<[u32]>, +} + +impl Gf16 { + pub fn new() -> Self { + let mut exp = vec![0u16; FIELD_SIZE * 4 + 1]; + let mut log = vec![0u32; FIELD_SIZE + 1]; + initialize_field(&mut exp, &mut log); + Self { + exp: exp.into_boxed_slice(), + log: log.into_boxed_slice(), + } + } + + pub fn add(&self, left: u16, right: u16) -> u16 { + left ^ right + } + + fn view(&self) -> GfView<'_> { + GfView { + exp: &self.exp, + log: &self.log, + } + } + pub fn mul(&self, left: u16, right: u16) -> u16 { + self.view().mul(left, right) + } + pub fn inv(&self, value: u16) -> Result { + self.view().inv(value) + } + + pub fn div(&self, numerator: u16, denominator: u16) -> Result { + Ok(self.mul(numerator, self.inv(denominator)?)) + } +} + +fn initialize_field(exp: &mut [u16], log: &mut [u32]) { + let mut value = 1u32; + for power in 0..FIELD_SIZE { + log[value as usize] = power as u32; + exp[power] = value as u16; + exp[power + FIELD_SIZE] = value as u16; + value <<= 1; + if value > FIELD_MASK { + value ^= PRIMITIVE_POLYNOMIAL; + } + } + log[0] = ZERO_LOG_SENTINEL; +} +struct GfView<'a> { + exp: &'a [u16], + log: &'a [u32], +} +impl GfView<'_> { + #[inline] + fn mul(&self, left: u16, right: u16) -> u16 { + if left == 0 || right == 0 { + return 0; + } + let index = self.log[left as usize] + self.log[right as usize]; + self.exp[index as usize] + } + + #[inline] + fn inv(&self, value: u16) -> Result { + if value == 0 { + return Err(Error::SingularElement); + } + let index = FIELD_SIZE as u32 - self.log[value as usize]; + Ok(self.exp[index as usize]) + } +} + +impl Default for Gf16 { + fn default() -> Self { + Self::new() + } +} + +pub fn make_encoder_matrix(data_shards: usize, recovery_shards: usize) -> Result>> { + if data_shards == 0 + || recovery_shards == 0 + || data_shards + .checked_add(recovery_shards) + .is_none_or(|total| total > FIELD_SIZE) + { + return Err(Error::TooManyShards); + } + let gf = shared_gf16(); + let mut matrix = vec![vec![0u16; data_shards]; recovery_shards]; + for (i, row) in matrix.iter_mut().enumerate() { + for (j, cell) in row.iter_mut().enumerate() { + let denominator = ((i + data_shards) ^ j) as u16; + *cell = gf.inv(denominator)?; + } + } + Ok(matrix) +} + +pub fn encode_parity_shards(data: &[&[u8]], recovery_shards: usize) -> Result>> { + let Some(first) = data.first() else { + return Err(Error::TooManyShards); + }; + if !first.len().is_multiple_of(2) { + return Err(Error::OddShardSize); + } + if data.iter().any(|shard| shard.len() != first.len()) { + return Err(Error::ShardSizeMismatch); + } + + let matrix = make_encoder_matrix(data.len(), recovery_shards)?; + let gf = shared_gf16(); + let mut parity = vec![vec![0u8; first.len()]; recovery_shards]; + for (recovery_index, row) in matrix.iter().enumerate() { + for word_offset in (0..first.len()).step_by(2) { + let mut symbol = 0u16; + for (data_index, shard) in data.iter().enumerate() { + let data_symbol = u16::from_le_bytes([shard[word_offset], shard[word_offset + 1]]); + symbol ^= gf.mul(row[data_index], data_symbol); + } + parity[recovery_index][word_offset..word_offset + 2] + .copy_from_slice(&symbol.to_le_bytes()); + } + } + Ok(parity) +} + +/// The pre-streaming recovery writer, kept verbatim as the oracle for +/// byte-identity tests. If these two ever disagree, the streaming rewrite has +/// changed the format. +#[cfg(test)] +mod legacy_reference { + use super::*; + + pub(super) fn build_structural_inline_recovery_data( + archive_prefix: &[u8], + recovery_percent: u64, + ) -> Result> { + let (plan, parity) = encode_inline_recovery_parity(archive_prefix, recovery_percent)?; + let shard_ranges = split_prefix_shard_ranges(archive_prefix.len(), plan)?; + let total_len = usize::try_from(plan.payload_size()?).map_err(|_| Error::PlanOverflow)?; + let header_size = usize::try_from(plan.header_size).map_err(|_| Error::PlanOverflow)?; + let shard_size = usize::try_from(plan.shard_size).map_err(|_| Error::PlanOverflow)?; + let data_shards = usize::try_from(plan.data_shards).map_err(|_| Error::PlanOverflow)?; + let recovery_shards = + usize::try_from(plan.recovery_shards).map_err(|_| Error::PlanOverflow)?; + let total_size = u32::try_from(plan.shard_size).map_err(|_| Error::PlanOverflow)?; + let header_size_u32 = u32::try_from(plan.header_size).map_err(|_| Error::PlanOverflow)?; + let data_shards_u16 = u16::try_from(plan.data_shards).map_err(|_| Error::PlanOverflow)?; + let recovery_shards_u16 = + u16::try_from(plan.recovery_shards).map_err(|_| Error::PlanOverflow)?; + let chunk_data_extent = shard_ranges.last().map_or(0usize, std::ops::Range::len); + let chunk_data_extent_u32 = + u32::try_from(chunk_data_extent).map_err(|_| Error::PlanOverflow)?; + let data_shard_states: Vec = shard_ranges + .iter() + .map(|range| crc64_rar_state(&archive_prefix[range.clone()])) + .collect(); + let final_state = parity + .first() + .map(|payload| crc64_rar_state(payload)) + .unwrap_or(0); + + let mut out = Vec::with_capacity(total_len); + for (shard_index, payload) in parity.iter().enumerate() { + if payload.len() + header_size != shard_size { + return Err(Error::PlanOverflow); + } + + let chunk_start = out.len(); + out.extend_from_slice(b"{RB}"); + out.extend_from_slice(&0u64.to_le_bytes()); + out.extend_from_slice(&total_size.to_le_bytes()); + out.extend_from_slice(&header_size_u32.to_le_bytes()); + out.push(1); + out.push(1); + out.extend_from_slice(&0u64.to_le_bytes()); + out.extend_from_slice(&chunk_data_extent_u32.to_le_bytes()); + out.extend_from_slice(&(archive_prefix.len() as u64).to_le_bytes()); + out.extend_from_slice(&plan.group_count.to_le_bytes()); + out.extend_from_slice(&plan.shard_size.to_le_bytes()); + out.extend_from_slice(&data_shards_u16.to_le_bytes()); + out.extend_from_slice(&recovery_shards_u16.to_le_bytes()); + out.extend_from_slice( + &u16::try_from(shard_index) + .map_err(|_| Error::PlanOverflow)? + .to_le_bytes(), + ); + for &state in &data_shard_states { + out.extend_from_slice(&state.to_le_bytes()); + } + out.extend_from_slice(&final_state.to_le_bytes()); + if out.len() - chunk_start != header_size { + return Err(Error::PlanOverflow); + } + out.extend_from_slice(payload); + if out.len() - chunk_start != shard_size { + return Err(Error::PlanOverflow); + } + + let chunk_end = chunk_start + .checked_add(shard_size) + .ok_or(Error::PlanOverflow)?; + let crc_start = chunk_start.checked_add(0x0c).ok_or(Error::PlanOverflow)?; + let crc = crc64_xz(out.get(crc_start..chunk_end).ok_or(Error::PlanOverflow)?); + let crc_field_start = chunk_start.checked_add(0x04).ok_or(Error::PlanOverflow)?; + let crc_field_end = chunk_start.checked_add(0x0c).ok_or(Error::PlanOverflow)?; + out.get_mut(crc_field_start..crc_field_end) + .ok_or(Error::PlanOverflow)? + .copy_from_slice(&crc.to_le_bytes()); + } + if out.len() != total_len { + return Err(Error::PlanOverflow); + } + debug_assert_eq!(parity.len(), recovery_shards); + debug_assert_eq!(data_shard_states.len(), data_shards); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::{ + apply_inverse_matrix, build_inline_recovery_data_for_plan, + build_structural_inline_recovery_data, crc64_rar_state, crc64_xz, + encode_inline_recovery_parity, encode_parity_shards, invert_linear_system_matrix, + make_encoder_matrix, plan_inline_recovery, reconstruct_data_shards, recovery_equations, + repair_inline_recovery_archive, repair_inline_recovery_prefix, + repair_inline_recovery_prefix_shards, shared_gf16, split_prefix_shard_ranges, + split_prefix_shards, Error, Gf16, InlineRecoveryPlan, MAX_WINRAR602_DATA_SHARDS, + }; + + fn recovery_test_bytes(len: usize, seed: u32) -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|index| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + if index % 5 == 0 { + (index % 251) as u8 + } else { + (state >> 24) as u8 + } + }) + .collect() + } + + /// Sizes and percentages that between them cover an empty body, odd tails, + /// the sub-200 KiB shard formula, the 200-shard switchover, and a body big + /// enough to need many stripes. + const RECOVERY_CASES: &[usize] = &[ + 0, + 1, + 2, + 3, + 1023, + 1024, + 1025, + 2050, + 200 * 1024 - 1, + 200 * 1024, + 200 * 1024 + 1, + 1024 * 1024 + 7, + ]; + + #[test] + fn aggregate_recovery_planning_selects_stripes_beside_retained_capacity() { + let data = vec![7; 131072]; + let plan = super::plan_inline_recovery(data.len() as u64, 100).unwrap(); + let ledger = super::Allowance::limited(900000); + let retained = super::Buffer::filled(4096, 0u8, &ledger).unwrap(); + let (mode, required) = + super::choose_recovery_capacity_mode(plan, 8 * 1024 * 1024, &ledger).unwrap(); + assert!(matches!(mode, super::RecoveryMemoryMode::Striped { .. })); + assert!(required <= 8 * 1024 * 1024); + let mut expected = Vec::new(); + super::build_streamed_inline_recovery( + &mut std::io::Cursor::new(&data), + data.len() as u64, + 100, + super::RecoveryMemoryMode::Resident, + None, + &mut expected, + None, + 1, + ) + .unwrap(); + let mut actual = Vec::new(); + super::streamed_recovery_with_allowance( + &mut std::io::Cursor::new(&data), + data.len() as u64, + plan, + mode, + Some(&mut std::io::Cursor::new(Vec::new())), + &mut actual, + None, + 1, + &ledger, + ) + .unwrap(); + assert_eq!(actual, expected); + assert_eq!(ledger.used(), 4096); + drop(retained); + assert_eq!(ledger.used(), 0); + let small = super::Allowance::limited(1); + let error = + super::choose_recovery_capacity_mode(plan, 8 * 1024 * 1024, &small).unwrap_err(); + assert_eq!( + crate::Error::from(error).kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(small.used(), 0); + } + + #[test] + fn recovery_allowance_covers_tables_parity_and_framing_in_both_modes() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::Cursor; + let data = recovery_test_bytes(200_003, 17); + let plan = plan_inline_recovery(data.len() as u64, 10).unwrap(); + let expected = + super::legacy_reference::build_structural_inline_recovery_data(&data, 10).unwrap(); + let field_bytes = ((super::FIELD_SIZE * 4 + 1) * 2 + (super::FIELD_SIZE + 1) * 4) as u64; + for mode in [ + RecoveryMemoryMode::Resident, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + ] { + let mut successes = 0; + let mut refusals = 0; + for limit in [ + 0, + field_bytes - 1, + field_bytes, + field_bytes + 2048, + field_bytes + 16384, + field_bytes + 131072, + ] { + let allowance = Allowance::limited(limit); + let mut body = Cursor::new(&data); + let mut scratch = Cursor::new(Vec::new()); + let mut output = Vec::new(); + let result = streamed_recovery_with_allowance( + &mut body, + data.len() as u64, + plan, + mode, + Some(&mut scratch), + &mut output, + None, + 0, + &allowance, + ); + match result { + Ok(built) => { + successes += 1; + assert_eq!(output, expected); + assert_eq!(built.payload_len, output.len() as u64); + assert_eq!(built.payload_crc32, crate::crc32::crc32(&output)); + } + Err(error) => { + refusals += 1; + assert_eq!( + crate::Error::from(error).kind(), + crate::ErrorKind::ResourceLimit + ); + if limit <= field_bytes { + assert_eq!(body.position(), 0); + assert!(output.is_empty()); + } + } + } + assert_eq!(allowance.used(), 0, "{mode:?} limit={limit}"); + } + assert!(successes > 0 && refusals >= 3); + } + } + + #[test] + fn recovery_allowance_releases_every_owner_on_cancellation_and_io_failure() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::{Cursor, Read, Seek, SeekFrom, Write}; + use std::sync::atomic::{AtomicBool, Ordering}; + struct Progress { + cancelled: AtomicBool, + finished: AtomicBool, + } + impl crate::WriteProgress for Progress { + fn report(&self, event: crate::WriteProgressEvent<'_>) { + match event { + crate::WriteProgressEvent::Advanced { .. } => { + self.cancelled.store(true, Ordering::Relaxed) + } + crate::WriteProgressEvent::OperationFinished { .. } => { + self.finished.store(true, Ordering::Relaxed) + } + _ => {} + } + } + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } + } + struct FailedScratch; + impl Read for FailedScratch { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + unreachable!() + } + } + impl Seek for FailedScratch { + fn seek(&mut self, _: SeekFrom) -> std::io::Result { + Ok(0) + } + } + impl Write for FailedScratch { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let data = recovery_test_bytes(200_003, 17); + let plan = plan_inline_recovery(data.len() as u64, 10).unwrap(); + let allowance = Allowance::limited(2 * 1048576); + for mode in [ + RecoveryMemoryMode::Resident, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + ] { + for initially_cancelled in [false, true] { + let progress = Progress { + cancelled: AtomicBool::new(initially_cancelled), + finished: AtomicBool::new(false), + }; + let mut body = Cursor::new(&data); + let mut output = Vec::new(); + let result = streamed_recovery_with_allowance( + &mut body, + data.len() as u64, + plan, + mode, + Some(&mut Cursor::new(Vec::new())), + &mut output, + Some(crate::progress::ProgressReporter(&progress)), + 0, + &allowance, + ); + assert!(matches!(result, Err(Error::Cancelled))); + assert!(!progress.finished.load(Ordering::Relaxed)); + assert!(output.is_empty()); + if initially_cancelled { + assert_eq!(body.position(), 0); + } + assert_eq!(allowance.used(), 0); + } + let error = streamed_recovery_with_allowance( + &mut Cursor::new(&data[..7]), + data.len() as u64, + plan, + mode, + Some(&mut Cursor::new(Vec::new())), + &mut Vec::new(), + None, + 0, + &allowance, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::Io(std::io::ErrorKind::UnexpectedEof) + )); + assert_eq!(allowance.used(), 0); + let error = streamed_recovery_with_allowance( + &mut Cursor::new(&data), + data.len() as u64, + plan, + mode, + Some(&mut FailedScratch), + &mut FailedScratch, + None, + 0, + &allowance, + ) + .unwrap_err(); + assert!(matches!(error, Error::Io(std::io::ErrorKind::BrokenPipe))); + assert_eq!(allowance.used(), 0); + let mut output = Vec::new(); + streamed_recovery_with_allowance( + &mut Cursor::new(&data), + data.len() as u64, + plan, + mode, + Some(&mut Cursor::new(Vec::new())), + &mut output, + None, + 0, + &allowance, + ) + .unwrap(); + assert!(!output.is_empty()); + assert_eq!(allowance.used(), 0); + } + } + + #[test] + fn streamed_recovery_reports_parity_sink_failure_after_header() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::{Cursor, Write}; + + struct HeaderOnlySink { + header_len: usize, + written: usize, + } + impl Write for HeaderOnlySink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.written == self.header_len { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + let count = bytes.len().min(self.header_len - self.written); + self.written += count; + Ok(count) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let body = vec![0x5a; 4096]; + let plan = plan_inline_recovery(body.len() as u64, 10).unwrap(); + for mode in [ + RecoveryMemoryMode::Resident, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + ] { + let allowance = Allowance::limited(2 * 1048576); + let mut sink = HeaderOnlySink { + header_len: plan.header_size as usize, + written: 0, + }; + let error = streamed_recovery_with_allowance( + &mut Cursor::new(&body), + body.len() as u64, + plan, + mode, + Some(&mut Cursor::new(Vec::new())), + &mut sink, + None, + 0, + &allowance, + ) + .unwrap_err(); + assert_eq!(error, Error::Io(std::io::ErrorKind::BrokenPipe)); + assert_eq!(sink.written, plan.header_size as usize); + assert_eq!(allowance.used(), 0); + } + } + + #[test] + fn striped_recovery_stops_writing_after_mid_row_sink_failure() { + use std::io::{Cursor, Write}; + + struct HeaderSink(usize); + impl Write for HeaderSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.0 != 0 { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + self.0 += bytes.len(); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let body = vec![7; super::RECOVERY_IO_BLOCK + 2]; + let plan = InlineRecoveryPlan { + data_shards: 1, + recovery_shards: 1, + group_count: body.len() as u64, + header_size: 80, + shard_size: 80 + body.len() as u64, + }; + let mut sink = HeaderSink(0); + let error = super::build_streamed_inline_recovery_for_plan( + &mut Cursor::new(&body), + body.len() as u64, + plan, + super::RecoveryMemoryMode::Striped { stripe_len: 4096 }, + Some(&mut Cursor::new(Vec::new())), + &mut sink, + None, + 1, + ) + .unwrap_err(); + assert_eq!(error, Error::Io(std::io::ErrorKind::BrokenPipe)); + assert_eq!(sink.0, plan.header_size as usize); + } + + #[test] + fn striped_recovery_reports_scratch_read_failure_before_output() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::{Cursor, Read, Seek, SeekFrom, Write}; + + struct UnreadableScratch(Cursor>); + impl Read for UnreadableScratch { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + } + impl Write for UnreadableScratch { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.write(bytes) + } + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } + } + impl Seek for UnreadableScratch { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.0.seek(pos) + } + } + + let body = vec![0x5a; 4096]; + let plan = plan_inline_recovery(body.len() as u64, 10).unwrap(); + let allowance = Allowance::limited(2 * 1048576); + let mut output = Vec::new(); + let error = streamed_recovery_with_allowance( + &mut Cursor::new(&body), + body.len() as u64, + plan, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + Some(&mut UnreadableScratch(Cursor::new(Vec::new()))), + &mut output, + None, + 0, + &allowance, + ) + .unwrap_err(); + assert_eq!(error, Error::Io(std::io::ErrorKind::BrokenPipe)); + assert!(output.is_empty()); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn striped_recovery_preserves_scratch_seek_failures() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::{Cursor, Read, Seek, SeekFrom, Write}; + + struct FaultScratch { + data: Cursor>, + reads: usize, + fail_initial_seek: bool, + } + impl Read for FaultScratch { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + self.reads += 1; + self.data.read(bytes) + } + } + impl Write for FaultScratch { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.data.write(bytes) + } + fn flush(&mut self) -> std::io::Result<()> { + self.data.flush() + } + } + impl Seek for FaultScratch { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + if self.fail_initial_seek || self.reads != 0 { + return Err(std::io::ErrorKind::PermissionDenied.into()); + } + self.data.seek(from) + } + } + + let body = vec![0x5a; 4096]; + let plan = plan_inline_recovery(body.len() as u64, 10).unwrap(); + for fail_initial_seek in [true, false] { + let allowance = Allowance::limited(2 * 1048576); + let mut scratch = FaultScratch { + data: Cursor::new(Vec::new()), + reads: 0, + fail_initial_seek, + }; + let mut output = Vec::new(); + let error = streamed_recovery_with_allowance( + &mut Cursor::new(&body), + body.len() as u64, + plan, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + Some(&mut scratch), + &mut output, + None, + 0, + &allowance, + ) + .unwrap_err(); + assert_eq!(error, Error::Io(std::io::ErrorKind::PermissionDenied)); + assert_eq!(scratch.reads == 0, fail_initial_seek); + assert!(output.is_empty()); + assert_eq!(allowance.used(), 0); + } + } + + #[test] + fn recovery_encoder_preserves_body_seek_failures() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::{Cursor, Read, Seek, SeekFrom}; + + struct UnseekableBody(Cursor>); + impl Read for UnseekableBody { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + self.0.read(bytes) + } + } + impl Seek for UnseekableBody { + fn seek(&mut self, _: SeekFrom) -> std::io::Result { + Err(std::io::ErrorKind::PermissionDenied.into()) + } + } + + let body = vec![0x5a; 4096]; + let plan = plan_inline_recovery(body.len() as u64, 10).unwrap(); + for mode in [ + RecoveryMemoryMode::Resident, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + ] { + let allowance = Allowance::limited(2 * 1048576); + let mut output = Vec::new(); + let error = streamed_recovery_with_allowance( + &mut UnseekableBody(Cursor::new(body.clone())), + body.len() as u64, + plan, + mode, + Some(&mut Cursor::new(Vec::new())), + &mut output, + None, + 0, + &allowance, + ) + .unwrap_err(); + assert_eq!(error, Error::Io(std::io::ErrorKind::PermissionDenied)); + assert!(output.is_empty()); + assert_eq!(allowance.used(), 0); + } + } + + #[test] + fn streamed_recovery_rejects_invalid_plan_before_output() { + use super::{streamed_recovery_with_allowance, Allowance, RecoveryMemoryMode}; + use std::io::Cursor; + + let body = vec![0x5a; 4096]; + let valid = plan_inline_recovery(body.len() as u64, 10).unwrap(); + let allowance = Allowance::limited(2 * 1048576); + for (name, plan, expected) in [ + ( + "payload length overflow", + InlineRecoveryPlan { + recovery_shards: u64::MAX, + ..valid + }, + Error::PlanOverflow, + ), + ( + "header state multiplication overflow", + InlineRecoveryPlan { + data_shards: u64::MAX, + ..valid + }, + Error::PlanOverflow, + ), + ( + "header state addition overflow", + InlineRecoveryPlan { + data_shards: (u64::MAX - 0x48) / 8 + 1, + ..valid + }, + Error::PlanOverflow, + ), + ( + "shard length addition overflow", + InlineRecoveryPlan { + group_count: u64::MAX, + ..valid + }, + Error::PlanOverflow, + ), + ( + "u32 shard length", + InlineRecoveryPlan { + group_count: u32::MAX as u64, + shard_size: u32::MAX as u64 + valid.header_size, + ..valid + }, + Error::PlanOverflow, + ), + ( + "u16 data count", + InlineRecoveryPlan { + data_shards: u16::MAX as u64 + 1, + group_count: 0, + header_size: 0x48 + 8 * (u16::MAX as u64 + 1), + shard_size: 0x48 + 8 * (u16::MAX as u64 + 1), + ..valid + }, + Error::PlanOverflow, + ), + ( + "u16 recovery count", + InlineRecoveryPlan { + recovery_shards: u16::MAX as u64 + 1, + ..valid + }, + Error::PlanOverflow, + ), + ( + "header length", + InlineRecoveryPlan { + header_size: valid.header_size + 1, + ..valid + }, + Error::PlanOverflow, + ), + ( + "shard length", + InlineRecoveryPlan { + shard_size: valid.shard_size + 1, + ..valid + }, + Error::PlanOverflow, + ), + ( + "prefix capacity", + InlineRecoveryPlan { + group_count: 2, + shard_size: valid.header_size + 2, + ..valid + }, + Error::PrefixExceedsPlan, + ), + ( + "zero recovery shards", + InlineRecoveryPlan { + recovery_shards: 0, + ..valid + }, + Error::TooManyShards, + ), + ] { + let mut output = Vec::new(); + let result = streamed_recovery_with_allowance( + &mut Cursor::new(&body), + body.len() as u64, + plan, + RecoveryMemoryMode::Resident, + None, + &mut output, + None, + 0, + &allowance, + ); + assert_eq!(result.unwrap_err(), expected, "{name}"); + assert!(output.is_empty(), "{name}"); + assert_eq!(allowance.used(), 0, "{name}"); + } + let mut output = Vec::new(); + assert_eq!( + streamed_recovery_with_allowance( + &mut Cursor::new(&body), + body.len() as u64, + valid, + RecoveryMemoryMode::Striped { stripe_len: 64 }, + None, + &mut output, + None, + 0, + &allowance, + ) + .unwrap_err(), + Error::PlanOverflow + ); + assert!(output.is_empty()); + assert_eq!(allowance.used(), 0); + } + + #[test] + fn rar5_record_builder_preserves_plan_errors() { + let plan = InlineRecoveryPlan { + data_shards: 1, + recovery_shards: 1, + group_count: 0, + header_size: 80, + shard_size: 80, + }; + assert_eq!( + super::build_inline_recovery_data_for_plan_with_control( + b"abc", + plan, + &crate::read_control::ReadControl::default() + ), + Err(Error::PrefixExceedsPlan) + ); + } + + #[test] + fn streamed_recovery_reports_progress_through_completion() { + use std::io::Cursor; + use std::sync::Mutex; + + #[derive(Default)] + struct Progress(Mutex>); + impl crate::WriteProgress for Progress { + fn report(&self, event: crate::WriteProgressEvent<'_>) { + let item = match event { + crate::WriteProgressEvent::OperationStarted { + total_bytes: Some(total), + .. + } => (0, total), + crate::WriteProgressEvent::Advanced { + completed_bytes, .. + } => (1, completed_bytes), + crate::WriteProgressEvent::OperationFinished { + total_bytes: Some(total), + .. + } => (2, total), + _ => panic!("unexpected recovery progress event"), + }; + self.0.lock().unwrap().push(item); + } + } + + let body = vec![0x5a; 4096]; + let progress = Progress::default(); + let mut output = Vec::new(); + let result = super::build_streamed_inline_recovery( + &mut Cursor::new(&body), + body.len() as u64, + 10, + super::RecoveryMemoryMode::Resident, + None, + &mut output, + Some(crate::progress::ProgressReporter(&progress)), + 3, + ) + .unwrap(); + let events = progress.0.into_inner().unwrap(); + assert_eq!(events.first(), Some(&(0, result.payload_len))); + assert_eq!(events.last(), Some(&(2, result.payload_len))); + assert!(events[1..events.len() - 1] + .iter() + .all(|&(kind, bytes)| kind == 1 && bytes <= result.payload_len)); + assert!(events.len() > 2); + assert_eq!(output.len() as u64, result.payload_len); + } + + #[test] + fn streamed_recovery_handles_empty_and_unused_data_shards() { + use std::io::Cursor; + + for (body, plan) in [ + ( + &b""[..], + InlineRecoveryPlan { + data_shards: 1, + recovery_shards: 1, + group_count: 2, + header_size: 80, + shard_size: 82, + }, + ), + ( + &b"abc"[..], + InlineRecoveryPlan { + data_shards: 3, + recovery_shards: 1, + group_count: 4, + header_size: 96, + shard_size: 100, + }, + ), + ] { + let mut output = Vec::new(); + super::build_streamed_inline_recovery_for_plan( + &mut Cursor::new(body), + body.len() as u64, + plan, + super::RecoveryMemoryMode::Resident, + None, + &mut output, + None, + 1, + ) + .unwrap(); + assert_eq!( + repair_inline_recovery_prefix(body, &output), + Ok(body.to_vec()) + ); + } + } + + #[test] + fn streamed_recovery_matches_the_legacy_writer_byte_for_byte() { + for &len in RECOVERY_CASES { + let body = recovery_test_bytes(len, len as u32); + // High percentages on the large cases cost O(shards^2) work for no + // extra coverage: the geometry is already pinned by the small ones. + let percents: &[u64] = if len < 200 * 1024 { + &[1, 5, 10, 50, 100] + } else { + &[1, 10] + }; + for &percent in percents { + let expected = + super::legacy_reference::build_structural_inline_recovery_data(&body, percent) + .unwrap(); + let produced = build_structural_inline_recovery_data(&body, percent).unwrap(); + assert_eq!( + produced, expected, + "streamed output differs for {len} bytes at {percent}%" + ); + } + } + } + + #[test] + fn striped_recovery_matches_resident_recovery_byte_for_byte() { + // Small bodies get a deliberately tiny stripe so stripe boundaries land + // everywhere; large ones use a realistic stripe, since their job is to + // cover the 200-shard geometry rather than boundary arithmetic. + for &len in RECOVERY_CASES { + let body = recovery_test_bytes(len, len as u32 ^ 0x5555); + let (stripe_len, percents): (usize, &[u64]) = if len <= 4096 { + (64, &[1, 10, 100]) + } else { + (4096, &[1, 10]) + }; + for &percent in percents { + let expected = build_structural_inline_recovery_data(&body, percent).unwrap(); + + let plan = plan_inline_recovery(body.len() as u64, percent).unwrap(); + let mut scratch = std::io::Cursor::new(vec![ + 0u8; + (plan.recovery_shards * plan.group_count) + as usize + ]); + let mut produced = Vec::new(); + let output = super::build_streamed_inline_recovery( + &mut std::io::Cursor::new(&body), + body.len() as u64, + percent, + super::RecoveryMemoryMode::Striped { stripe_len }, + Some(&mut scratch), + &mut produced, + None, + 1, + ) + .unwrap(); + + assert_eq!( + produced, expected, + "striped output differs for {len} bytes at {percent}%" + ); + assert_eq!(output.plan, plan); + assert_eq!(output.payload_len, produced.len() as u64); + assert_eq!(output.payload_crc32, crate::crc32::crc32(&produced)); + } + } + } + + #[test] + fn recovery_memory_mode_switches_to_striping_under_pressure() { + let plan = plan_inline_recovery(4 * 1024 * 1024, 10).unwrap(); + let parity_bytes = plan.recovery_shards * plan.group_count; + + let (mode, required) = + super::choose_recovery_memory_mode(plan, parity_bytes + 1024 * 1024).unwrap(); + assert_eq!(mode, super::RecoveryMemoryMode::Resident); + assert!(required >= parity_bytes); + + let limit = parity_bytes / 2; + let (mode, required) = super::choose_recovery_memory_mode(plan, limit).unwrap(); + let super::RecoveryMemoryMode::Striped { stripe_len } = mode else { + panic!("expected striped mode when parity does not fit"); + }; + assert!(stripe_len.is_multiple_of(2), "stripes must be word aligned"); + assert!( + required <= limit, + "striping must fit the budget it was given: {required} > {limit}" + ); + assert!(required < parity_bytes); + assert_eq!( + super::choose_recovery_memory_mode( + InlineRecoveryPlan { + recovery_shards: u64::MAX, + group_count: 2, + ..plan + }, + u64::MAX, + ), + Err(Error::PlanOverflow) + ); + let allowance = super::Allowance::limited(2 * 1048576); + assert_eq!( + super::choose_recovery_capacity_mode( + InlineRecoveryPlan { + recovery_shards: u64::MAX, + group_count: 2, + ..plan + }, + u64::MAX, + &allowance, + ), + Err(Error::PlanOverflow) + ); + } + + #[test] + fn rar5_inline_recovery_plan_matches_fixture_formula_examples() { + assert_eq!( + plan_inline_recovery(65_681, 5).unwrap(), + InlineRecoveryPlan { + data_shards: 65, + recovery_shards: 3, + group_count: 1012, + header_size: 592, + shard_size: 1604, + } + ); + assert_eq!( + plan_inline_recovery(65_681, 20).unwrap(), + InlineRecoveryPlan { + data_shards: 65, + recovery_shards: 13, + group_count: 1012, + header_size: 592, + shard_size: 1604, + } + ); + } + + #[test] + fn rar5_inline_recovery_plan_handles_clamps_and_large_prefixes() { + assert_eq!( + plan_inline_recovery(0, 0).unwrap(), + InlineRecoveryPlan { + data_shards: 1, + recovery_shards: 1, + group_count: 0, + header_size: 80, + shard_size: 80, + } + ); + assert_eq!( + plan_inline_recovery(200 * 1024, 1000).unwrap(), + InlineRecoveryPlan { + data_shards: 200, + recovery_shards: 200, + group_count: 1024, + header_size: 1672, + shard_size: 2696, + } + ); + let largest = plan_inline_recovery(u64::MAX, 1000).unwrap(); + assert_eq!(largest.data_shards, MAX_WINRAR602_DATA_SHARDS); + assert_eq!(largest.header_size, 1672); + assert_eq!(largest.group_count, u64::MAX.div_ceil(200) + 1); + assert_eq!( + largest.shard_size, + largest.header_size + largest.group_count + ); + } + + #[test] + fn rar5_inline_recovery_plan_keeps_fixed_header_above_64k_groups() { + let boundary = MAX_WINRAR602_DATA_SHARDS * 0x10000; + assert_eq!( + plan_inline_recovery(boundary, 1).unwrap(), + InlineRecoveryPlan { + data_shards: 200, + recovery_shards: 2, + group_count: 65_536, + header_size: 1672, + shard_size: 67_208, + } + ); + assert_eq!( + plan_inline_recovery(boundary + 1, 1).unwrap(), + InlineRecoveryPlan { + data_shards: 200, + recovery_shards: 2, + group_count: 65_538, + header_size: 1672, + shard_size: 67_210, + } + ); + } + + #[test] + fn rar5_recovery_planning_handles_zero_percent_and_tight_budgets() { + let large = plan_inline_recovery(200 * 1024, 0).unwrap(); + assert_eq!(large.recovery_shards, 0); + let body = vec![0; 200 * 1024]; + assert_eq!( + encode_inline_recovery_parity(&body, 0), + Err(Error::TooManyShards) + ); + assert_eq!( + build_structural_inline_recovery_data(&body, 0), + Err(Error::TooManyShards) + ); + + let plan = plan_inline_recovery(4096, 10).unwrap(); + let (mode, _) = super::choose_recovery_memory_mode(plan, 0).unwrap(); + assert!(matches!(mode, super::RecoveryMemoryMode::Striped { .. })); + + let allowance = super::Allowance::limited(2 * 1048576); + let (mode, required) = + super::choose_recovery_capacity_mode(plan, 2 * 1048576, &allowance).unwrap(); + assert_eq!(mode, super::RecoveryMemoryMode::Resident); + assert!(required <= allowance.available()); + } + + #[test] + fn gf16_matches_rar5_polynomial_wrap() { + let gf = Gf16::new(); + + assert_eq!(gf.mul(0x8000, 2), 0x100b); + assert_eq!(gf.mul(0, 0x1234), 0); + assert_eq!(gf.mul(0x1234, 0), 0); + assert_eq!(gf.mul(0, 0), 0); + assert_eq!(gf.mul(1, 0x1234), 0x1234); + } + + #[test] + fn shared_gf16_reuses_field_tables() { + let first = shared_gf16() as *const Gf16; + let second = shared_gf16() as *const Gf16; + + assert_eq!(first, second); + assert_eq!(shared_gf16().mul(0x8000, 2), 0x100b); + } + + #[test] + fn crc64_xz_matches_reference_vectors() { + assert_eq!(crc64_xz(b""), 0); + assert_eq!(crc64_xz(b"123456789"), 0x995d_c9bb_df19_39fa); + assert_eq!(crc64_xz(b"testtesttest"), 0x7b1c_2d23_0ede_b436); + } + + #[test] + fn raw_crc64_state_matches_reference_vector() { + assert_eq!(crc64_rar_state(b""), 0); + assert_eq!(crc64_rar_state(b"te\x80st"), 0xb5db_f958_3a6e_ed4a); + } + + #[test] + fn rar5_prefix_split_produces_even_padded_data_shards() { + let plan = InlineRecoveryPlan { + data_shards: 3, + recovery_shards: 1, + group_count: 4, + header_size: 96, + shard_size: 100, + }; + let shards = split_prefix_shards(b"abcdefghij", plan).unwrap(); + + assert_eq!( + shards, + vec![b"abcd".to_vec(), b"efgh".to_vec(), b"ij\0\0".to_vec()] + ); + } + + #[test] + fn rar5_prefix_split_rejects_prefix_larger_than_plan_capacity() { + let plan = InlineRecoveryPlan { + data_shards: 2, + recovery_shards: 1, + group_count: 2, + header_size: 88, + shard_size: 90, + }; + + assert_eq!( + split_prefix_shards(b"abcde", plan), + Err(Error::PrefixExceedsPlan) + ); + assert_eq!( + split_prefix_shard_ranges( + 0, + InlineRecoveryPlan { + group_count: usize::MAX as u64, + ..plan + } + ), + Err(Error::PlanOverflow) + ); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn rar5_prefix_split_rejects_unrepresentable_native_geometry() { + let plan = InlineRecoveryPlan { + data_shards: u32::MAX as u64 + 1, + recovery_shards: 1, + group_count: 1, + header_size: 80, + shard_size: 81, + }; + assert_eq!(split_prefix_shard_ranges(0, plan), Err(Error::PlanOverflow)); + assert_eq!( + split_prefix_shard_ranges( + 0, + InlineRecoveryPlan { + data_shards: 1, + group_count: u32::MAX as u64 + 1, + ..plan + } + ), + Err(Error::PlanOverflow) + ); + assert_eq!( + split_prefix_shards( + b"", + InlineRecoveryPlan { + data_shards: 1, + group_count: u32::MAX as u64 + 1, + ..plan + } + ), + Err(Error::PlanOverflow) + ); + } + + #[test] + fn rar5_prefix_split_handles_extra_empty_data_shards() { + let plan = InlineRecoveryPlan { + data_shards: 3, + recovery_shards: 1, + group_count: 4, + header_size: 96, + shard_size: 100, + }; + + assert_eq!( + split_prefix_shard_ranges(3, plan).unwrap(), + vec![0..3, 3..3, 3..3] + ); + assert_eq!( + split_prefix_shards(b"abc", plan).unwrap(), + vec![b"abc\0".to_vec(), vec![0; 4], vec![0; 4]] + ); + } + + #[test] + fn gf16_inverse_round_trips_nonzero_elements() { + let gf = Gf16::new(); + + for value in [1, 2, 3, 0x100b, 0x8000, 0xffff] { + let inverse = gf.inv(value).unwrap(); + assert_eq!(gf.mul(value, inverse), 1); + } + assert_eq!(gf.inv(0), Err(Error::SingularElement)); + } + + #[test] + fn gf16_public_arithmetic_matches_field_identities() { + let gf = Gf16::default(); + assert_eq!(gf.add(0x1234, 0x1234), 0); + assert_eq!(gf.add(0x1234, 0), 0x1234); + assert_eq!(gf.div(0x1234, 1), Ok(0x1234)); + assert_eq!(gf.div(0, 0x1234), Ok(0)); + assert_eq!(gf.div(1, 0), Err(Error::SingularElement)); + } + + #[test] + fn rar5_recovery_errors_display_and_preserve_typed_sources() { + for error in [ + Error::Cancelled, + Error::BadRecoveryChunk, + Error::OddShardSize, + Error::PlanOverflow, + Error::PrefixExceedsPlan, + Error::TooManyDamagedShards, + Error::ShardSizeMismatch, + Error::TooManyShards, + Error::SingularElement, + Error::RebuildTooLarge, + Error::RepairWorkspaceLimitExceeded { + limit: 1, + required: 2, + }, + Error::Io(std::io::ErrorKind::BrokenPipe), + ] { + assert!(!error.to_string().is_empty()); + assert!(std::error::Error::source(&error).is_none()); + } + let typed = Error::TypedIo(Box::new(crate::Error::Cancelled)); + assert_eq!(typed.to_string(), crate::Error::Cancelled.to_string()); + assert!(std::error::Error::source(&typed).is_some()); + } + + #[test] + fn rar5_cauchy_encoder_matrix_uses_inverse_xor_denominators() { + let gf = Gf16::new(); + let matrix = make_encoder_matrix(3, 2).unwrap(); + + assert_eq!(matrix.len(), 2); + assert_eq!(matrix[0].len(), 3); + for (i, row) in matrix.iter().enumerate() { + for (j, &cell) in row.iter().enumerate() { + let denominator = ((i + 3) ^ j) as u16; + assert_eq!(gf.mul(cell, denominator), 1); + } + } + } + + #[test] + fn rar5_recovery_equations_do_not_materialize_unused_rows_or_columns() { + let gf = shared_gf16(); + let equations = recovery_equations( + gf, + 32_000, + &[0, 31_999], + &[(32_000, &[])], + &crate::read_control::ReadControl::default(), + ) + .unwrap(); + assert_eq!( + equations, + vec![vec![ + gf.inv(64_000).unwrap(), + gf.inv(64_000 ^ 31_999).unwrap() + ]] + ); + assert_eq!(equations[0].capacity(), 2); + assert_eq!( + make_encoder_matrix(usize::MAX, 1), + Err(Error::TooManyShards) + ); + } + + #[test] + fn rar5_repair_keeps_empty_shards_and_padding_virtual() { + // A valid sparse record describes 8 MiB of padded data capacity, but + // only one byte of real data and one available high-index parity row. + let data_count = 2048usize; + let row = data_count - 1; + let group_count = 4096usize; + let header_size = 0x48 + data_count * 8; + let shard_size = header_size + group_count; + let mut chunk = vec![0; shard_size]; + chunk[..4].copy_from_slice(b"{RB}"); + chunk[0x0c..0x10].copy_from_slice(&(shard_size as u32).to_le_bytes()); + chunk[0x10..0x14].copy_from_slice(&(header_size as u32).to_le_bytes()); + chunk[0x14] = 1; + chunk[0x15] = 1; + chunk[0x22..0x2a].copy_from_slice(&1u64.to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&(group_count as u64).to_le_bytes()); + chunk[0x32..0x3a].copy_from_slice(&(shard_size as u64).to_le_bytes()); + chunk[0x3a..0x3c].copy_from_slice(&(data_count as u16).to_le_bytes()); + chunk[0x3c..0x3e].copy_from_slice(&(data_count as u16).to_le_bytes()); + chunk[0x3e..0x40].copy_from_slice(&(row as u16).to_le_bytes()); + chunk[0x40..0x48].copy_from_slice(&crc64_rar_state(b"a").to_le_bytes()); + let gf = shared_gf16(); + let parity = gf.mul(gf.inv((data_count + row) as u16).unwrap(), b'a' as u16); + chunk[header_size..header_size + 2].copy_from_slice(&parity.to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[4..0x0c].copy_from_slice(&crc.to_le_bytes()); + assert_eq!(repair_inline_recovery_prefix(b"a", &chunk).unwrap(), b"a"); + assert_eq!(repair_inline_recovery_prefix(b"b", &chunk).unwrap(), b"a"); + let mut empty_reads = 0; + let repaired = repair_inline_recovery_prefix_shards(1, &chunk, |range| { + assert!(range.len() <= 1); + if range.is_empty() { + empty_reads += 1; + } + Ok(b"b"[range].to_vec()) + }) + .unwrap(); + assert_eq!(repaired, vec![(0..1, b"a".to_vec())]); + // Empty columns are checked once and never reread during RHS solving. + assert_eq!(empty_reads, data_count - 1); + } + + #[test] + fn rar5_cauchy_encoder_matrix_rejects_impossible_shard_counts() { + assert_eq!(make_encoder_matrix(0, 1), Err(Error::TooManyShards)); + assert_eq!(make_encoder_matrix(1, 0), Err(Error::TooManyShards)); + assert_eq!(make_encoder_matrix(65535, 1), Err(Error::TooManyShards)); + } + + #[test] + fn rar5_recovery_inverse_matrix_solves_reused_equations() { + let gf = shared_gf16(); + let equations = vec![vec![3, 5], vec![7, 11]]; + let expected = [0x1234, 0xabcd]; + let rhs = equations + .iter() + .map(|row| gf.mul(row[0], expected[0]) ^ gf.mul(row[1], expected[1])) + .collect::>(); + + let inverse = invert_linear_system_matrix(gf, &equations).unwrap(); + let solved = apply_inverse_matrix(gf, &inverse, &rhs).unwrap(); + + assert_eq!(solved, expected); + } + + #[test] + fn rar5_recovery_matrix_rejects_non_square_inputs() { + let gf = shared_gf16(); + assert_eq!( + invert_linear_system_matrix(gf, &[vec![1], vec![1]]), + Err(Error::BadRecoveryChunk) + ); + assert_eq!( + apply_inverse_matrix(gf, &[vec![1]], &[1, 2]), + Err(Error::BadRecoveryChunk) + ); + assert_eq!( + apply_inverse_matrix(gf, &[vec![1], vec![1]], &[1, 2]), + Err(Error::BadRecoveryChunk) + ); + } + + #[test] + fn rar5_recovery_matrix_handles_zero_elimination_factors() { + let gf = shared_gf16(); + assert_eq!( + invert_linear_system_matrix(gf, &[vec![1, 0], vec![0, 1]]), + Ok(vec![vec![1, 0], vec![0, 1]]) + ); + } + + #[test] + fn rar5_budgeted_encoder_rejects_invalid_shard_counts() { + let allowance = super::Allowance::default(); + let gf = shared_gf16().view(); + for (data, recovery) in [(0, 1), (1, 0), (usize::MAX, 1)] { + assert_eq!( + super::encoder_matrix_with_allowance(data, recovery, &gf, &allowance).unwrap_err(), + Error::TooManyShards + ); + } + } + + #[test] + fn rar5_parity_encoder_generates_systematic_recovery_shards() { + let first = [1, 0, 2, 0, 3, 0, 4, 0]; + let parity = encode_parity_shards(&[&first], 1).unwrap(); + + assert_eq!(parity, [first.to_vec()]); + } + + #[test] + fn rar5_parity_encoder_applies_cauchy_matrix_coefficients() { + let gf = Gf16::new(); + let first = [1, 0, 2, 0]; + let second = [3, 0, 4, 0]; + let matrix = make_encoder_matrix(2, 2).unwrap(); + let parity = encode_parity_shards(&[&first, &second], 2).unwrap(); + + for recovery_index in 0..2 { + for word_index in 0..2 { + let offset = word_index * 2; + let left = u16::from_le_bytes([first[offset], first[offset + 1]]); + let right = u16::from_le_bytes([second[offset], second[offset + 1]]); + let expected = gf.mul(matrix[recovery_index][0], left) + ^ gf.mul(matrix[recovery_index][1], right); + assert_eq!( + u16::from_le_bytes([ + parity[recovery_index][offset], + parity[recovery_index][offset + 1], + ]), + expected + ); + } + } + } + + #[test] + fn rar5_inline_recovery_parity_splits_and_encodes_prefix() { + let prefix = b"RAR5 inline recovery parity payload input"; + let (plan, parity) = encode_inline_recovery_parity(prefix, 10).unwrap(); + + assert_eq!(plan, plan_inline_recovery(prefix.len() as u64, 10).unwrap()); + assert_eq!(parity.len(), plan.recovery_shards as usize); + assert!(parity + .iter() + .all(|shard| shard.len() == plan.group_count as usize)); + + let data_shards = split_prefix_shards(prefix, plan).unwrap(); + let shard_refs: Vec<&[u8]> = data_shards.iter().map(Vec::as_slice).collect(); + assert_eq!( + parity, + encode_parity_shards(&shard_refs, plan.recovery_shards as usize).unwrap() + ); + } + + #[test] + fn rar5_structural_inline_recovery_data_writes_chunks_and_crc64() { + let prefix = b"RAR5 structural inline recovery data"; + let (plan, parity) = encode_inline_recovery_parity(prefix, 10).unwrap(); + let data = build_structural_inline_recovery_data(prefix, 10).unwrap(); + + assert_eq!(data.len(), plan.payload_size().unwrap() as usize); + for (shard_index, payload) in parity.iter().enumerate() { + let chunk_start = shard_index * plan.shard_size as usize; + let chunk = &data[chunk_start..chunk_start + plan.shard_size as usize]; + assert_eq!(&chunk[..4], b"{RB}"); + assert_eq!( + u64::from_le_bytes(chunk[4..12].try_into().unwrap()), + crc64_xz(&chunk[0x0c..]) + ); + assert_eq!( + u32::from_le_bytes(chunk[0x0c..0x10].try_into().unwrap()) as u64, + plan.shard_size + ); + assert_eq!( + u32::from_le_bytes(chunk[0x10..0x14].try_into().unwrap()) as u64, + plan.header_size + ); + assert_eq!(chunk[0x14], 1); + assert_eq!(chunk[0x15], 1); + assert_eq!( + u64::from_le_bytes(chunk[0x22..0x2a].try_into().unwrap()), + prefix.len() as u64 + ); + assert_eq!( + u16::from_le_bytes(chunk[0x3e..0x40].try_into().unwrap()) as usize, + shard_index + ); + let shard_ranges = split_prefix_shard_ranges(prefix.len(), plan).unwrap(); + assert_eq!( + u32::from_le_bytes(chunk[0x1e..0x22].try_into().unwrap()) as usize, + shard_ranges.last().unwrap().len() + ); + for (data_index, range) in shard_ranges.iter().enumerate() { + let state_offset = 0x40 + data_index * 8; + assert_eq!( + u64::from_le_bytes(chunk[state_offset..state_offset + 8].try_into().unwrap()), + crc64_rar_state(&prefix[range.clone()]) + ); + } + assert_eq!(&chunk[plan.header_size as usize..], payload); + } + } + + #[test] + fn rar5_structural_inline_recovery_round_trips_above_64k_groups() { + let prefix_len = (MAX_WINRAR602_DATA_SHARDS * 0x10000 + 1) as usize; + let prefix: Vec = (0..prefix_len).map(|index| index as u8).collect(); + let plan = plan_inline_recovery(prefix.len() as u64, 1).unwrap(); + let recovery_data = build_structural_inline_recovery_data(&prefix, 1).unwrap(); + + assert_eq!(plan.header_size, 1672); + assert_eq!(plan.group_count, 65_538); + assert_eq!(recovery_data.len(), plan.payload_size().unwrap() as usize); + for shard_index in 0..plan.recovery_shards as usize { + let chunk_start = shard_index * plan.shard_size as usize; + let chunk_end = chunk_start + plan.shard_size as usize; + let chunk = &recovery_data[chunk_start..chunk_end]; + assert_eq!( + u32::from_le_bytes(chunk[0x0c..0x10].try_into().unwrap()) as u64, + plan.shard_size + ); + assert_eq!( + u32::from_le_bytes(chunk[0x10..0x14].try_into().unwrap()) as u64, + plan.header_size + ); + assert_eq!( + u64::from_le_bytes(chunk[0x04..0x0c].try_into().unwrap()), + crc64_xz(&chunk[0x0c..]) + ); + } + + assert_eq!( + repair_inline_recovery_prefix(&prefix, &recovery_data).unwrap(), + prefix + ); + let mut damaged = prefix.clone(); + damaged[0] ^= 0xff; + assert_eq!( + repair_inline_recovery_prefix(&damaged, &recovery_data).unwrap(), + prefix + ); + } + + #[test] + fn rar5_structural_inline_recovery_uses_shared_final_state() { + let prefix: Vec = (0..(256 * 1024)).map(|index| index as u8).collect(); + let (plan, parity) = encode_inline_recovery_parity(&prefix, 20).unwrap(); + assert!(plan.recovery_shards > 1); + let data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let expected = crc64_rar_state(&parity[0]); + + for shard_index in 0..plan.recovery_shards as usize { + let chunk_start = shard_index * plan.shard_size as usize; + let final_state_offset = chunk_start + 0x40 + plan.data_shards as usize * 8; + assert_eq!( + u64::from_le_bytes( + data[final_state_offset..final_state_offset + 8] + .try_into() + .unwrap() + ), + expected + ); + } + } + + #[test] + fn rar5_inline_recovery_rejects_malformed_chunk_fields() { + let valid = build_structural_inline_recovery_data(b"prefix", 10).unwrap(); + let parse = |chunk: &[u8]| { + super::parse_inline_recovery_chunk_with_control( + chunk, + &crate::read_control::ReadControl::default(), + ) + .unwrap_err() + }; + let mut cases = Vec::new(); + cases.push(("short header", valid[..0x47].to_vec())); + let mut wrong_magic = valid.clone(); + wrong_magic[0] = b'X'; + cases.push(("wrong magic", wrong_magic)); + let mut truncated = valid.clone(); + truncated.pop(); + cases.push(("truncated chunk", truncated)); + let mut bad_crc = valid.clone(); + bad_crc[0x40] ^= 1; + cases.push(("bad CRC", bad_crc)); + + let mut altered = |name, edit: fn(&mut Vec)| { + let mut chunk = valid.clone(); + edit(&mut chunk); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + cases.push((name, chunk)); + }; + altered("short declared header", |chunk| { + chunk[0x10..0x14].copy_from_slice(&0x47u32.to_le_bytes()) + }); + altered("header larger than chunk", |chunk| { + let size = u32::try_from(chunk.len()).unwrap() + 1; + chunk[0x10..0x14].copy_from_slice(&size.to_le_bytes()); + }); + altered("wrong format version", |chunk| chunk[0x14] = 2); + altered("wrong record version", |chunk| chunk[0x15] = 2); + altered("wrong shard size", |chunk| chunk[0x32] ^= 1); + altered("out-of-range shard index", |chunk| chunk[0x3e] = 1); + altered("wrong data-state count", |chunk| chunk[0x3a] = 2); + altered("no data shards", |chunk| { + let parity_len = (chunk.len() - 0x48) as u64; + chunk[0x10..0x14].copy_from_slice(&0x48u32.to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&parity_len.to_le_bytes()); + chunk[0x3a..0x3c].copy_from_slice(&0u16.to_le_bytes()); + }); + altered("wrong parity length", |chunk| chunk[0x2a] ^= 2); + altered("no recovery shards", |chunk| chunk[0x3c..0x3e].fill(0)); + + for (name, chunk) in cases { + assert_eq!(parse(&chunk), Error::BadRecoveryChunk, "{name}"); + } + } + + #[test] + fn rar5_recovery_scan_ignores_truncated_markers() { + for data in [&b""[..], &b"{RB"[..], &b"noise{RB}"[..]] { + assert_eq!( + repair_inline_recovery_prefix(b"prefix", data), + Err(Error::BadRecoveryChunk) + ); + } + let valid = build_structural_inline_recovery_data(b"prefix", 10).unwrap(); + let mut data = b"{RB}".to_vec(); + data.extend_from_slice(&valid); + assert_eq!( + repair_inline_recovery_prefix(b"prefix", &data), + Ok(b"prefix".to_vec()) + ); + } + + #[test] + fn rar5_repair_rejects_odd_gf16_shards() { + let mut chunk = build_structural_inline_recovery_data(b"x", 10).unwrap(); + chunk.pop(); + let size = chunk.len() as u32; + chunk[0x0c..0x10].copy_from_slice(&size.to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&1u64.to_le_bytes()); + chunk[0x32..0x3a].copy_from_slice(&(size as u64).to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + + assert_eq!( + super::parse_inline_recovery_chunk_with_control( + &chunk, + &crate::read_control::ReadControl::default() + ) + .unwrap_err(), + Error::BadRecoveryChunk + ); + assert_eq!( + repair_inline_recovery_prefix(b"y", &chunk), + Err(Error::BadRecoveryChunk) + ); + } + + #[test] + fn rar5_recovery_rejects_shard_counts_above_field_capacity() { + let data_shards = 32_768u16; + let recovery_shards = 32_768u16; + let header_size = 0x48 + data_shards as usize * 8; + let mut chunk = vec![0u8; header_size]; + chunk[..4].copy_from_slice(b"{RB}"); + chunk[0x0c..0x10].copy_from_slice(&(header_size as u32).to_le_bytes()); + chunk[0x10..0x14].copy_from_slice(&(header_size as u32).to_le_bytes()); + chunk[0x14] = 1; + chunk[0x15] = 1; + chunk[0x32..0x3a].copy_from_slice(&(header_size as u64).to_le_bytes()); + chunk[0x3a..0x3c].copy_from_slice(&data_shards.to_le_bytes()); + chunk[0x3c..0x3e].copy_from_slice(&recovery_shards.to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + + assert_eq!( + super::parse_inline_recovery_chunk_with_control( + &chunk, + &crate::read_control::ReadControl::default() + ) + .unwrap_err(), + Error::BadRecoveryChunk + ); + } + + #[test] + fn rar5_repair_rejects_inconsistent_recovery_chunks() { + let prefix = recovery_test_bytes(32_000, 33); + let plan = plan_inline_recovery(prefix.len() as u64, 20).unwrap(); + assert!(plan.recovery_shards > 2); + let valid = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let start = plan.shard_size as usize; + let end = start + plan.shard_size as usize; + for (name, edit) in [ + ("states", 0x40), + ("protected size", 0x22), + ("recovery shard count", 0x3c), + ] { + let mut recovery_data = valid.clone(); + recovery_data[start + edit] ^= 1; + let crc = crc64_xz(&recovery_data[start + 0x0c..end]); + recovery_data[start + 0x04..start + 0x0c].copy_from_slice(&crc.to_le_bytes()); + + assert_eq!( + repair_inline_recovery_prefix(&prefix, &recovery_data), + Err(Error::BadRecoveryChunk), + "{name}" + ); + assert_eq!( + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + Ok(prefix[range].to_vec()) + }), + Err(Error::BadRecoveryChunk), + "{name}" + ); + } + } + + #[test] + fn rar5_prefix_repair_checks_protected_size_and_healthy_ranges() { + let prefix = b"prefix"; + let recovery_data = build_structural_inline_recovery_data(prefix, 10).unwrap(); + assert_eq!( + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + Ok(prefix[range].to_vec()) + }), + Ok(Vec::new()) + ); + assert_eq!( + repair_inline_recovery_prefix(b"prefix!", &recovery_data), + Err(Error::BadRecoveryChunk) + ); + assert_eq!( + repair_inline_recovery_prefix_shards(prefix.len() + 1, &recovery_data, |_| { + panic!("unexpected read after mismatched protected size") + }), + Err(Error::BadRecoveryChunk) + ); + } + + #[test] + fn rar5_range_repair_preserves_short_and_failed_callback_reads() { + let prefix = recovery_test_bytes(32_000, 41); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut damaged = prefix.clone(); + damaged[0] ^= 1; + + let short_first = + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + Ok(damaged[range.start..range.end - 1].to_vec()) + }); + assert_eq!(short_first, Err(Error::ShardSizeMismatch)); + let failed_first = + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |_| { + Err(Error::Io(std::io::ErrorKind::PermissionDenied)) + }); + assert_eq!( + failed_first, + Err(Error::Io(std::io::ErrorKind::PermissionDenied)) + ); + + let mut reads = 0; + let data_shards = plan_inline_recovery(prefix.len() as u64, 20) + .unwrap() + .data_shards as usize; + let short_second = + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + reads += 1; + let end = if reads > data_shards { + range.end - 1 + } else { + range.end + }; + Ok(damaged[range.start..end].to_vec()) + }); + assert_eq!(short_second, Err(Error::ShardSizeMismatch)); + assert!(reads > data_shards); + + reads = 0; + let failed_second = + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + reads += 1; + if reads > data_shards { + Err(Error::Io(std::io::ErrorKind::PermissionDenied)) + } else { + Ok(damaged[range].to_vec()) + } + }); + assert_eq!( + failed_second, + Err(Error::Io(std::io::ErrorKind::PermissionDenied)) + ); + assert!(reads > data_shards); + } + + #[test] + fn rar5_range_repair_rejects_damage_beyond_available_parity() { + let prefix = recovery_test_bytes(10_000, 73); + let plan = plan_inline_recovery(prefix.len() as u64, 10).unwrap(); + assert_eq!(plan.recovery_shards, 1); + let recovery_data = build_structural_inline_recovery_data(&prefix, 10).unwrap(); + let mut damaged = prefix.clone(); + damaged[0] ^= 1; + damaged[plan.group_count as usize] ^= 1; + assert_eq!( + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + Ok(damaged[range].to_vec()) + }), + Err(Error::TooManyDamagedShards) + ); + } + + #[test] + fn rar5_prefix_repair_rejects_duplicate_parity_rows() { + let prefix = recovery_test_bytes(10_000, 83); + let plan = plan_inline_recovery(prefix.len() as u64, 20).unwrap(); + assert!(plan.recovery_shards >= 2); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let first = &recovery_data[..plan.shard_size as usize]; + let duplicated = [first, first].concat(); + let mut damaged = prefix.clone(); + damaged[0] ^= 1; + damaged[plan.group_count as usize] ^= 1; + assert_eq!( + repair_inline_recovery_prefix(&damaged, &duplicated), + Err(Error::SingularElement) + ); + assert_eq!( + repair_inline_recovery_prefix_shards(prefix.len(), &duplicated, |range| { + Ok(damaged[range].to_vec()) + }), + Err(Error::SingularElement) + ); + } + + #[test] + fn rar5_range_repair_restores_short_final_shard() { + let prefix = recovery_test_bytes(10_001, 79); + let plan = plan_inline_recovery(prefix.len() as u64, 20).unwrap(); + let final_start = (plan.data_shards as usize - 1) * plan.group_count as usize; + assert!(final_start < prefix.len()); + assert!((prefix.len() - final_start) % 2 == 1); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut damaged = prefix.clone(); + damaged[final_start] ^= 1; + let repaired = + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + Ok(damaged[range].to_vec()) + }) + .unwrap(); + assert_eq!( + repaired, + vec![(final_start..prefix.len(), prefix[final_start..].to_vec())] + ); + } + + #[test] + fn rar5_parity_encoder_rejects_invalid_shard_shapes() { + assert_eq!(encode_parity_shards(&[], 1), Err(Error::TooManyShards)); + assert_eq!( + encode_parity_shards(&[&[1, 2, 3]], 1), + Err(Error::OddShardSize) + ); + assert_eq!( + encode_parity_shards(&[&[1, 2], &[3, 4, 5, 6]], 1), + Err(Error::ShardSizeMismatch) + ); + } + + #[test] + fn rar5_inline_recovery_repairs_single_damaged_data_shard() { + let prefix: Vec = (0..32_000).map(|index| (index * 17) as u8).collect(); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut damaged = prefix.clone(); + damaged[1500..1537].fill(0xa5); + + let repaired = repair_inline_recovery_prefix(&damaged, &recovery_data).unwrap(); + + assert_eq!(repaired, prefix); + } + + #[test] + fn rar5_inline_recovery_skips_damaged_recovery_chunks_if_enough_survive() { + let prefix: Vec = (0..32_000).map(|index| (index * 13) as u8).collect(); + let mut recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + recovery_data[0x48] ^= 0xff; + let mut damaged = prefix.clone(); + damaged[1024..1300].fill(0xa5); + + let repaired = repair_inline_recovery_prefix(&damaged, &recovery_data).unwrap(); + + assert_eq!(repaired, prefix); + } + + #[test] + fn rar5_inline_recovery_repairs_multiple_damaged_data_shards() { + let prefix: Vec = (0..128_000).map(|index| (index * 31) as u8).collect(); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut damaged = prefix.clone(); + damaged[100..500].fill(0x11); + damaged[4_000..4_400].fill(0x22); + damaged[9_000..9_400].fill(0x33); + + let repaired = repair_inline_recovery_prefix(&damaged, &recovery_data).unwrap(); + + assert_eq!(repaired, prefix); + } + + #[test] + fn rar5_inline_recovery_returns_only_repaired_shard_ranges() { + let prefix: Vec = (0..128_000).map(|index| (index * 29) as u8).collect(); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut damaged = prefix.clone(); + damaged[100..500].fill(0x11); + damaged[9_000..9_400].fill(0x33); + + let repaired_shards = + repair_inline_recovery_prefix_shards(prefix.len(), &recovery_data, |range| { + Ok(damaged[range].to_vec()) + }) + .unwrap(); + assert!(!repaired_shards.is_empty()); + + let mut repaired = damaged; + for (range, data) in repaired_shards { + assert_eq!(range.len(), data.len()); + repaired[range].copy_from_slice(&data); + } + + assert_eq!(repaired, prefix); + } + + #[test] + fn rar5_inline_recovery_archive_scans_chunks_and_repairs_prefix() { + let prefix: Vec = (0..32_000).map(|index| (index * 13) as u8).collect(); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut archive = prefix.clone(); + archive.extend_from_slice(b"service header bytes before chunks"); + archive.extend_from_slice(&recovery_data); + archive.extend_from_slice(b"end bytes"); + let mut damaged = archive.clone(); + damaged[256..320].fill(0x5a); + + let repaired = repair_inline_recovery_archive(&damaged).unwrap(); + + assert_eq!(repaired, archive); + } + + #[test] + fn rar5_inline_recovery_rebuilds_a_record_above_one_hundred_percent() { + // WinRAR takes -rr up to 1000%, so a record can hold more recovery + // shards than data shards. plan_inline_recovery never produces one, + // and a rebuild that works back from a percentage cannot match it. + let prefix: Vec = (0..32_000).map(|index| (index * 13) as u8).collect(); + let mut plan = plan_inline_recovery(prefix.len() as u64, 100).unwrap(); + plan.recovery_shards = plan.data_shards * 3; + let recovery_data = build_inline_recovery_data_for_plan(&prefix, plan).unwrap(); + let mut archive = prefix.clone(); + archive.extend_from_slice(b"service header bytes before chunks"); + let record_start = archive.len(); + archive.extend_from_slice(&recovery_data); + archive.extend_from_slice(b"end bytes"); + + let mut damaged = archive.clone(); + damaged[256..320].fill(0x5a); + damaged[record_start + 0x48] ^= 0xff; + + let repaired = repair_inline_recovery_archive(&damaged).unwrap(); + + assert_eq!(repaired, archive); + } + + #[test] + fn rar5_inline_recovery_picks_the_outermost_record() { + // A stored archive nested inside this one has its own {RB} chunks, and + // they sit earlier in the file than ours. + let inner_prefix: Vec = (0..16_000).map(|index| (index * 7) as u8).collect(); + let mut inner = inner_prefix.clone(); + inner.extend_from_slice(&build_structural_inline_recovery_data(&inner_prefix, 20).unwrap()); + + let mut prefix = b"outer archive header bytes".to_vec(); + prefix.extend_from_slice(&inner); + prefix.extend_from_slice(&recovery_test_bytes(9_000, 5)); + let mut archive = prefix.clone(); + archive.extend_from_slice(b"service header bytes before chunks"); + let record_start = archive.len(); + archive.extend_from_slice(&build_structural_inline_recovery_data(&prefix, 20).unwrap()); + archive.extend_from_slice(b"end bytes"); + + let mut damaged = archive.clone(); + damaged[record_start + 0x48] ^= 0xff; + + let repaired = repair_inline_recovery_archive(&damaged).unwrap(); + + assert_eq!(repaired, archive); + } + + #[test] + fn rar5_inline_recovery_archive_accepts_healthy_archive() { + let prefix: Vec = (0..32_000).map(|index| (index * 17) as u8).collect(); + let recovery_data = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut archive = prefix.clone(); + archive.extend_from_slice(b"service header bytes before chunks"); + archive.extend_from_slice(&recovery_data); + archive.extend_from_slice(b"end bytes"); + + let repaired = repair_inline_recovery_archive(&archive).unwrap(); + + assert_eq!(repaired, archive); + } + + #[test] + fn rar5_archive_repair_rejects_impossible_protected_ranges() { + let chunk = build_structural_inline_recovery_data(b"x", 10).unwrap(); + assert_eq!( + repair_inline_recovery_archive(&chunk), + Err(Error::BadRecoveryChunk) + ); + + let mut beyond_end = chunk.clone(); + let protected_size = beyond_end.len() as u64 + 1; + beyond_end[0x22..0x2a].copy_from_slice(&protected_size.to_le_bytes()); + let crc = crc64_xz(&beyond_end[0x0c..]); + beyond_end[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + assert_eq!( + repair_inline_recovery_archive(&beyond_end), + Err(Error::BadRecoveryChunk) + ); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn rar5_archive_repair_rejects_protected_size_above_native_range() { + let mut chunk = build_structural_inline_recovery_data(b"x", 10).unwrap(); + chunk[0x22..0x2a].copy_from_slice(&(u32::MAX as u64 + 1).to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + assert_eq!( + repair_inline_recovery_archive(&chunk), + Err(Error::PlanOverflow) + ); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn rar5_archive_repair_rejects_native_payload_and_offset_overflows() { + let mut chunk = build_structural_inline_recovery_data(b"x", 10).unwrap(); + let group_count = 65_536u64; + let shard_size = 80 + group_count; + chunk.resize(shard_size as usize, 0); + chunk[0x0c..0x10].copy_from_slice(&(shard_size as u32).to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&group_count.to_le_bytes()); + chunk[0x32..0x3a].copy_from_slice(&shard_size.to_le_bytes()); + chunk[0x3c..0x3e].copy_from_slice(&(u16::MAX - 1).to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + let mut archive = b"x".to_vec(); + archive.extend_from_slice(&chunk); + assert_eq!( + repair_inline_recovery_archive(&archive), + Err(Error::PlanOverflow) + ); + chunk[0x3e..0x40].copy_from_slice(&(u16::MAX - 2).to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + archive.truncate(1); + archive.extend_from_slice(&chunk); + assert_eq!( + repair_inline_recovery_archive(&archive), + Err(Error::BadRecoveryChunk) + ); + + let mut chunk = build_structural_inline_recovery_data(b"abcd", 10).unwrap(); + let group_count = 65_458u64; + let shard_size = 80 + group_count; + assert_eq!((u16::MAX as u64 - 1) * shard_size, u32::MAX as u64 - 3); + chunk.resize(shard_size as usize, 0); + chunk[0x0c..0x10].copy_from_slice(&(shard_size as u32).to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&group_count.to_le_bytes()); + chunk[0x32..0x3a].copy_from_slice(&shard_size.to_le_bytes()); + chunk[0x3c..0x3e].copy_from_slice(&(u16::MAX - 1).to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + let mut archive = b"abcd".to_vec(); + archive.extend_from_slice(&chunk); + assert_eq!( + repair_inline_recovery_archive(&archive), + Err(Error::PlanOverflow) + ); + } + + #[test] + fn rar5_archive_repair_rejects_missing_chunks_and_invalid_end_header_source() { + assert_eq!( + repair_inline_recovery_archive(b"no recovery chunks"), + Err(Error::BadRecoveryChunk) + ); + + let prefix = b"not a RAR archive"; + let mut archive = prefix.to_vec(); + archive.extend_from_slice(&build_structural_inline_recovery_data(prefix, 10).unwrap()); + assert_eq!( + repair_inline_recovery_archive(&archive), + Err(Error::BadRecoveryChunk) + ); + + let options = super::InlineRepairOptions { + record_range: Some(0..0), + ..Default::default() + }; + assert_eq!( + super::repair_inline_recovery_archive_with_report(&archive, &options).unwrap_err(), + Error::BadRecoveryChunk + ); + } + + #[test] + fn rar5_end_header_errors_preserve_cancellation() { + assert_eq!( + super::map_recovery_end_error(crate::Error::Cancelled), + Error::Cancelled + ); + assert_eq!( + super::map_recovery_end_error(crate::Error::WrongPasswordOrCorruptData), + Error::BadRecoveryChunk + ); + } + + #[test] + fn rar5_record_rebuild_checks_resource_and_record_length() { + let control = crate::read_control::ReadControl::default(); + let impossible = InlineRecoveryPlan { + data_shards: 1, + recovery_shards: u64::MAX, + group_count: 2, + header_size: 80, + shard_size: 82, + }; + assert_eq!( + super::rebuild_inline_recovery_record_with_control( + &[], + b"x", + 0..0, + impossible, + &control + ), + Err(Error::PlanOverflow) + ); + let huge = InlineRecoveryPlan { + data_shards: 1, + recovery_shards: 1000, + group_count: 2_000_000, + header_size: 80, + shard_size: 2_000_080, + }; + assert_eq!( + super::rebuild_inline_recovery_record_with_control(&[], b"x", 0..0, huge, &control), + Err(Error::RebuildTooLarge) + ); + + let plan = plan_inline_recovery(1, 10).unwrap(); + assert_eq!( + super::rebuild_inline_recovery_record_with_control(&[], b"x", 0..0, plan, &control), + Err(Error::BadRecoveryChunk) + ); + } + + #[test] + fn rar5_archive_repair_keeps_prefix_when_record_rebuild_exceeds_limit() { + let mut chunk = build_structural_inline_recovery_data(b"x", 10).unwrap(); + let group_count = 8192u64; + let shard_size = 80 + group_count; + let recovery_shards = u16::MAX - 1; + chunk.resize(shard_size as usize, 0); + chunk[0x0c..0x10].copy_from_slice(&(shard_size as u32).to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&group_count.to_le_bytes()); + chunk[0x32..0x3a].copy_from_slice(&shard_size.to_le_bytes()); + chunk[0x3c..0x3e].copy_from_slice(&recovery_shards.to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[0x04..0x0c].copy_from_slice(&crc.to_le_bytes()); + + let mut archive = b"x".to_vec(); + archive.extend_from_slice(&chunk); + let (repaired, report) = super::repair_inline_recovery_archive_with_report( + &archive, + &super::InlineRepairOptions::default(), + ) + .unwrap(); + assert_eq!(repaired, archive); + assert!(!report.changed); + assert!(!report.recovery_record_rebuilt); + assert_eq!(report.available_recovery_shards, Some(1)); + assert_eq!( + report.expected_recovery_shards, + Some(recovery_shards as u64) + ); + } + + #[test] + fn rar5_inline_recovery_rejects_unrepairable_damage_count() { + let prefix = b"small prefix with only one parity shard".repeat(100); + let recovery_data = build_structural_inline_recovery_data(&prefix, 1).unwrap(); + let mut damaged = prefix.clone(); + damaged[0] ^= 0xff; + damaged[1024] ^= 0xff; + + assert_eq!( + repair_inline_recovery_prefix(&damaged, &recovery_data), + Err(Error::TooManyDamagedShards) + ); + } + + #[test] + fn rar5_reconstruct_data_shards_repairs_missing_shards_from_parity() { + let first = b"abcdefgh".to_vec(); + let second = b"ijklmnop".to_vec(); + let third = b"qrstuvwx".to_vec(); + let refs = [first.as_slice(), second.as_slice(), third.as_slice()]; + let parity = encode_parity_shards(&refs, 2).unwrap(); + + let reconstructed = reconstruct_data_shards( + &[Some(&first), None, Some(&third)], + &[(0, parity[0].as_slice())], + ) + .unwrap(); + + assert_eq!(reconstructed[0], first); + assert_eq!(reconstructed[1], second); + assert_eq!(reconstructed[2], third); + } + + #[test] + fn rar5_reconstruct_data_shards_validates_shapes_and_missing_count() { + assert_eq!(reconstruct_data_shards(&[], &[]), Err(Error::TooManyShards)); + assert_eq!( + reconstruct_data_shards(&[None], &[]), + Err(Error::TooManyDamagedShards) + ); + assert_eq!( + reconstruct_data_shards(&[Some(&[1, 2, 3])], &[]), + Err(Error::OddShardSize) + ); + assert_eq!( + reconstruct_data_shards(&[None], &[(0, &[1, 2]), (1, &[3, 4, 5, 6])]), + Err(Error::ShardSizeMismatch) + ); + assert_eq!( + reconstruct_data_shards(&[Some(&[1, 2, 3, 4]), None], &[(0, &[5, 6])]), + Err(Error::ShardSizeMismatch) + ); + assert_eq!( + reconstruct_data_shards(&[None, None], &[(0, &[1, 2])]), + Err(Error::TooManyDamagedShards) + ); + assert_eq!( + reconstruct_data_shards(&[None], &[(usize::MAX, &[1, 2])]), + Err(Error::TooManyShards) + ); + let mut data_shards = vec![vec![0, 0]]; + assert_eq!( + super::recover_damaged_shards(&mut data_shards, &[0], &[(usize::MAX, &[1, 2])]), + Err(Error::TooManyShards) + ); + assert_eq!( + super::recover_damaged_shards(&mut data_shards, &[0], &[]), + Err(Error::TooManyDamagedShards) + ); + assert_eq!( + reconstruct_data_shards(&[None], &[(super::FIELD_SIZE - 1, &[1, 2])]), + Err(Error::TooManyShards) + ); + assert_eq!( + reconstruct_data_shards(&[Some(&[1, 2]), Some(&[3])], &[]), + Ok(vec![vec![1, 2], vec![3, 0]]) + ); + } + + #[test] + fn rar5_reconstruct_data_shards_repairs_multiple_missing_shards() { + let first = b"abcdefgh".to_vec(); + let second = b"ijklmnop".to_vec(); + let third = b"qrstuvwx".to_vec(); + let refs = [first.as_slice(), second.as_slice(), third.as_slice()]; + let parity = encode_parity_shards(&refs, 2).unwrap(); + + assert_eq!( + reconstruct_data_shards( + &[None, Some(&second), None], + &[(0, parity[0].as_slice()), (0, parity[0].as_slice())] + ), + Err(Error::SingularElement) + ); + + let reconstructed = reconstruct_data_shards( + &[None, Some(&second), None], + &[(0, parity[0].as_slice()), (1, parity[1].as_slice())], + ) + .unwrap(); + + assert_eq!(reconstructed[0], first); + assert_eq!(reconstructed[1], second); + assert_eq!(reconstructed[2], third); + } +} + +fn check_repair(control: &crate::read_control::ReadControl) -> Result<()> { + control.check().map_err(|_| Error::Cancelled) +} +fn repair_crc( + data: &[u8], + initial: u64, + control: &crate::read_control::ReadControl, +) -> Result { + let mut state = initial; + for chunk in data.chunks(64 * 1024) { + check_repair(control)?; + state = crc64_update(chunk, state); + } + Ok(state) +} +fn repair_marker( + input: &[u8], + control: &crate::read_control::ReadControl, +) -> Result> { + for start in (0..input.len()).step_by(64 * 1024) { + check_repair(control)?; + let end = (start + 64 * 1024 + 3).min(input.len()); + if let Some(offset) = find_recovery_marker(&input[start..end]) { + return Ok(Some(start + offset)); + } + } + Ok(None) +} + +#[cfg(test)] +mod repair_workspace_tests { + use super::*; + + #[test] + fn rar5_repair_admits_discovery_reads_before_calling_range_reader() { + let prefix = b"data"; + let plan = InlineRecoveryPlan { + data_shards: 1, + recovery_shards: 1, + group_count: 4, + header_size: 80, + shard_size: 84, + }; + let chunk = build_inline_recovery_data_for_plan(prefix, plan).unwrap(); + let control = crate::read_control::ReadControl::default(); + let required = repair_solve_workspace_required(1, 0, 4, 0, 4).unwrap(); + let unexpected_read = |_| -> Result> { + panic!("discovery must be refused before requesting range bytes") + }; + assert_eq!( + repair_inline_recovery_prefix_shards_with_workspace_limit( + prefix.len(), + &chunk, + unexpected_read, + &control, + required - 1, + ), + Err(Error::RepairWorkspaceLimitExceeded { + limit: required - 1, + required, + }) + ); + assert_eq!( + repair_inline_recovery_prefix_shards_with_workspace_limit( + prefix.len(), + &chunk, + |range| Ok(prefix[range].to_vec()), + &control, + required, + ), + Ok(Vec::new()) + ); + // Discovery fits, but the solve and repaired output do not. It must + // stop after discovery rather than reread undamaged RHS shards. + let mut reads = 0; + let result = repair_inline_recovery_prefix_shards_with_workspace_limit( + prefix.len(), + &chunk, + |range| { + reads += 1; + Ok(b"fail"[range].to_vec()) + }, + &control, + required, + ); + assert_eq!(reads, 1); + assert!(matches!(result, + Err(Error::RepairWorkspaceLimitExceeded { limit, required: solve }) + if limit == required && solve > required + )); + let token = crate::ReadCancellation::new(); + token.cancel(); + assert_eq!( + repair_inline_recovery_prefix_shards_with_workspace_limit( + prefix.len(), + &chunk, + unexpected_read, + &crate::read_control::ReadControl::new(Some(&token)), + 0, + ), + Err(Error::Cancelled) + ); + } + + #[test] + fn rar5_repair_workspace_admission_is_checked_and_inclusive() { + let limit = super::REPAIR_WORKSPACE_FLOOR_BYTES; + assert_eq!(super::admit_repair_workspace(limit, 0), Ok(())); + let refusal = Error::RepairWorkspaceLimitExceeded { + limit, + required: limit + 1, + }; + assert_eq!( + super::admit_repair_workspace(limit + 1, 0), + Err(refusal.clone()) + ); + let typed = crate::Error::from(refusal.clone()).at_archive_offset(42); + assert_eq!(typed.kind(), crate::ErrorKind::ResourceLimit); + assert!( + matches!(typed.root_cause(), crate::Error::Rar5Recovery(actual) if actual == &refusal) + ); + assert!(typed.to_string().contains("repair workspace limit")); + assert_eq!( + super::repair_solve_workspace_required(1, 1, 0, u64::MAX, 0), + Err(Error::PlanOverflow) + ); + assert_eq!( + super::repair_workspace_product(u64::MAX, 2), + Err(Error::PlanOverflow) + ); + // Ordinary generated geometry still fits even with all 200 shards + // damaged, full output buffers and read scratch. + let required = + super::repair_solve_workspace_required(200, 200, 1024, 200 * 1024, 1024).unwrap(); + assert_eq!(super::admit_repair_workspace(required, 0), Ok(())); + } + + #[test] + fn rar5_repair_workspace_scales_with_genuine_input() { + const MIB: u64 = 1024 * 1024; + // A REV set of five 100 MiB volumes, one missing, with two REV files: + // padded output alone exceeds the floor, yet master rebuilt it. + let volume = 100 * MIB as usize; + let required = + super::repair_solve_workspace_required(5, 1, volume, 5 * volume as u64, 0).unwrap(); + assert!(required > super::REPAIR_WORKSPACE_FLOOR_BYTES); + assert_eq!( + super::admit_repair_workspace(required, 4 * 100 * MIB + 2 * 100 * MIB), + Ok(()) + ); + + // A 10 GiB archive with a 10% record has 51 MiB shards; repairing all + // 20 recoverable shards must fit beside the archive and its record. + let archive = 10 * 1024 * MIB; + let shard = archive.div_ceil(200) as usize; + let required = + super::repair_solve_workspace_required(200, 20, shard, 20 * shard as u64, shard) + .unwrap(); + assert!(required > super::REPAIR_WORKSPACE_FLOOR_BYTES); + assert_eq!( + super::admit_repair_workspace(required, archive + archive / 10), + Ok(()) + ); + + // The multiple still refuses records declaring far more than they hold. + let input = super::REPAIR_WORKSPACE_FLOOR_BYTES; + assert!(matches!( + super::admit_repair_workspace(4 * input + 1, input), + Err(Error::RepairWorkspaceLimitExceeded { .. }) + )); + } + + #[test] + fn rar5_tiny_incomplete_record_keeps_repaired_prefix_without_giant_rebuild() { + // One 128 KiB chunk declares 16,000 recovery rows. Only 32 KiB of + // parity would be rebuilt, but its full coefficient matrix is almost + // 0.5 GiB and repeated headers would produce a 1.9 GiB recovery record. + // Frame this one chunk directly, without generating the claimed rows. + let data_count = 16_000usize; + let recovery_count = 16_000usize; + let header_size = 0x48 + data_count * 8; + let shard_size = header_size + 2; + let mut chunk = vec![0; shard_size]; + chunk[..4].copy_from_slice(b"{RB}"); + chunk[0x0c..0x10].copy_from_slice(&(shard_size as u32).to_le_bytes()); + chunk[0x10..0x14].copy_from_slice(&(header_size as u32).to_le_bytes()); + chunk[0x14] = 1; + chunk[0x15] = 1; + chunk[0x22..0x2a].copy_from_slice(&1u64.to_le_bytes()); + chunk[0x2a..0x32].copy_from_slice(&2u64.to_le_bytes()); + chunk[0x32..0x3a].copy_from_slice(&(shard_size as u64).to_le_bytes()); + chunk[0x3a..0x3c].copy_from_slice(&(data_count as u16).to_le_bytes()); + chunk[0x3c..0x3e].copy_from_slice(&(recovery_count as u16).to_le_bytes()); + chunk[0x40..0x48].copy_from_slice(&crc64_rar_state(b"a").to_le_bytes()); + let gf = shared_gf16(); + let parity = gf.mul(gf.inv(data_count as u16).unwrap(), b'a' as u16); + chunk[header_size..].copy_from_slice(&parity.to_le_bytes()); + let crc = crc64_xz(&chunk[0x0c..]); + chunk[4..0x0c].copy_from_slice(&crc.to_le_bytes()); + + let mut archive = b"b".to_vec(); + archive.extend_from_slice(&chunk); + let repaired = super::repair_inline_recovery_archive(&archive).unwrap(); + assert_eq!(repaired.len(), archive.len()); + assert_eq!(repaired[0], b'a'); + assert_eq!(&repaired[1..], &archive[1..]); + let (reported, report) = super::repair_inline_recovery_archive_with_report( + &archive, + &super::InlineRepairOptions::default(), + ) + .unwrap(); + assert_eq!(reported, repaired); + assert!(report.changed); + assert!(report.data_repaired); + assert!(!report.recovery_record_rebuilt); + assert!(!report.end_record_rebuilt); + assert_eq!(report.available_recovery_shards, Some(1)); + assert_eq!(report.expected_recovery_shards, Some(recovery_count as u64)); + + let plan = InlineRecoveryPlan { + data_shards: data_count as u64, + recovery_shards: recovery_count as u64, + group_count: 2, + header_size: header_size as u64, + shard_size: shard_size as u64, + }; + let end = 1 + plan.payload_size().unwrap() as usize; + assert!(matches!( + super::rebuild_inline_recovery_record_with_control( + &archive, + b"a", + 1..end, + plan, + &crate::read_control::ReadControl::default(), + ), + Err(Error::RepairWorkspaceLimitExceeded { limit, required }) + if limit == super::REPAIR_WORKSPACE_FLOOR_BYTES && required > limit + )); + let token = crate::ReadCancellation::new(); + token.cancel(); + let control = crate::read_control::ReadControl::new(Some(&token)); + assert_eq!( + super::rebuild_inline_recovery_record_with_control( + &archive, + b"a", + 1..end, + plan, + &control + ), + Err(Error::Cancelled) + ); + } + + #[test] + fn rar5_reconstruction_admits_padded_output_and_all_solve_matrices() { + // Less than 1 MiB of caller metadata and 8 KiB of parity used to + // allocate 512 MiB of zero padding before the solve was considered. + let empty = &[][..]; + let mut data = vec![Some(empty); super::FIELD_SIZE - 1]; + data[0] = None; + let parity = vec![0; 8192]; + assert!(matches!( + reconstruct_data_shards(&data, &[(0, &parity)]), + Err(Error::RepairWorkspaceLimitExceeded { limit, required }) + if limit == super::REPAIR_WORKSPACE_FLOOR_BYTES && required > limit + )); + + // Tiny two-byte shards can still ask for hundreds of MiB across + // original equations, their working copy and the inverse together. + let count = 7000; + let data = vec![None; count]; + let recovery = (0..count).map(|row| (row, &[0, 0][..])).collect::>(); + assert!(matches!( + reconstruct_data_shards(&data, &recovery), + Err(Error::RepairWorkspaceLimitExceeded { limit, required }) + if limit == super::REPAIR_WORKSPACE_FLOOR_BYTES && required > limit + )); + } + + #[test] + fn rar5_reconstruction_validates_before_oversized_padding() { + let parity = vec![0; 8192]; + let missing = vec![None; super::FIELD_SIZE - 1]; + assert_eq!( + reconstruct_data_shards(&missing, &[(0, &parity)]), + Err(Error::TooManyDamagedShards) + ); + let mut data = vec![Some(&[][..]); super::FIELD_SIZE - 1]; + data[0] = None; + assert_eq!( + reconstruct_data_shards(&data, &[(usize::MAX, &parity)]), + Err(Error::TooManyShards) + ); + let long = vec![0; parity.len() + 1]; + *data.last_mut().unwrap() = Some(&long); + assert_eq!( + reconstruct_data_shards(&data, &[(0, &parity)]), + Err(Error::ShardSizeMismatch) + ); + assert_eq!( + reconstruct_data_shards(&vec![Some(&[][..]); super::FIELD_SIZE + 1], &[]), + Err(Error::TooManyShards) + ); + } +} + +#[cfg(test)] +mod cancellation_tests { + use super::*; + + fn scheduled(checks: usize) -> crate::read_control::ReadControl { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(checks); + control + } + + #[test] + fn cancellation_stops_marker_search_and_chunk_checksums() { + let noise = vec![0; 256 * 1024]; + assert_eq!( + find_inline_recovery_chunks_with_control(&noise, &scheduled(2)).unwrap_err(), + Error::Cancelled + ); + let prefix = vec![42; 256 * 1024]; + let data = build_structural_inline_recovery_data(&prefix, 10).unwrap(); + assert_eq!( + parse_inline_recovery_chunk_with_control(&data, &scheduled(1)).unwrap_err(), + Error::Cancelled + ); + } + + #[test] + fn cancellation_stops_reconstruction_and_record_rebuilding() { + let prefix = vec![42; 128 * 1024]; + let plan = plan_inline_recovery(prefix.len() as u64, 10).unwrap(); + let mut shards = split_prefix_shards(&prefix, plan).unwrap(); + let refs: Vec<_> = shards.iter().map(Vec::as_slice).collect(); + let parity = encode_parity_shards(&refs, plan.recovery_shards as usize).unwrap(); + shards[0].fill(0); + assert_eq!( + recover_damaged_shards_with_control( + &mut shards, + &[0], + &[(0, &parity[0])], + &scheduled(2) + ) + .unwrap_err(), + Error::Cancelled + ); + assert_eq!( + build_inline_recovery_data_for_plan_with_control(&prefix, plan, &scheduled(2)) + .unwrap_err(), + Error::Cancelled + ); + } + + #[test] + fn cancellation_stops_selected_equation_generation_and_inverse_application() { + let damaged: Vec<_> = (0..128).collect(); + let rows: Vec<_> = (0..128).map(|row| (row, &[][..])).collect(); + assert_eq!( + recovery_equations(shared_gf16(), 128, &damaged, &rows, &scheduled(2)).unwrap_err(), + Error::Cancelled + ); + let inverse = vec![vec![1; 128]; 128]; + assert_eq!( + apply_inverse_matrix_with_control(shared_gf16(), &inverse, &[1; 128], &scheduled(2)) + .unwrap_err(), + Error::Cancelled + ); + } + + #[test] + fn raw_repair_never_tolerates_cancellation_as_a_failed_record_rebuild() { + let prefix = vec![42; 20_000]; + let recovery = build_structural_inline_recovery_data(&prefix, 20).unwrap(); + let mut archive = prefix; + let record_start = archive.len(); + archive.extend_from_slice(&recovery); + archive.extend_from_slice(b"end bytes"); + let expected = archive.clone(); + // Damage one recovery chunk; surviving chunks can rebuild the record. + archive[record_start + 0x48] ^= 1; + let mut cancelled = 0; + let mut completed = 0; + for checks in (0..256).chain([usize::MAX]) { + let token = crate::ReadCancellation::new(); + let control = crate::read_control::ReadControl::new(Some(&token)); + control.cancel_after_checks(checks); + let options = InlineRepairOptions { + control: control.clone(), + ..Default::default() + }; + let result = repair_inline_recovery_archive_with_report(&archive, &options); + match result { + Err(Error::Cancelled) => cancelled += 1, + Ok((data, report)) => { + assert!(!token.is_cancelled()); + assert_eq!(data, expected); + assert!(report.recovery_record_rebuilt); + completed += 1; + } + other => panic!("unexpected repair result: {other:?}"), + } + } + assert!(cancelled > 0 && completed > 0); + } + + #[test] + fn recovery_cancellation_has_the_public_cancelled_error_kind() { + assert_eq!( + crate::Error::from(Error::Cancelled), + crate::Error::Cancelled + ); + assert_eq!( + crate::Error::Rar5Recovery(Error::Cancelled).kind(), + crate::ErrorKind::Cancelled + ); + } +} + +#[cfg(all(test, feature = "write"))] +#[test] +fn missing_end_header_repair_preserves_an_active_cancellation_token() { + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50) + .store(true) + .recovery_percent(Some(20)); + builder + .add_bytes( + b"recoverable".to_vec(), + b"protected payload".to_vec(), + None, + None, + ) + .unwrap(); + let original = builder.to_bytes().unwrap(); + let archive = crate::rar50::Archive::parse(&original).unwrap(); + let end_start = archive + .blocks + .iter() + .find_map(|block| match block { + crate::rar50::Block::End(end) => Some(end.block.offset), + _ => None, + }) + .unwrap(); + let truncated = &original[..end_start]; + let token = crate::ReadCancellation::new(); + let options = InlineRepairOptions { + control: crate::read_control::ReadControl::new(Some(&token)), + ..Default::default() + }; + let (repaired, report) = + repair_inline_recovery_archive_with_report(truncated, &options).unwrap(); + assert!(!token.is_cancelled()); + assert!(report.end_record_rebuilt); + assert_eq!(repaired, original); + token.cancel(); + assert_eq!( + repair_inline_recovery_archive_with_report(truncated, &options).unwrap_err(), + Error::Cancelled + ); +} diff --git a/arca-rar/vendor/rars/src/rewrite.rs b/arca-rar/vendor/rars/src/rewrite.rs new file mode 100644 index 00000000..4b5290a7 --- /dev/null +++ b/arca-rar/vendor/rars/src/rewrite.rs @@ -0,0 +1,642 @@ +//! Conservative preflight for the currently supported rewrite model. + +use crate::{Archive, ArchiveMemberMeta, AttrSource}; +use std::collections::HashSet; + +impl Archive { + /// Decodes comments in member order with one metadata traversal. Like + /// `member_comment_at`, this does not retain resource limits from parsing. + pub fn member_comments(&self, password: Option<&[u8]>) -> crate::Result>>> { + match self { + Archive::Rar13(a) => a.entries.iter().map(|entry| entry.file_comment()).collect(), + Archive::Rar15To40(a) => a.files().map(|file| file.file_comment()).collect(), + Archive::Rar50Plus(a) => { + let mut comments: Vec> = Vec::new(); + for block in &a.blocks { + match block { + crate::rar50::Block::File(_) => comments.push(None), + crate::rar50::Block::Service(service) if service.name == b"CMT" => { + if let Some(comment) = comments.last_mut() { + if comment.replace(service).is_some() { + return Err(crate::Error::InvalidHeader( + "duplicate member comment records", + )); + } + } + } + _ => {} + } + } + comments + .into_iter() + .map(|comment| { + comment + .map(|service| { + let mut data = Vec::new(); + service.write_to(a, password, &mut data)?; + Ok(data) + }) + .transpose() + }) + .collect() + } + } + } + + /// Decodes a member comment by original archive index, including directories. + /// Missing comments return `None`; an invalid index returns `EntryNotFound`. + /// Duplicate RAR5 CMT records are refused rather than silently dropping one. + /// This helper does not retain parsing/extraction resource policies. + pub fn member_comment_at( + &self, + index: usize, + password: Option<&[u8]>, + ) -> crate::Result>> { + match self { + Archive::Rar13(a) => a + .entries + .get(index) + .ok_or(crate::Error::EntryNotFound)? + .file_comment(), + Archive::Rar15To40(a) => a + .files() + .nth(index) + .ok_or(crate::Error::EntryNotFound)? + .file_comment(), + Archive::Rar50Plus(a) => { + let mut current = None; + let mut next = 0; + let mut found = false; + let mut comment = None; + for block in &a.blocks { + match block { + crate::rar50::Block::File(_) => { + if found { + break; + } + current = Some(next); + next += 1; + found = current == Some(index); + } + crate::rar50::Block::Service(service) + if current == Some(index) && service.name == b"CMT" => + { + if comment.is_some() { + return Err(crate::Error::InvalidHeader( + "duplicate member comment records", + )); + } + comment = Some(service); + } + _ => {} + } + } + if !found { + return Err(crate::Error::EntryNotFound); + } + comment + .map(|service| { + let mut data = Vec::new(); + service.write_to(a, password, &mut data)?; + Ok(data) + }) + .transpose() + } + } + } + + /// Decode the native target bytes carried as a legacy Unix symlink payload. + /// This never follows a filesystem link. Payload integrity is checked. + pub fn legacy_symlink_target_at( + &self, + index: usize, + password: Option<&[u8]>, + ) -> crate::Result>> { + let member = self + .members() + .nth(index) + .ok_or(crate::Error::EntryNotFound)?; + if !member.is_legacy_unix_symlink() { + return Ok(None); + } + let target = self + .read_member_at(index, password)? + .ok_or(crate::Error::EntryNotFound)?; + validate_legacy_link_target(&target, &member.meta.name)?; + Ok(Some(target)) + } + + /// Decode all legacy Unix link targets with one controlled traversal. + /// Results follow original member order, including `None` for directories, + /// ordinary files and modern redirections. No payloads are read when there + /// are no legacy links. This never follows filesystem links. + /// + /// Independent non-link payloads are skipped. Solid predecessors through the + /// last link are decoded and verified, but discarded. All collected targets + /// must pass integrity and target validation before any are returned. Targets + /// are retained in memory; this password-only helper uses default extraction + /// policies and does not retain limits supplied during parsing. + pub fn legacy_symlink_targets( + &self, + password: Option<&[u8]>, + ) -> crate::Result>>> { + use crate::{ArchiveReadOptions, Error, ExtractionDecision, SharedBuffer}; + use std::sync::{Arc, Mutex}; + + let Archive::Rar15To40(legacy) = self else { + return Ok(self.members().map(|_| None).collect()); + }; + let members: Vec<_> = self.members().collect(); + let Some(last) = members + .iter() + .rposition(|member| member.is_legacy_unix_symlink()) + else { + return Ok((0..members.len()).map(|_| None).collect()); + }; + let mut targets: Vec> = (0..members.len()).map(|_| None).collect(); + let solid = legacy.main.is_solid(); + let mut index = 0; + self.extract_with_control( + ArchiveReadOptions::with_optional_password(password), + |member| { + let current = index; + index += 1; + if current > last { + return Ok(ExtractionDecision::Stop); + } + // Finding a legacy link above confines this traversal to + // RAR1.5–4, whose member metadata has no redirection flag. + if member.is_legacy_unix_symlink() { + let bytes = Arc::new(Mutex::new(Some(Vec::new()))); + targets[current] = Some(SharedBuffer(bytes.clone())); + Ok(ExtractionDecision::Extract(Box::new(SharedBuffer(bytes)))) + } else if solid && !member.meta.is_directory { + Ok(ExtractionDecision::Extract(Box::new(std::io::sink()))) + } else { + Ok(ExtractionDecision::Skip) + } + }, + )?; + targets + .into_iter() + .zip(members) + .map(|(target, member)| { + if !member.is_legacy_unix_symlink() { + return Ok(None); + } + let target = + target + .and_then(|target| target.lock().take()) + .ok_or(Error::InvalidHeader( + "legacy link target disappeared while reading", + ))?; + validate_legacy_link_target(&target, &member.meta.name)?; + Ok(Some(target)) + }) + .collect() + } + + /// Configure a builder with supported source format, solid and encryption settings. + /// Member data/comment passwords are retained separately when entries are copied. + pub fn preserving_builder(&self, password: Option<&[u8]>) -> crate::Result { + let issues = self.rewrite_preservation_issues(); + if !issues.is_empty() { + return Err(crate::Error::InvalidArgument( + "archive has unsupported preservation settings", + )); + } + match self { + Archive::Rar13(archive) => { + let password = if archive.entries.iter().any(|entry| entry.is_encrypted()) { + Some( + password + .filter(|value| !value.is_empty()) + .ok_or(crate::Error::NeedPassword)? + .to_vec(), + ) + } else { + None + }; + Ok(crate::Builder::new(crate::ArchiveVersion::Rar14) + .compression_level(Some(3)) + .solid(archive.main.is_solid()) + .password(password)) + } + Archive::Rar15To40(archive) => { + let encrypted = archive.main.has_encrypted_headers() + || archive.files().any(|file| file.is_encrypted()) + || archive.new_subs().any(|sub| sub.file.is_encrypted()); + let password = if encrypted { + Some( + password + .filter(|password| !password.is_empty()) + .ok_or(crate::Error::NeedPassword)? + .to_vec(), + ) + } else { + None + }; + let version = archive.preservation_version(); + Ok(crate::Builder::new(version) + .compression_level(Some(3)) + .legacy_unpack_version( + (version == crate::ArchiveVersion::Rar20 + && archive.files().any(|file| file.unp_ver == 26)) + .then_some(26), + ) + .solid(archive.main.is_solid()) + // Preflight accepts only CMT new-sub records, and at most one. + .archive_comment_password( + archive + .new_subs() + .any(|sub| sub.file.is_encrypted()) + .then(|| password.clone()) + .flatten(), + ) + .password(password) + .header_encryption(archive.main.has_encrypted_headers()) + .legacy_archive_comment_metadata( + archive + .new_subs() + .next() + .map(|sub| (sub.file.file_time, sub.file.host_os)), + )) + } + Archive::Rar50Plus(archive) => { + let version = if archive + .files() + .any(|file| file.compression_info & 0x3f == 1) + { + crate::ArchiveVersion::Rar70 + } else { + crate::ArchiveVersion::Rar50 + }; + let rar7_dictionary_size = archive + .files() + .find(|file| file.compression_info & 0x3f == 1) + .map(|file| { + file.decoded_compression_info() + .map(|info| info.dictionary_size) + }) + .transpose()?; + let encrypted = archive.main.encrypted_headers + || archive.blocks.iter().any(|block| match block { + crate::rar50::Block::File(file) | crate::rar50::Block::Service(file) => { + file.encrypted + } + _ => false, + }); + let password = if encrypted { + Some( + password + .filter(|password| !password.is_empty()) + .ok_or(crate::Error::NeedPassword)? + .to_vec(), + ) + } else { + None + }; + let archive_comment_encrypted = archive.blocks.iter().take_while(|block| !matches!(block, crate::rar50::Block::File(_))) + .any(|block| matches!(block, crate::rar50::Block::Service(service) if service.name == b"CMT" && service.encrypted)); + let mut quick_open = false; + let mut recovery_percent = None; + for block in &archive.blocks { + if let crate::rar50::Block::Service(service) = block { + if service.name == b"QO" || service.name == b"RR" { + service.write_to(archive, password.as_deref(), &mut std::io::sink())?; + quick_open |= service.name == b"QO"; + if service.name == b"RR" { + recovery_percent = + service.recovery_record()?.map(|record| record.percent); + } + } + } + } + let metadata = archive.main.extras.iter().find_map(|extra| match extra { + crate::rar50::MainExtraRecord::ArchiveMetadata(metadata) => { + Some(metadata.clone()) + } + _ => None, + }); + crate::Builder::new(version) + .compression_level(Some(3)) + .rar50_dictionary_size(rar7_dictionary_size) + .solid(archive.main.is_solid()) + .password(password.clone()) + .header_encryption(archive.main.encrypted_headers) + .archive_comment_password(if archive_comment_encrypted { + password + } else { + None + }) + .recovery_percent(recovery_percent) + .archive_metadata(metadata, archive.main.is_locked(), quick_open) + } + } + } + + /// Whether each member's comment payload is encrypted, in archive member order. + pub fn member_comment_encryption(&self) -> Vec { + let Archive::Rar50Plus(archive) = self else { + return self.members().map(|_| false).collect(); + }; + let mut encrypted = Vec::new(); + for block in &archive.blocks { + match block { + crate::rar50::Block::File(_) => encrypted.push(false), + crate::rar50::Block::Service(service) if service.name == b"CMT" => { + if let Some(last) = encrypted.last_mut() { + *last = service.encrypted; + } + } + _ => {} + } + } + encrypted + } + + /// Properties the current rewrite adapters cannot promise to preserve. + /// + /// An empty list certifies only the supported metadata subset, not payload + /// integrity or byte-identical output. Legacy preservation accepts only + /// ordinary unpacker-29 files with supported native metadata and encryption. + /// Parsed unknown/incomplete RAR5 extras remain visible to this check even + /// though ordinary extraction tolerates them. Source files must stay stable. + pub fn rewrite_preservation_issues(&self) -> Vec { + let mut issues = Vec::new(); + if self.sfx_offset() != 0 { + issues.push("SFX executable prefix".into()); + } + let mut link_targets = std::collections::HashMap::new(); + for (index, member) in self.members().enumerate() { + let meta = &member.meta; + let label = format!("member {index} ({:?})", String::from_utf8_lossy(&meta.name)); + let link = member.supported_redirection(); + if let Some(link) = link.filter(|link| link.redirection_type >= 4) { + if link_targets.get(&link.target_name) != Some(&meta.unpacked_size) { + issues.push(format!( + "{label}: missing, forward or inconsistent redirection target" + )); + } + } + if !meta.is_directory + && (!meta.is_redirection || link.is_some_and(|link| link.redirection_type >= 4)) + { + link_targets.insert(meta.name.clone(), meta.unpacked_size); + } + if crate::builder::validate_entry_name(meta.name.clone()).is_err() { + issues.push(format!("{label}: unsupported output name")); + } + + if meta.is_split_before || meta.is_split_after { + issues.push(format!("{label}: split-volume layout")); + } + if meta.attr_source() == AttrSource::Unknown { + issues.push(format!("{label}: unknown host attributes")); + } + if special_entry(meta) + && member.supported_redirection().is_none() + && !member.is_legacy_unix_symlink() + { + issues.push(format!("{label}: special entry type or directory contents")); + } + if (meta.attr_source() == AttrSource::Unix + && (meta.file_attr & !0o177777 != 0 + || (meta.is_directory && meta.file_attr & 0o170000 == 0o100000))) + || (meta.attr_source() == AttrSource::Dos + && (meta.file_attr & 0x10 != 0) != meta.is_directory) + { + issues.push(format!("{label}: unsupported or inconsistent attributes")); + } + } + match self { + Archive::Rar13(archive) => { + issues.extend(archive.rewrite_preservation_issues()); + } + Archive::Rar15To40(archive) => { + issues.extend(archive.rewrite_preservation_issues()); + } + Archive::Rar50Plus(archive) => { + use crate::rar50::Block; + let main = &archive.main; + + if main.is_volume() || main.volume_number.is_some() { + issues.push("volume layout".into()); + } + + if !main.rewrite_metadata_complete + || main.archive_flags & !0x1f != 0 + || main.block.flags & !1 != 0 + || main.block.data_size.unwrap_or(0) != 0 + { + issues.push("main header metadata, extra records or unknown flags".into()); + } + let mut derived_services = HashSet::new(); + for extra in &main.extras { + if let crate::rar50::MainExtraRecord::ArchiveMetadata(metadata) = extra { + if crate::rar50::write::headers::retained_archive_metadata( + metadata, + &crate::WriterResources::default(), + ) + .is_err() + { + issues.push("unsupported archive metadata".into()); + } + } + } + let mut index = 0; + let mut comment_seen = false; + for block in &archive.blocks { + match block { + Block::File(file) => { + comment_seen = false; + if !file.rewrite_metadata_complete + || file.file_flags & !7 != 0 + || file.block.flags & !0x1b != 0 + || file.compression_info + & if file.compression_info & 0x3f == 0 { + !0x7fff + } else { + !0x1fffff + } + != 0 + { + issues.push(format!( + "member {index}: unsupported, duplicate or incomplete metadata" + )); + } + if file.compression_info & 0x3f > 1 { + issues.push(format!( + "member {index}: source format requires an unsupported compression algorithm" + )); + } + if file.compression_info & 0x40 != 0 && !main.is_solid() { + issues.push(format!( + "member {index}: solid dependency without a solid archive" + )); + } + index += 1; + } + Block::Service(service) => { + if service.name == b"QO" || service.name == b"RR" { + if !derived_services.insert(service.name.clone()) + || !service.rewrite_metadata_complete + || service.encrypted + || service.modification_time().is_some() + || service.file_times.is_some() + || service.file_flags & !4 != 0 + || service.block.flags & !3 != 0 + || service.attributes != 0 + || service.host_os != 0 + || service.compression_info != 0 + || (service.name == b"RR" + && !service.recovery_record().ok().flatten().is_some_and( + |record| (1..=100).contains(&record.percent), + )) + { + issues.push(format!( + "unsupported derived service {:?}", + String::from_utf8_lossy(&service.name) + )); + } + continue; + } + // One CMT per owner (archive or preceding member). + // Other services and ambiguous duplicates remain rejected. + if comment_seen + || service.name != b"CMT" + || !service.rewrite_metadata_complete + || service.modification_time().is_some() + || service.file_times.is_some() + || service.file_flags & !4 != 0 + || service.block.flags & !3 != 0 + || service.attributes != 0 + || service.host_os != 0 + || service.compression_info & !0x7fff != 0 + || service.compression_info & 0x7f != 0 + { + issues.push(format!( + "service record {:?}", + String::from_utf8_lossy(&service.name) + )); + } + comment_seen = true; + } + Block::End(end) => { + // Canonical end headers contain only type, block flags + // and end flags. Reject extra fields instead of silently + // certifying metadata this reader does not expose. + if end.flags != 0 || end.block.flags != 0 || end.block.header_size != 3 + { + issues.push( + "end header flags, volume continuation or extra metadata" + .into(), + ); + } + } + Block::Unknown(_) => issues.push("unknown archive block".into()), + } + } + if main.has_recovery_record() != derived_services.contains(b"RR".as_slice()) { + issues.push("recovery flag and service disagree".into()); + } + if main.encrypted_headers && derived_services.contains(b"QO".as_slice()) { + issues.push("quick-open index with encrypted headers".into()); + } + for extra in &main.extras { + if let crate::rar50::MainExtraRecord::Locator(locator) = extra { + if locator.quick_open_offset.is_some_and(|offset| offset != 0) + && !derived_services.contains(b"QO".as_slice()) + || locator + .recovery_record_offset + .is_some_and(|offset| offset != 0) + && !derived_services.contains(b"RR".as_slice()) + { + issues.push("locator refers to a missing service".into()); + } + } + } + } + } + issues + } +} + +pub(crate) fn special_entry(meta: &ArchiveMemberMeta) -> bool { + let kind = meta.file_attr & 0o170000; + meta.is_redirection + || (meta.is_directory && meta.unpacked_size != 0) + || (meta.attr_source() == AttrSource::Unix + && !matches!(kind, 0 | 0o100000) + && !(meta.is_directory && kind == 0o040000)) + || (meta.attr_source() == AttrSource::Dos && meta.file_attr & 0x400 != 0) +} + +fn validate_legacy_link_target(target: &[u8], name: &[u8]) -> crate::Result<()> { + if target.is_empty() || target.contains(&0) { + return Err(crate::Error::InvalidArgument( + "legacy symbolic link target is empty or contains NUL", + ) + .at_entry(name.to_vec(), "reading link target")); + } + Ok(()) +} + +impl crate::ArchiveMember { + /// Legacy Unix links store the native target as ordinary member data. + pub fn is_legacy_unix_symlink(&self) -> bool { + self.meta.family == crate::ArchiveFamily::Rar15To40 + && self.meta.attr_source() == AttrSource::Unix + && self.meta.file_attr & !0o7777 == 0o120000 + && !self.meta.is_directory + } + + /// Complete supported file times. Legacy DOS fields use the established local-zone policy. + pub fn file_times(&self) -> crate::Result> { + match &self.detail { + crate::ArchiveMemberDetail::Rar50Plus { file_times, .. } => Ok(*file_times), + crate::ArchiveMemberDetail::Rar15To40 { extended_times, .. } => { + crate::FileTimes::legacy(extended_times, self.meta.file_time) + } + _ => Ok(None), + } + } + + /// A redirection whose known kind and header metadata can be retained. + pub fn supported_redirection(&self) -> Option<&crate::rar50::FileRedirection> { + let crate::ArchiveMemberDetail::Rar50Plus { + redirection: Some(link), + .. + } = &self.detail + else { + return None; + }; + (link.supports_header( + self.meta.host_os?, + self.meta.file_attr, + self.meta.is_directory, + ) && (link.redirection_type != 1 || self.unix_symlink().is_some()) + && self.meta.packed_size == 0) + .then_some(link) + } + + /// Supported RAR5 Unix symbolic link metadata. No filesystem lookup occurs. + pub fn unix_symlink(&self) -> Option<&crate::rar50::FileRedirection> { + let crate::ArchiveMemberDetail::Rar50Plus { + redirection: Some(link), + .. + } = &self.detail + else { + return None; + }; + (link.is_supported_unix_symlink() + && self.meta.host_os == Some(1) + && self.meta.file_attr & !0o7777 == 0o120000 + && !self.meta.is_directory + && (self.meta.unpacked_size == 0 + || self.meta.unpacked_size + == crate::filename::decode_rar50(&link.target_name).len() as u64) + && self.meta.packed_size == 0) + .then_some(link) + } +} diff --git a/arca-rar/vendor/rars/src/rewrite_staging.rs b/arca-rar/vendor/rars/src/rewrite_staging.rs new file mode 100644 index 00000000..c845ab1b --- /dev/null +++ b/arca-rar/vendor/rars/src/rewrite_staging.rs @@ -0,0 +1,533 @@ +//! Verified payload staging for archive rewrites. + +use crate::{Archive, ArchiveReadOptions, EntrySource, Result}; +use std::path::PathBuf; +use std::sync::Arc; + +/// Disk policy for [`Archive::stage_rewrite_sources`]. +/// +/// The directory must already exist and be trusted: private files are reopened +/// by path. Payloads are plaintext, even for encrypted input. Cleanup on drop is +/// best effort, not secure erasure; process termination may leave files behind. +/// The byte limit covers retained payloads only, not filesystem overhead, +/// discarded solid dependencies, decoder workspace or writer output spools. +#[derive(Debug, Clone)] +pub struct RewriteStaging { + /// Existing, trusted directory for private temporary payload files. + pub directory: PathBuf, + /// Inclusive limit on the sum of retained, uncompressed payload bytes. + pub max_staged_bytes: u64, +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +mod pipeline; + +impl Archive { + /// Supply verified payloads on demand during one writer call. Decoding uses + /// one archive-order traversal, including required solid predecessors. + /// Sources belong to one write in this call only: assign each source to one + /// output member, and do not retain it after the call returns. + /// The staging limit covers simultaneously retained plaintext payloads. + /// Writers release compressed payload sources when no reread is needed; + /// stored fallback can retain sources through emission. Output streams may + /// contain a prefix on failure; use staged path publication for rollback. + pub fn with_rewrite_sources( + &self, + indices: &[usize], + options: ArchiveReadOptions<'_>, + staging: &RewriteStaging, + progress: Option>, + consume: impl FnOnce(Vec) -> Result, + ) -> Result { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + { + pipeline::run(self, indices, options, staging, progress, consume) + } + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + { + let _ = (indices, options, staging, progress, consume); + Err(crate::Error::InvalidArgument( + "rewrite staging requires disk storage", + )) + } + } + + /// Verify and stage selected payloads in one archive-order extraction pass. + /// + /// Indices count all members, including directories and redirections, but + /// selected indices must identify distinct, unsplit payload members. Results + /// follow the requested order. Metadata must be transferred separately. + /// Solid predecessors are decoded and verified even when not selected; + /// unrelated independent payloads and the suffix after the last selection + /// are not decoded. Read options apply to this entire traversal. + /// + /// No sources are returned on failure. Successful sources can be reopened + /// independently without reading the archive again. Temporary files survive + /// until the last source or open reader drops. This stages the entire + /// selection before returning, with an explicit disk limit; it is not a + /// total memory limit or an incremental writer pipeline. Bare WebAssembly + /// is unsupported because this operation requires disk storage. + pub fn stage_rewrite_sources( + &self, + indices: &[usize], + options: ArchiveReadOptions<'_>, + staging: &RewriteStaging, + ) -> Result> { + self.stage_rewrite_sources_with_progress(indices, options, staging, None) + } + + /// Staging with a separate [`crate::WriteOperation::Staging`] progress phase. + /// Counts decoded bytes, including discarded solid predecessors. Entry indices + /// count decoded payloads in traversal order, not original archive indices. + /// Byte progress precedes integrity verification; finished events follow it. + /// Declared progress totals saturate at `u64::MAX`. + /// + /// Callback cancellation is checked around reports, between members and at + /// bounded output writes. Supply `options.cancellation` to also interrupt + /// decoder work that has not produced output. Blocked I/O cannot be preempted. + pub fn stage_rewrite_sources_with_progress( + &self, + indices: &[usize], + options: ArchiveReadOptions<'_>, + staging: &RewriteStaging, + progress: Option>, + ) -> Result> { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + { + native::stage(self, indices, options, staging, progress, None) + } + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + { + let _ = (indices, options, staging, progress); + Err(crate::Error::InvalidArgument( + "rewrite staging requires disk storage", + )) + } + } +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +mod native { + use super::*; + use crate::atomic64::AtomicU64; + use crate::{ + streaming::Spool, Error, ExtractionDecision, WriteOperation, WriteProgress, + WriteProgressEvent, WriterResources, + }; + use std::sync::atomic::Ordering; + use std::{ + cell::{Cell, RefCell}, + collections::BTreeMap, + io::Write, + rc::Rc, + }; + + struct Sink { + spool: Rc>, + used: Arc, + limit: u64, + } + + impl Write for Sink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let count = bytes.len() as u64; + let mut used = self.used.load(Ordering::Acquire); + loop { + let Some(required) = used + .checked_add(count) + .filter(|required| *required <= self.limit) + else { + return Err(std::io::Error::other(Error::RewriteStagingLimitExceeded { + limit: self.limit, + required: used.saturating_add(count), + })); + }; + match self.used.compare_exchange_weak( + used, + required, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break, + Err(actual) => used = actual, + } + } + let written = match self.spool.borrow_mut().write(bytes) { + Ok(written) => written, + Err(error) => { + self.used.fetch_sub(count, Ordering::AcqRel); + return Err(error); + } + }; + self.used + .fetch_sub(count - written as u64, Ordering::AcqRel); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.spool.borrow_mut().flush() + } + } + + impl Drop for Sink { + fn drop(&mut self) { + self.spool.borrow_mut().park(); + } + } + + struct Progress { + callback: Arc, + completed: Cell, + total: u64, + entries: usize, + } + + impl Progress { + fn check(&self) -> Result<()> { + if self.callback.is_cancelled() { + Err(Error::Cancelled) + } else { + Ok(()) + } + } + + fn report(&self, event: WriteProgressEvent<'_>) -> Result<()> { + self.check()?; + self.callback.report(event); + self.check() + } + + fn finish_entry(&self, entry: &(usize, Vec, u64)) -> Result<()> { + self.report(WriteProgressEvent::EntryFinished { + operation: WriteOperation::Staging, + index: entry.0, + total_entries: self.entries, + name: &entry.1, + input_bytes: entry.2, + }) + } + } + + struct ProgressSink { + inner: Box, + progress: Rc, + } + + impl Write for ProgressSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.progress.check().map_err(std::io::Error::other)?; + let written = self.inner.write(&bytes[..bytes.len().min(64 * 1024)])?; + let completed = self.progress.completed.get().saturating_add(written as u64); + self.progress.completed.set(completed); + self.progress + .report(WriteProgressEvent::Advanced { + operation: WriteOperation::Staging, + completed_bytes: completed, + total_bytes: self.progress.total, + pass: 1, + }) + .map_err(std::io::Error::other)?; + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } + } + + pub(super) fn stage( + archive: &Archive, + indices: &[usize], + options: ArchiveReadOptions<'_>, + staging: &RewriteStaging, + progress: Option>, + delivery: Option<&super::pipeline::Delivery>, + ) -> Result> { + options.check_cancelled()?; + if progress + .as_ref() + .is_some_and(|progress| progress.is_cancelled()) + { + return Err(Error::Cancelled); + } + let mut selected = BTreeMap::new(); + for &index in indices { + if selected.insert(index, None).is_some() { + return Err(Error::DuplicateEntry); + } + } + let Some(&last) = selected.keys().next_back() else { + return Ok(Vec::new()); + }; + let mut required = 0u64; + let mut found = 0; + let solid = match archive { + Archive::Rar13(a) => a.main.is_solid(), + Archive::Rar15To40(a) => a.main.is_solid(), + Archive::Rar50Plus(a) => a.main.is_solid(), + }; + let mut total = 0u64; + let mut entries = 0; + for (index, member) in archive.members().enumerate() { + options.check_cancelled()?; + if index <= last + && (selected.contains_key(&index) || solid) + && !member.meta.is_directory + && !member.meta.is_redirection + { + total = total.saturating_add(member.meta.unpacked_size); + entries += 1; + } + if !selected.contains_key(&index) { + continue; + } + let meta = member.meta; + if meta.is_directory + || meta.is_redirection + || meta.is_split_before + || meta.is_split_after + { + return Err(Error::InvalidArgument( + "rewrite staging requires unsplit payload members", + )); + } + let sum = if delivery.is_some() { + Some(required.max(meta.unpacked_size)) + } else { + required.checked_add(meta.unpacked_size) + }; + if sum.is_none_or(|sum| sum > staging.max_staged_bytes) { + return Err(Error::RewriteStagingLimitExceeded { + limit: staging.max_staged_bytes, + required: sum.unwrap_or(u64::MAX), + }); + } + required = sum.unwrap(); + found += 1; + } + if found != selected.len() { + return Err(Error::EntryNotFound); + } + let progress = progress.map(|callback| { + Rc::new(Progress { + callback, + completed: Cell::new(0), + total, + entries, + }) + }); + if let Some(progress) = &progress { + progress.report(WriteProgressEvent::OperationStarted { + operation: WriteOperation::Staging, + total_bytes: Some(total), + total_entries: Some(entries), + pass: 1, + })?; + } + let resources = WriterResources::new(0).with_temp_dir(&staging.directory); + let used = delivery.map_or_else( + || Arc::new(AtomicU64::new(0)), + |delivery| delivery.used.clone(), + ); + let mut index = 0; + let mut pending = None; + let mut decoded = 0; + let mut staged_pending = None; + archive.extract_with_control(options, |member| { + publish_pending(&mut staged_pending, &mut selected, delivery)?; + if let Some(progress) = &progress { + if let Some(entry) = pending.take() { + progress.finish_entry(&entry)?; + } + } + let current = index; + index += 1; + if current > last { + return Ok(ExtractionDecision::Stop); + } + let wanted = selected.contains_key(¤t); + let dependency = solid && !member.meta.is_directory && !member.meta.is_redirection; + if !wanted && !dependency { + return Ok(ExtractionDecision::Skip); + } + if let Some(delivery) = delivery { + // Dependencies are decoded only on demand for the next retained member. + let next = selected + .range(current..) + .next() + .map(|(&index, _)| index) + .unwrap_or(last); + delivery.wait_for(next)?; + } + if let Some(progress) = &progress { + progress.report(WriteProgressEvent::EntryStarted { + operation: WriteOperation::Staging, + index: decoded, + total_entries: entries, + name: &member.meta.name, + input_bytes: member.meta.unpacked_size, + })?; + pending = Some((decoded, member.meta.name.clone(), member.meta.unpacked_size)); + } + decoded += 1; + let sink: Box = if let Some(slot) = selected.get_mut(¤t) { + let spool = Rc::new(RefCell::new(Spool::create(&resources)?)); + *slot = Some(spool.clone()); + staged_pending = Some(current); + Box::new(Sink { + spool, + used: used.clone(), + limit: staging.max_staged_bytes, + }) + } else { + Box::new(std::io::sink()) + }; + Ok(ExtractionDecision::Extract( + if let Some(progress) = &progress { + Box::new(ProgressSink { + inner: sink, + progress: progress.clone(), + }) + } else { + sink + }, + )) + })?; + if let Some(progress) = &progress { + if let Some(entry) = pending.take() { + progress.finish_entry(&entry)?; + } + progress.report(WriteProgressEvent::OperationFinished { + operation: WriteOperation::Staging, + total_bytes: Some(total), + total_entries: Some(entries), + pass: 1, + })?; + } + publish_pending(&mut staged_pending, &mut selected, delivery)?; + if delivery.is_some() { + return Ok(Vec::new()); + } + indices + .iter() + .map(|index| { + let spool = selected + .remove(index) + .flatten() + .ok_or(Error::InvalidHeader("staged rewrite member disappeared"))?; + let spool = Rc::try_unwrap(spool) + .map_err(|_| Error::InvalidHeader("rewrite sink still active"))? + .into_inner(); + Ok(spool.into_source()) + }) + .collect() + } + + fn publish_pending( + pending: &mut Option, + selected: &mut BTreeMap>>>, + delivery: Option<&super::pipeline::Delivery>, + ) -> Result<()> { + if let (Some(index), Some(delivery)) = (pending.take(), delivery) { + let spool = selected + .remove(&index) + .flatten() + .ok_or(Error::WriterFailure("rewrite spool disappeared"))?; + let spool = Rc::try_unwrap(spool) + .map_err(|_| Error::WriterFailure("rewrite sink still active"))? + .into_inner(); + delivery.publish(index, spool.into_source())?; + } + Ok(()) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn progress_sink_forwards_flush_failure() { + struct FailingFlush(Arc); + + impl Write for FailingFlush { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.fetch_add(1, Ordering::Relaxed); + Err(std::io::Error::other("injected staging flush failure")) + } + } + + let calls = Arc::new(AtomicU64::new(0)); + let progress = Rc::new(Progress { + callback: Arc::new(|_: WriteProgressEvent<'_>| {}), + completed: Cell::new(0), + total: 0, + entries: 0, + }); + let mut sink = ProgressSink { + inner: Box::new(FailingFlush(calls.clone())), + progress, + }; + assert_eq!( + sink.flush().unwrap_err().to_string(), + "injected staging flush failure" + ); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } + + #[test] + fn runtime_limit_counts_all_sinks_and_refuses_before_writing() { + let root = crate::scratch::case("rewrite-runtime-limit"); + let resources = WriterResources::new(0).with_temp_dir(&*root); + let used = Arc::new(AtomicU64::new(0)); + let make_sink = || Sink { + spool: Rc::new(RefCell::new(Spool::create(&resources).unwrap())), + used: used.clone(), + limit: 5, + }; + let mut first = make_sink(); + let mut second = make_sink(); + first.write_all(b"abc").unwrap(); + second.write_all(b"de").unwrap(); + let error = Error::from(second.write_all(b"f").unwrap_err()); + assert_eq!( + error, + Error::RewriteStagingLimitExceeded { + limit: 5, + required: 6 + } + ); + assert_eq!(used.load(Ordering::Acquire), 5); + assert_eq!(second.spool.borrow().len(), 2); + } + + #[test] + fn spool_write_failure_releases_the_provisional_staging_charge() { + let root = crate::scratch::case("rewrite-spool-write-failure"); + let resources = WriterResources::new(0).with_temp_dir(&*root); + let used = Arc::new(AtomicU64::new(0)); + let mut sink = Sink { + spool: Rc::new(RefCell::new(Spool::create(&resources).unwrap())), + used: used.clone(), + limit: 16, + }; + sink.flush().unwrap(); + sink.spool.borrow_mut().park(); + let path = std::fs::read_dir(&root) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + std::fs::remove_file(path).unwrap(); + assert_eq!( + sink.write(b"payload").unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + assert_eq!(used.load(Ordering::Acquire), 0); + } + } +} diff --git a/arca-rar/vendor/rars/src/rewrite_staging/pipeline.rs b/arca-rar/vendor/rars/src/rewrite_staging/pipeline.rs new file mode 100644 index 00000000..23eb0e3a --- /dev/null +++ b/arca-rar/vendor/rars/src/rewrite_staging/pipeline.rs @@ -0,0 +1,496 @@ +//! Demand-driven, archive-order decoding for one scoped writer call. +use super::*; +use crate::atomic64::AtomicU64; +use crate::streaming::SourceFactory; +use crate::{Error, ReadCancellation}; +use std::collections::BTreeMap; +use std::io::{Read, Seek, SeekFrom}; +use std::sync::{atomic::Ordering, Condvar, Mutex}; + +#[derive(Default)] +struct State { + requested: Option, + ready: BTreeMap>, + error: Option, + done: bool, + published: std::collections::BTreeSet, +} + +pub(super) struct Delivery { + state: Mutex, + changed: Condvar, + cancellation: ReadCancellation, + pub(super) used: Arc, + max_preparation_window: usize, +} + +struct Lease { + source: Option, + size: u64, + used: Arc, +} +impl Drop for Lease { + fn drop(&mut self) { + drop(self.source.take()); + self.used.fetch_sub(self.size, Ordering::AcqRel); + } +} + +struct Reader { + reader: Box, + _lease: Arc, +} +impl Read for Reader { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + self.reader.read(bytes) + } +} +impl Seek for Reader { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.reader.seek(position) + } +} +struct Source { + index: usize, + size: u64, + delivery: Arc, +} +impl SourceFactory for Source { + fn max_preparation_window(&self) -> usize { + self.delivery.max_preparation_window + } + fn len(&self) -> Result { + Ok(self.size) + } + fn open(&self) -> Result> { + self.open_with_cancellation(|| self.delivery.cancellation.is_cancelled()) + } + fn release(&self) { + self.delivery + .state + .lock() + .unwrap() + .ready + .remove(&self.index); + self.delivery.changed.notify_all(); + } +} + +impl Source { + fn open_with_cancellation( + &self, + mut is_cancelled: impl FnMut() -> bool, + ) -> Result> { + let mut state = self.delivery.state.lock().unwrap(); + state.requested = Some( + state + .requested + .map_or(self.index, |index| index.max(self.index)), + ); + self.delivery.changed.notify_all(); + loop { + if let Some(error) = &state.error { + return Err(error.clone()); + } + if is_cancelled() { + return Err(Error::Cancelled); + } + if let Some(lease) = state.ready.get(&self.index).cloned() { + drop(state); + return Ok(Box::new(Reader { + reader: lease.source.as_ref().expect("live lease").open()?, + _lease: lease, + })); + } + if state.published.contains(&self.index) { + return Err(Error::WriterFailure("rewrite source already consumed")); + } + if state.done || is_cancelled() { + return Err(Error::Cancelled); + } + state = self.delivery.changed.wait(state).unwrap(); + } + } +} + +impl Delivery { + pub(super) fn wait_for(&self, index: usize) -> Result<()> { + let mut state = self.state.lock().unwrap(); + while state.requested.is_none_or(|requested| requested < index) { + if self.cancellation.is_cancelled() { + return Err(Error::Cancelled); + } + state = self + .changed + .wait_timeout(state, std::time::Duration::from_millis(50)) + .unwrap() + .0; + } + if self.cancellation.is_cancelled() { + return Err(Error::Cancelled); + } + Ok(()) + } + pub(super) fn publish(&self, index: usize, source: EntrySource) -> Result<()> { + let size = source.len()?; + if self.cancellation.is_cancelled() { + return Err(Error::Cancelled); + } + let mut state = self.state.lock().unwrap(); + state.published.insert(index); + state.ready.insert( + index, + Arc::new(Lease { + source: Some(source), + size, + used: self.used.clone(), + }), + ); + self.changed.notify_all(); + Ok(()) + } +} + +struct StopOnDrop(Arc); +impl Drop for StopOnDrop { + fn drop(&mut self) { + self.0.cancellation.cancel(); + self.0.state.lock().unwrap().ready.clear(); + self.0.changed.notify_all(); + } +} + +pub(super) fn run( + archive: &Archive, + indices: &[usize], + options: ArchiveReadOptions<'_>, + staging: &RewriteStaging, + progress: Option>, + consume: impl FnOnce(Vec) -> Result, +) -> Result { + options.check_cancelled()?; + let sizes: BTreeMap<_, _> = archive + .members() + .enumerate() + .map(|(i, member)| (i, member.meta.unpacked_size)) + .collect(); + // A preparation window can retain every source until its members are emitted. + // Bound it by the largest selected payload so even mixed-size windows fit. + let largest = indices + .iter() + .filter_map(|index| sizes.get(index)) + .copied() + .max() + .unwrap_or(0); + let max_preparation_window = usize::try_from(staging.max_staged_bytes / largest.max(1)) + .unwrap_or(usize::MAX) + .max(1); + let delivery = Arc::new(Delivery { + state: Mutex::new(State::default()), + changed: Condvar::new(), + cancellation: options + .cancellation + .map_or_else(ReadCancellation::new, ReadCancellation::child), + used: Arc::new(AtomicU64::new(0)), + max_preparation_window, + }); + let sources = indices + .iter() + .map(|&index| { + let size = *sizes.get(&index).ok_or(Error::EntryNotFound)?; + Ok(EntrySource::from_factory(Source { + index, + size, + delivery: delivery.clone(), + })) + }) + .collect::>>()?; + std::thread::scope(|scope| { + let worker = &delivery; + let decoder = scope.spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + super::native::stage( + archive, + indices, + options.with_cancellation(&worker.cancellation), + staging, + progress, + Some(worker), + ) + })) + .unwrap_or(Err(Error::WriterFailure("rewrite decoder thread panicked"))); + let mut state = worker.state.lock().unwrap(); + state.error = result.as_ref().err().cloned(); + state.done = true; + worker.changed.notify_all(); + result.map(|_| ()) + }); + // Unwind, writer failure and unused sources must all wake a waiting decoder. + let guard = StopOnDrop(delivery.clone()); + let result = consume(sources); + let requested = delivery.state.lock().unwrap().requested; + if result.is_err() || requested != indices.iter().copied().max() { + delivery.cancellation.cancel(); + delivery.changed.notify_all(); + } + let decoded = decoder + .join() + .map_err(|_| Error::WriterFailure("rewrite decoder thread panicked"))?; + drop(guard); + match result { + Err(error) => Err(error), + Ok(value) => { + finish_decoder(decoded, requested != indices.iter().copied().max())?; + Ok(value) + } + } + }) +} + +fn finish_decoder(decoded: Result<()>, unused_sources: bool) -> Result<()> { + match decoded { + // Stopping an unused source can race with entry selection, which adds + // context to cancellation. Its category is unchanged by that context. + Err(error) if unused_sources && error.kind() == crate::ErrorKind::Cancelled => Ok(()), + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + + #[test] + fn unused_source_shutdown_accepts_contextual_cancellation_only() { + let cancelled = Error::AtEntry { + name: b"file".to_vec(), + operation: "selecting", + source: Box::new(Error::Cancelled), + }; + assert_eq!(finish_decoder(Err(cancelled.clone()), true), Ok(())); + assert_eq!( + finish_decoder(Err(cancelled.clone()), false), + Err(cancelled) + ); + let error = Error::InvalidArgument("decoder refused"); + assert_eq!(finish_decoder(Err(error.clone()), true), Err(error)); + } + + struct CancelAtStagingFinish(AtomicBool); + + impl crate::WriteProgress for CancelAtStagingFinish { + fn report(&self, event: crate::WriteProgressEvent<'_>) { + if matches!( + event, + crate::WriteProgressEvent::OperationFinished { + operation: crate::WriteOperation::Staging, + .. + } + ) { + self.0.store(true, Ordering::Relaxed); + } + } + + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + + fn delivery() -> Arc { + Arc::new(Delivery { + state: Mutex::new(State::default()), + changed: Condvar::new(), + cancellation: ReadCancellation::new(), + used: Arc::new(AtomicU64::new(0)), + max_preparation_window: usize::MAX, + }) + } + + #[test] + fn rewrite_source_reopens_a_live_lease_and_refuses_a_consumed_one() { + let delivery = delivery(); + delivery.used.store(6, Ordering::Relaxed); + delivery.state.lock().unwrap().ready.insert( + 0, + Arc::new(Lease { + source: Some(EntrySource::from_bytes(b"abcdef".to_vec())), + size: 6, + used: delivery.used.clone(), + }), + ); + let source = Source { + index: 0, + size: 6, + delivery: delivery.clone(), + }; + let mut reader = source.open().unwrap(); + assert_eq!(reader.seek(SeekFrom::Start(3)).unwrap(), 3); + let mut suffix = Vec::new(); + reader.read_to_end(&mut suffix).unwrap(); + assert_eq!(suffix, b"def"); + delivery.wait_for(0).unwrap(); + source.release(); + assert_eq!(delivery.used.load(Ordering::Relaxed), 6); + drop(reader); + assert_eq!(delivery.used.load(Ordering::Relaxed), 0); + delivery.state.lock().unwrap().published.insert(0); + assert!(matches!( + source.open(), + Err(Error::WriterFailure("rewrite source already consumed")) + )); + delivery.state.lock().unwrap().published.clear(); + delivery.state.lock().unwrap().done = true; + assert!(matches!(source.open(), Err(Error::Cancelled))); + } + + #[test] + fn cancelled_delivery_refuses_waits_opens_and_publication() { + let delivery = delivery(); + delivery.cancellation.cancel(); + let source = Source { + index: 0, + size: 1, + delivery: delivery.clone(), + }; + assert!(matches!(delivery.wait_for(1), Err(Error::Cancelled))); + assert!(matches!(source.open(), Err(Error::Cancelled))); + assert!(matches!(delivery.wait_for(0), Err(Error::Cancelled))); + assert!(matches!( + delivery.publish(0, EntrySource::from_bytes(b"x".to_vec())), + Err(Error::Cancelled) + )); + assert_eq!(delivery.used.load(Ordering::Relaxed), 0); + } + + #[test] + fn cancellation_between_observations_refuses_to_wait_for_a_source() { + let delivery = delivery(); + let source = Source { + index: 0, + size: 1, + delivery: delivery.clone(), + }; + let mut observations = 0; + let result = source.open_with_cancellation(|| { + let cancelled = delivery.cancellation.is_cancelled(); + observations += 1; + if observations == 1 { + // Model cancellation after the first atomic load returned false. + delivery.cancellation.cancel(); + } + cancelled + }); + assert!(matches!(result, Err(Error::Cancelled))); + assert_eq!(observations, 2); + let state = delivery.state.lock().unwrap(); + assert_eq!(state.requested, Some(0)); + assert!(!state.done); + assert!(state.ready.is_empty()); + assert!(state.published.is_empty()); + assert_eq!(delivery.used.load(Ordering::Relaxed), 0); + } + + #[test] + fn published_source_delivers_bytes_and_releases_its_charge() { + let delivery = delivery(); + let source = Source { + index: 0, + size: 7, + delivery: delivery.clone(), + }; + assert_eq!(source.len().unwrap(), 7); + // Native staging charges the bytes before publishing the lease. + delivery.used.store(7, Ordering::Relaxed); + delivery + .publish(0, EntrySource::from_bytes(b"payload".to_vec())) + .unwrap(); + let mut reader = source.open().unwrap(); + delivery.wait_for(0).unwrap(); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"payload"); + source.release(); + assert_eq!(delivery.used.load(Ordering::Relaxed), 7); + drop(reader); + assert_eq!(delivery.used.load(Ordering::Relaxed), 0); + assert!(matches!( + source.open(), + Err(Error::WriterFailure("rewrite source already consumed")) + )); + } + + #[test] + fn writer_that_never_opens_its_rewrite_source_stops_the_decoder() { + let root = crate::scratch::case("rewrite-unused-source"); + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 7, + }; + + let value = run( + &archive, + &[0], + ArchiveReadOptions::default(), + &staging, + None, + |sources| { + assert_eq!(sources.len(), 1); + Ok(17) + }, + ) + .unwrap(); + assert_eq!(value, 17); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + + assert_eq!( + run( + &archive, + &[0], + ArchiveReadOptions::default(), + &staging, + None, + |_sources| -> Result<()> { Err(Error::InvalidArgument("writer refused")) }, + ) + .unwrap_err(), + Error::InvalidArgument("writer refused") + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + + #[test] + fn cancellation_after_final_source_request_reaches_writer() { + let root = crate::scratch::case("rewrite-final-request-cancelled"); + let mut builder = crate::Builder::new(crate::ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = crate::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 7, + }; + let progress = Arc::new(CancelAtStagingFinish(AtomicBool::new(false))); + + let result = run( + &archive, + &[0], + ArchiveReadOptions::default(), + &staging, + Some(progress.clone()), + |sources| { + let error = sources[0].open().err().expect("staging should cancel"); + assert_eq!(error.kind(), crate::ErrorKind::Cancelled, "{error:?}"); + Ok(17) + }, + ); + assert!(progress.0.load(Ordering::Relaxed)); + assert_eq!(result.unwrap_err(), Error::Cancelled); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/source.rs b/arca-rar/vendor/rars/src/source.rs new file mode 100644 index 00000000..1789157f --- /dev/null +++ b/arca-rar/vendor/rars/src/source.rs @@ -0,0 +1,253 @@ +use crate::error::{Error, Result}; +use crate::io_util::read_exact_at; +use std::fs::File; +use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +use std::ops::Range; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +trait ReadSeek: Read + Seek + Send {} +impl ReadSeek for T {} + +pub(crate) struct ReaderSource { + reader: Mutex>, + len: usize, +} + +impl std::fmt::Debug for ReaderSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReaderSource") + .field("len", &self.len) + .finish_non_exhaustive() + } +} + +impl ReaderSource { + pub(crate) fn new(mut reader: impl Read + Seek + Send + 'static) -> Result> { + let len = usize::try_from(reader.seek(SeekFrom::End(0))?) + .map_err(|_| Error::InvalidHeader("archive size overflows host address size"))?; + Ok(Arc::new(Self { + reader: Mutex::new(Box::new(reader)), + len, + })) + } + + pub(crate) fn cursor(self: &Arc) -> SourceCursor { + SourceCursor { + source: self.clone(), + position: 0, + } + } +} + +pub(crate) struct SourceCursor { + source: Arc, + position: u64, +} + +impl Read for SourceCursor { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + let remaining = (self.source.len as u64).saturating_sub(self.position); + let count = bytes.len().min(remaining as usize); + if count == 0 { + return Ok(0); + } + let mut reader = self + .source + .reader + .lock() + .map_err(|_| std::io::Error::other("archive reader lock poisoned"))?; + // The seek and read share one lock; sibling cursors never share a position. + reader.seek(SeekFrom::Start(self.position))?; + let count = reader.read(&mut bytes[..count])?; + self.position += count as u64; + Ok(count) + } +} + +impl Seek for SourceCursor { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + let position = match from { + SeekFrom::Start(position) => position as i128, + SeekFrom::Current(offset) => self.position as i128 + offset as i128, + SeekFrom::End(offset) => self.source.len as i128 + offset as i128, + }; + self.position = u64::try_from(position) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidInput))?; + Ok(self.position) + } +} + +/// Range cursors contain only inline state and references to the archive's +/// existing source. Reading a member does not need another heap allocation. +pub(crate) enum RangeReader<'a> { + Memory(Cursor<&'a [u8]>), + File(std::io::Take), + Reader(std::io::Take), +} +impl Read for RangeReader<'_> { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + match self { + Self::Memory(reader) => reader.read(bytes), + Self::File(reader) => reader.read(bytes), + Self::Reader(reader) => reader.read(bytes), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) enum ArchiveSource { + Memory(Arc<[u8]>), + File(Arc), + Reader(Arc), +} + +impl ArchiveSource { + pub(crate) fn read_range(&self, range: Range) -> Result> { + match self { + Self::Memory(data) => data + .get(range) + .map(|data| data.to_vec()) + .ok_or(Error::TooShort), + Self::File(path) => { + let mut file = File::open(path.as_ref())?; + read_exact_at(&mut file, range.start, range.len()) + } + Self::Reader(source) => read_exact_at(&mut source.cursor(), range.start, range.len()), + } + } + + pub(crate) fn copy_range_to(&self, range: Range, writer: &mut dyn Write) -> Result<()> { + match self { + Self::Reader(_) => { + let mut reader = self.range_reader(range)?; + std::io::copy(&mut reader, writer)?; + } + Self::Memory(data) => { + let data = data.get(range).ok_or(Error::TooShort)?; + writer.write_all(data)?; + } + Self::File(path) => { + let mut file = File::open(path.as_ref())?; + file.seek(SeekFrom::Start(range.start as u64))?; + let mut limited = file.take(range.len() as u64); + std::io::copy(&mut limited, writer)?; + } + } + Ok(()) + } + + pub(crate) fn range_reader(&self, range: Range) -> Result> { + match self { + Self::Reader(source) => { + if range.start > range.end || range.end > source.len { + return Err(Error::TooShort); + } + let mut reader = source.cursor(); + reader.seek(SeekFrom::Start(range.start as u64))?; + Ok(RangeReader::Reader(reader.take(range.len() as u64))) + } + Self::Memory(data) => { + let data = data.get(range).ok_or(Error::TooShort)?; + Ok(RangeReader::Memory(Cursor::new(data))) + } + Self::File(path) => { + let mut file = File::open(path.as_ref())?; + file.seek(SeekFrom::Start(range.start as u64))?; + Ok(RangeReader::File(file.take(range.len() as u64))) + } + } + } + + pub(crate) fn len(&self) -> Result { + match self { + Self::Reader(source) => Ok(source.len), + Self::Memory(data) => Ok(data.len()), + Self::File(path) => usize::try_from(std::fs::metadata(path.as_ref())?.len()) + .map_err(|_| Error::InvalidHeader("archive size overflows host address size")), + } + } + + #[cfg(feature = "recovery")] + pub(crate) fn bytes(&self) -> Result> { + match self { + Self::Reader(source) => self.read_range(0..source.len), + Self::Memory(data) => Ok(data.to_vec()), + Self::File(path) => Ok(std::fs::read(path.as_ref())?), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sibling_cursors_keep_independent_positions_and_range_boundaries() { + let source = ReaderSource::new(Cursor::new(b"0123456789".to_vec())).unwrap(); + let mut first = source.cursor(); + let mut second = source.cursor(); + second.seek(SeekFrom::End(-3)).unwrap(); + let mut bytes = [0; 2]; + first.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"01"); + second.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"78"); + first.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"23"); + assert!(first.seek(SeekFrom::Current(-5)).is_err()); + assert_eq!(first.stream_position().unwrap(), 4); + let source = ArchiveSource::Reader(source); + let mut range = source.range_reader(4..6).unwrap(); + let mut result = Vec::new(); + range.read_to_end(&mut result).unwrap(); + assert_eq!(result, b"45"); + assert!(source.range_reader(9..11).is_err()); + let start = 6; + let end = 4; + assert!(source.range_reader(start..end).is_err()); + } + + #[test] + fn empty_and_exhausted_cursors_do_not_read_the_underlying_source() { + let source = ReaderSource::new(Cursor::new(b"0123".to_vec())).unwrap(); + assert_eq!(format!("{source:?}"), "ReaderSource { len: 4, .. }"); + let mut cursor = source.cursor(); + assert_eq!(cursor.read(&mut []).unwrap(), 0); + assert_eq!(cursor.stream_position().unwrap(), 0); + cursor.seek(SeekFrom::End(0)).unwrap(); + let mut bytes = [0xff; 2]; + assert_eq!(cursor.read(&mut bytes).unwrap(), 0); + assert_eq!(bytes, [0xff; 2]); + cursor.seek(SeekFrom::End(10)).unwrap(); + assert_eq!(cursor.read(&mut bytes).unwrap(), 0); + assert_eq!(cursor.stream_position().unwrap(), 14); + } + + #[test] + fn a_panicking_reader_leaves_sibling_cursors_with_a_typed_io_error() { + struct PanicReader(Cursor>); + impl Seek for PanicReader { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.0.seek(from) + } + } + impl Read for PanicReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + panic!("reader failed during callback"); + } + } + let source = ReaderSource::new(PanicReader(Cursor::new(vec![1]))).unwrap(); + let mut first = source.cursor(); + let mut sibling = source.cursor(); + assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + first.read_exact(&mut [0]).unwrap(); + })) + .is_err()); + let error = sibling.read(&mut [0]).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::Other); + assert_eq!(error.to_string(), "archive reader lock poisoned"); + assert!(matches!(Error::from(error), Error::Io(_))); + assert_eq!(sibling.stream_position().unwrap(), 0); + } +} diff --git a/arca-rar/vendor/rars/src/streaming.rs b/arca-rar/vendor/rars/src/streaming.rs new file mode 100644 index 00000000..ec38da5b --- /dev/null +++ b/arca-rar/vendor/rars/src/streaming.rs @@ -0,0 +1,1634 @@ +use crate::{Error, Result}; +#[cfg(any(test, all(target_arch = "wasm32", target_os = "unknown")))] +mod memory_spool; +pub(crate) mod output; +pub(crate) mod preparation; +use std::fmt; +use std::fs::File; +use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +#[cfg(test)] +use std::sync::atomic::AtomicU64; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +/// Default aggregate compression workspace budget (256 MiB). +pub const DEFAULT_WRITER_MEMORY_LIMIT: u64 = 256 * 1024 * 1024; + +/// A rewindable input reader used by streaming archive writers. +pub trait EntryReader: Read + Seek + Send {} +impl EntryReader for T {} + +pub(crate) trait SourceFactory: Send + Sync { + /// Bound simultaneous preparation when a scoped producer retains plaintext. + fn max_preparation_window(&self) -> usize { + usize::MAX + } + fn release(&self) {} + fn len(&self) -> Result; + fn open(&self) -> Result>; +} + +#[derive(Clone)] +/// A reopenable byte source for an archive member. +/// +/// Reopening is not a snapshot: callers must keep the source stable for the +/// duration of a write. Writers check emitted stored data against the prepared +/// size and archive checksums. A mismatch fails the write, but a caller-provided +/// output stream can already contain partial data when the error is returned. +pub struct EntrySource(Arc); + +impl fmt::Debug for EntrySource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EntrySource") + .field("len", &self.len().ok()) + .finish_non_exhaustive() + } +} + +impl EntrySource { + pub(crate) fn from_factory(factory: impl SourceFactory + 'static) -> Self { + Self(Arc::new(factory)) + } + + pub(crate) fn max_preparation_window(&self) -> usize { + self.0.max_preparation_window() + } + + /// Release a session-owned payload once this write will never reopen it. + pub(crate) fn release(&self) { + self.0.release(); + } + + pub fn from_bytes(data: impl Into>) -> Self { + Self::from_factory(MemorySource(data.into())) + } + + pub(crate) fn copy_for_writer(data: &[u8], resources: &WriterResources) -> Result { + let Some(mut charge) = resources.execution_charge() else { + return Ok(Self::from_bytes(Arc::<[u8]>::from(data))); + }; + let bytes = data + .len() + .checked_add(std::mem::size_of::()) + .and_then(|bytes| bytes.checked_add(std::mem::size_of::())) + .ok_or(Error::InvalidArgument("writer source capacity overflows"))?; + charge.grow_to(bytes as u64)?; + let data = Arc::new(WriterMemoryData { + bytes: data.to_vec(), + _charge: charge, + }); + Ok(Self::from_factory(WriterMemorySource { + data, + resources: resources.clone(), + })) + } + + pub fn from_path(path: impl Into) -> Self { + Self::from_factory(PathSource(path.into())) + } + + pub fn from_opener(len: u64, open: F) -> Self + where + F: Fn() -> Result> + Send + Sync + 'static, + { + Self::from_factory(OpenerSource { + len, + open: Arc::new(open), + }) + } + + pub fn len(&self) -> Result { + self.0.len() + } + + pub fn is_empty(&self) -> Result { + self.len().map(|len| len == 0) + } + + pub fn open(&self) -> Result> { + self.0.open() + } +} + +struct MemorySource(Arc<[u8]>); + +impl SourceFactory for MemorySource { + fn len(&self) -> Result { + Ok(self.0.len() as u64) + } + + fn open(&self) -> Result> { + Ok(Box::new(Cursor::new(Arc::clone(&self.0)))) + } +} + +// The shared owner keeps both its bytes and its charge alive if a reader +// outlives the converted entry. Reader boxes get separate capacity charges. +struct WriterMemoryData { + bytes: Vec, + _charge: CapacityCharge, +} +struct WriterMemorySource { + data: Arc, + resources: WriterResources, +} +struct WriterMemoryReader { + data: Arc, + position: u64, + _charge: CapacityCharge, +} +impl SourceFactory for WriterMemorySource { + fn len(&self) -> Result { + Ok(self.data.bytes.len() as u64) + } + fn open(&self) -> Result> { + // WriterMemorySource is constructed only after copy_for_writer has + // admitted an execution charge from these same resources. + let mut charge = self + .resources + .execution_charge() + .expect("writer memory source has an execution allowance"); + charge.grow_to(std::mem::size_of::() as u64)?; + Ok(Box::new(WriterMemoryReader { + data: self.data.clone(), + position: 0, + _charge: charge, + })) + } +} +impl Read for WriterMemoryReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let mut cursor = Cursor::new(&self.data.bytes); + cursor.set_position(self.position); + let count = cursor.read(out)?; + self.position = cursor.position(); + Ok(count) + } +} +impl Seek for WriterMemoryReader { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + let mut cursor = Cursor::new(&self.data.bytes); + cursor.set_position(self.position); + self.position = cursor.seek(from)?; + Ok(self.position) + } +} + +struct PathSource(PathBuf); + +struct OpenerSource { + len: u64, + open: Arc Result> + Send + Sync>, +} + +impl SourceFactory for OpenerSource { + fn len(&self) -> Result { + Ok(self.len) + } + + fn open(&self) -> Result> { + (self.open)() + } +} + +impl SourceFactory for PathSource { + fn len(&self) -> Result { + Ok(std::fs::metadata(&self.0)?.len()) + } + + fn open(&self) -> Result> { + Ok(Box::new(File::open(&self.0)?)) + } +} + +/// A shared, one-way cancellation signal for archive writes. +/// +/// Clone the token for another thread, or cancel it from an input/output callback. +/// Cancellation is cooperative: it is checked between chunks and during resource +/// waits; it cannot interrupt a caller's blocked I/O or an individual codec step. +#[derive(Clone, Debug, Default)] +pub struct WriteCancellation(Arc); + +impl WriteCancellation { + pub fn new() -> Self { + Self::default() + } + pub fn cancel(&self) { + self.0.store(true, Ordering::Relaxed); + } + pub fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } +} + +#[derive(Clone, Debug)] +/// Shared compression-workspace and temporary-file policy for streaming writers. +/// +/// Native RAR5 writers retain packed payloads on disk and bound active input and +/// spool handles by compression concurrency, rather than the archive entry count. +/// The workspace limit is not a total process-RAM or temporary-disk quota. On bare +/// WebAssembly (`wasm32-unknown-unknown`), spools stay in memory and their retained +/// payloads are additional to this limit. `with_max_spool_bytes` separately caps +/// logical spool contents on both backends. `with_max_spool_memory_bytes` caps +/// bare-WASM spool payload and index capacity, excluding other writer memory. +pub struct WriterResources { + memory_limit: u64, + temp_dir: Option>, + budget: Arc, + spool_budget: Option>, + spool_memory_budget: Option>, + prepared_header_budget: Option>, + preparation_budget: Option>, + cancellation: Option, + pub(crate) execution: Option, +} + +impl Default for WriterResources { + fn default() -> Self { + Self::new(DEFAULT_WRITER_MEMORY_LIMIT) + } +} + +impl WriterResources { + pub fn new(memory_limit: u64) -> Self { + Self { + memory_limit, + temp_dir: None, + budget: Arc::new(MemoryBudget::new(memory_limit)), + spool_budget: None, + spool_memory_budget: None, + prepared_header_budget: None, + preparation_budget: None, + cancellation: None, + execution: None, + } + } + + /// Set a hard ceiling on managed writer allocation capacity for RAR5/7. + /// Includes active work, retained spools/headers, conversion and owned output. + /// Caller-owned inputs and external sinks, allocator/runtime overhead, stacks + /// and OS memory are excluded. This is not a process-RAM limit. + /// Legacy writers refuse this policy. The estimated workspace policy remains + /// independent. Configure before execution; clones share the new ledger. + pub fn with_max_memory_bytes(mut self, limit: u64) -> Self { + self.execution = Some(crate::codec::workspace::Allowance::limited(limit)); + self + } + + pub fn max_memory_bytes(&self) -> Option { + self.execution + .as_ref() + .map(crate::codec::workspace::Limited::limit) + } + + /// Live managed capacity plus reservations for admitted workers. + pub fn managed_memory_in_use(&self) -> u64 { + self.execution + .as_ref() + .map_or(0, crate::codec::workspace::Limited::used) + } + + pub(crate) fn execution_charge(&self) -> Option { + CapacityCharge::new(self, &None) + } + + /// Cap the sum of live logical spool lengths, including reserved growth. + /// Native spools use files; bare-WASM spools use memory. This does not cap + /// filesystem allocation, Vec capacity, codec workspace or final output. + /// Zero allows empty spools only. Growth fails immediately rather than waiting. + /// + /// Configuring this creates a fresh quota group; subsequent clones share it. + /// Configure before dispatching work. Parking a spool or returning from a + /// write does not release storage still owned by an EntrySource or reader. + /// Failed native cleanup retains its charge conservatively in this group. + pub fn with_max_spool_bytes(mut self, limit: u64) -> Self { + self.spool_budget = Some(Arc::new(StorageBudget { + resource: StorageResource::LogicalBytes, + limit, + used: Mutex::new(0), + #[cfg(test)] + refusal: None, + })); + self + } + + /// The optional logical spool quota, independent of `memory_limit()`. + pub fn max_spool_bytes(&self) -> Option { + self.spool_budget.as_ref().map(|budget| budget.limit) + } + + /// Cap shared in-memory spool payload and index capacity on bare WASM. + /// Bounded spools allocate zeroed 4096-byte blocks and a boxed index. Growth + /// reserves both old and replacement indexes before allocation. Allocator + /// overhead, codec workspace and output are not included. + /// This is not an aggregate managed-memory or process-RAM ceiling. + /// + /// Native file spools have no in-memory payload charge. The default remains + /// an unbounded Vec backend on WASM; zero permits only empty memory spools. + /// Configuring creates a fresh group, shared by subsequent resource clones. + /// Logical spool lengths remain independently limited by `max_spool_bytes`. + pub fn with_max_spool_memory_bytes(mut self, limit: u64) -> Self { + self.spool_memory_budget = Some(Arc::new(StorageBudget { + resource: StorageResource::PayloadMemory, + limit, + used: Mutex::new(0), + #[cfg(test)] + refusal: None, + })); + self + } + + /// The optional memory-spool payload and index capacity quota. + pub fn max_spool_memory_bytes(&self) -> Option { + self.spool_memory_budget.as_ref().map(|budget| budget.limit) + } + + /// Cap final RAR5/7 member and service header images retained by single-archive + /// preparation. Counts encryption IVs and padding, with admission before + /// allocation. Serializer scratch, main/end headers, volume fragment headers, + /// legacy headers and coordinator records are outside this quota. + /// Defaults to unlimited. Configuring creates a fresh group shared by clones. + pub fn with_max_prepared_header_bytes(mut self, limit: u64) -> Self { + self.prepared_header_budget = Some(Arc::new(StorageBudget { + resource: StorageResource::PreparedHeaders, + limit, + used: Mutex::new(0), + #[cfg(test)] + refusal: None, + })); + self + } + + /// The optional quota for retained RAR5/7 prepared header images. + pub fn max_prepared_header_bytes(&self) -> Option { + self.prepared_header_budget + .as_ref() + .map(|budget| budget.limit) + } + + pub(crate) fn reserve_prepared_header(&self, bytes: u64) -> Result> { + self.prepared_header_budget + .as_ref() + .map(|budget| { + let mut charge = StorageCharge { + budget: budget.clone(), + bytes: 0, + }; + charge.grow_to(bytes)?; + Ok(charge) + }) + .transpose() + } + + /// Bound RAR5/7 engine header scratch, images, descriptor arrays and boxed + /// preparation state, including volume and recovery headers. Capacity is + /// reserved before allocation, including old/replacement overlap on growth. + /// Configuring creates a new ledger shared by resource clones; default is + /// unlimited. Legacy writers explicitly refuse this policy. + /// + /// Coordinator plan/job/result descriptor arrays also count; their pointed-to + /// input/history and codec payload buffers do not. Codec/KDF/recovery workspace, + /// input conversion, source/sink callbacks, collectors, diagnostics and allocator + /// overhead are outside this quota. This is not a total writer-memory cap. + pub fn with_max_preparation_bytes(mut self, limit: u64) -> Self { + self.preparation_budget = Some(Arc::new(StorageBudget { + resource: StorageResource::Preparation, + limit, + used: Mutex::new(0), + #[cfg(test)] + refusal: None, + })); + self + } + + #[cfg(test)] + pub(crate) fn refuse_preparation_growth_at( + self, + fail_at: usize, + ) -> (Self, Arc) { + let mut resources = self.with_max_preparation_bytes(u64::MAX); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + Arc::get_mut(resources.preparation_budget.as_mut().unwrap()) + .unwrap() + .refusal = Some((attempts.clone(), fail_at)); + (resources, attempts) + } + + /// The optional shared RAR5/7 engine preparation capacity limit. + pub fn max_preparation_bytes(&self) -> Option { + self.preparation_budget.as_ref().map(|budget| budget.limit) + } + + pub(crate) fn preparation_charge(&self) -> Option { + CapacityCharge::new(self, &self.preparation_budget) + } + + #[cfg(any(test, all(target_arch = "wasm32", target_os = "unknown")))] + fn spool_capacity_charge(&self) -> Option { + CapacityCharge::new(self, &self.spool_memory_budget) + } + + // A coordinator supplies a child scope to its admitted workers. + pub(crate) fn with_execution_allowance( + mut self, + allowance: crate::codec::workspace::Limited, + ) -> Self { + self.execution = Some(allowance); + self + } + + /// Place temporary spools in this existing directory (default: the current + /// directory). Use a directory whose contents untrusted users cannot replace: + /// idle spools are closed and later reopened by path. + /// + /// Spools may contain unencrypted compressed data even for encrypted output. + /// Files are created exclusively, with owner-only access on Unix (0600 before + /// applying the umask); other platforms use inherited directory permissions. + /// Dropping a spool closes its handle before attempting removal, including on + /// errors and unwind. Removal is best-effort, not secure erasure, and process + /// termination can leave files behind. Bare WASM ignores this setting. + pub fn with_temp_dir(mut self, path: impl Into) -> Self { + self.temp_dir = Some(Arc::new(path.into())); + self + } + + /// Attach a cancellation token to archive writes using this policy. + /// The token works without a progress callback. Cancellation returns + /// [`Error::Cancelled`]; a direct output sink may already contain partial data. + pub fn with_cancellation(mut self, cancellation: WriteCancellation) -> Self { + self.cancellation = Some(cancellation); + self + } + + pub(crate) fn has_cancellation(&self) -> bool { + self.cancellation.is_some() + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.cancellation + .as_ref() + .is_some_and(WriteCancellation::is_cancelled) + } + + pub fn memory_limit(&self) -> u64 { + self.memory_limit + } + + pub fn temp_dir(&self) -> Option<&Path> { + self.temp_dir.as_deref().map(PathBuf::as_path) + } + + #[cfg(test)] + pub(crate) fn workspace_in_use(&self) -> u64 { + *self.budget.used.lock().unwrap() + } + + #[cfg(test)] + pub(crate) fn preparation_in_use(&self) -> u64 { + self.preparation_budget + .as_ref() + .map_or(0, |budget| *budget.used.lock().unwrap()) + } + + #[cfg(test)] + pub(crate) fn acquire(&self, required: u64, dictionary_size: u64) -> Result { + self.acquire_cancellable(required, dictionary_size, &|| false) + } + + pub(crate) fn acquire_cancellable( + &self, + required: u64, + dictionary_size: u64, + cancelled: &dyn Fn() -> bool, + ) -> Result { + let cancelled = || self.is_cancelled() || cancelled(); + if cancelled() { + return Err(Error::Cancelled); + } + if required > self.memory_limit { + return Err(Error::MemoryLimitExceeded { + limit: self.memory_limit, + required, + dictionary_size, + }); + } + self.budget.acquire_cancellable(required, &cancelled) + } + + /// Reserves what a member needs, or the whole budget if one member needs + /// more than that. + /// + /// The RAR 1.3 to 4.x codecs compress a member as a unit, so there is no + /// smaller piece to fall back to when one does not fit. Here the budget + /// decides how many members are compressed at once rather than whether the + /// job can run at all, and a member larger than the budget runs alone. + pub(crate) fn acquire_serialising_cancellable( + &self, + required: u64, + cancelled: &dyn Fn() -> bool, + ) -> Result { + self.budget + .acquire_cancellable(required.min(self.memory_limit), &|| { + self.is_cancelled() || cancelled() + }) + } +} + +/// Live storage and reservations share one ledger across resource clones. +/// No waiter can make progress by waiting for spools it must itself retain. +#[derive(Debug, Clone, Copy)] +enum StorageResource { + LogicalBytes, + PayloadMemory, + PreparedHeaders, + Preparation, +} + +#[derive(Debug)] +struct StorageBudget { + resource: StorageResource, + limit: u64, + used: Mutex, + #[cfg(test)] + refusal: Option<(Arc, usize)>, +} + +#[derive(Debug)] +pub(crate) struct StorageCharge { + budget: Arc, + bytes: u64, +} + +impl StorageCharge { + fn new(resources: &WriterResources) -> Option { + resources.spool_budget.as_ref().map(|budget| Self { + budget: budget.clone(), + bytes: 0, + }) + } + + pub(crate) fn grow_to(&mut self, bytes: u64) -> Result<()> { + let growth = bytes.saturating_sub(self.bytes); + if growth == 0 { + return Ok(()); + } + #[cfg(test)] + if let Some((attempts, fail_at)) = &self.budget.refusal { + if attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed) == *fail_at { + return Err(Error::WriterFailure( + "injected preparation admission failure", + )); + } + } + let mut used = self.budget.used.lock().expect("spool budget lock poisoned"); + let required = used.checked_add(growth); + if required.is_none_or(|required| required > self.budget.limit) { + let required = required.unwrap_or(u64::MAX); + return Err(match self.budget.resource { + StorageResource::LogicalBytes => Error::WriterSpoolLimitExceeded { + limit: self.budget.limit, + required, + used: *used, + }, + StorageResource::Preparation => Error::WriterPreparationLimitExceeded { + limit: self.budget.limit, + required, + used: *used, + }, + StorageResource::PreparedHeaders => Error::WriterPreparedHeaderLimitExceeded { + limit: self.budget.limit, + required, + used: *used, + }, + StorageResource::PayloadMemory => Error::WriterSpoolMemoryLimitExceeded { + limit: self.budget.limit, + required, + used: *used, + }, + }); + } + *used = required.unwrap(); + self.bytes = bytes; + Ok(()) + } + + fn shrink_to(&mut self, bytes: u64) { + let released = self.bytes - bytes; + if released != 0 { + let mut used = self.budget.used.lock().expect("spool budget lock poisoned"); + *used -= released; + self.bytes = bytes; + } + } +} + +impl Drop for StorageCharge { + fn drop(&mut self) { + self.shrink_to(0); + } +} + +/// A capacity owner can satisfy its existing class quota and an execution +/// reservation together. Failed admission rolls both ledgers back. +#[derive(Debug)] +pub(crate) enum CapacityCharge { + Storage(StorageCharge), + Execution { + storage: Option, + charge: crate::codec::workspace::Charge, + }, +} +impl CapacityCharge { + fn new(_resources: &WriterResources, budget: &Option>) -> Option { + let storage = budget.as_ref().map(|budget| StorageCharge { + budget: budget.clone(), + bytes: 0, + }); + if let Some(allowance) = &_resources.execution { + return Some(Self::Execution { + storage, + charge: crate::codec::workspace::Budget::charge(allowance), + }); + } + storage.map(Self::Storage) + } + fn bytes(&self) -> u64 { + match self { + Self::Storage(storage) => storage.bytes, + Self::Execution { charge, .. } => charge.bytes(), + } + } + pub(crate) fn grow_to(&mut self, bytes: u64) -> Result<()> { + if bytes <= self.bytes() { + return Ok(()); + } + match self { + Self::Storage(storage) => storage.grow_to(bytes), + Self::Execution { storage, charge } => { + use crate::codec::workspace::{Budget, Limited}; + let old = charge.bytes(); + Limited::resize(charge, bytes)?; + if let Some(storage) = storage { + if let Err(error) = storage.grow_to(bytes) { + Limited::resize(charge, old).expect("rolling back capacity cannot fail"); + return Err(error); + } + } + Ok(()) + } + } + } + fn shrink_to(&mut self, bytes: u64) { + match self { + Self::Storage(storage) => storage.shrink_to(bytes), + Self::Execution { storage, charge } => { + use crate::codec::workspace::{Budget, Limited}; + if let Some(storage) = storage { + storage.shrink_to(bytes); + } + Limited::resize(charge, bytes).expect("releasing capacity cannot fail"); + } + } + } +} + +#[derive(Debug)] +struct MemoryBudget { + limit: u64, + used: Mutex, + changed: Condvar, +} + +impl MemoryBudget { + fn new(limit: u64) -> Self { + Self { + limit, + used: Mutex::new(0), + changed: Condvar::new(), + } + } + + fn acquire_cancellable( + self: &Arc, + bytes: u64, + cancelled: &dyn Fn() -> bool, + ) -> Result { + // Call user cancellation code outside the budget mutex: a callback can + // release a permit, and must not deadlock with its own admission wait. + loop { + if cancelled() { + return Err(Error::Cancelled); + } + let mut used = self.used.lock().expect("memory budget lock poisoned"); + if self.limit.saturating_sub(*used) >= bytes { + *used += bytes; + return Ok(MemoryPermit { + budget: Arc::clone(self), + bytes, + }); + } + // Tokens and progress callbacks need not notify this condition + // variable, so wake periodically even while all permits are held. + drop( + self.changed + .wait_timeout(used, std::time::Duration::from_millis(25)) + .expect("memory budget lock poisoned while waiting"), + ); + } + } +} + +pub(crate) struct MemoryPermit { + budget: Arc, + bytes: u64, +} + +/// Where a spool keeps its bytes. +/// +/// A file on every real platform, which is the point: the RAR 5 writer spools +/// so that a member larger than the memory budget still gets written. Bare +/// WebAssembly has no filesystem, so there it is a buffer, and the budget stops +/// covering their retained storage. The optional logical spool quota applies +/// to both backends, independently of Vec capacity. Both are `Read + Write + Seek`. +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +type SpoolStore = File; +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] +type SpoolStore = memory_spool::MemorySpool; + +/// Owns a temporary payload until drop; parking releases only its handle. +/// Writes are unbuffered, so parking requires no flush and preserves the cursor. +/// The file must live until assembly finishes, even when no handle is open. +pub(crate) struct Spool { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + path: PathBuf, + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + _path_charge: Option, + file: Option, + len: u64, + pos: u64, + charge: Option, +} + +impl Spool { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + pub(crate) fn create(resources: &WriterResources) -> Result { + Self::create_with_sequence(resources, crate::temp_file::next_sequence) + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + fn create_with_sequence( + resources: &WriterResources, + next_sequence: impl FnMut() -> u64, + ) -> Result { + let directory = resources.temp_dir().unwrap_or_else(|| Path::new(".")); + let (path, file, path_charge) = crate::temp_file::create_with_sequence( + directory, + |capacity| { + let mut charge = CapacityCharge::new(resources, &None); + if let Some(charge) = &mut charge { + charge.grow_to(capacity as u64)?; + } + Ok(charge) + }, + next_sequence, + )?; + Ok(Self { + path, + _path_charge: path_charge, + file: Some(file), + len: 0, + pos: 0, + charge: StorageCharge::new(resources), + }) + } + + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + pub(crate) fn create(resources: &WriterResources) -> Result { + crate::temp_file::next_sequence(); + Ok(Self { + file: Some(memory_spool::MemorySpool::new(resources)), + len: 0, + pos: 0, + charge: StorageCharge::new(resources), + }) + } + + /// Allocate a spool whose handle is not retained while waiting for work. + pub(crate) fn create_parked(resources: &WriterResources) -> Result { + let mut spool = Self::create(resources)?; + spool.park(); + Ok(spool) + } + + /// Release the OS handle, retaining ownership of the file and its cursor. + /// Bare WASM has no handle to release and must retain the in-memory bytes. + pub(crate) fn park(&mut self) { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + { + self.file = None; + } + } + + fn file(&mut self) -> std::io::Result<&mut SpoolStore> { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + if self.file.is_none() { + let mut file = File::options().read(true).write(true).open(&self.path)?; + file.seek(SeekFrom::Start(self.pos))?; + self.file = Some(file); + } + Ok(self.file.as_mut().expect("spool backing store is present")) + } + + pub(crate) fn len(&self) -> u64 { + self.len + } + + pub(crate) fn rewind(&mut self) -> Result<()> { + self.seek_to(0) + } + + /// Moves the read/write cursor to an absolute offset. + pub(crate) fn seek_to(&mut self, pos: u64) -> Result<()> { + self.seek(SeekFrom::Start(pos))?; + Ok(()) + } + + pub(crate) fn copy_to(&mut self, output: &mut dyn Write) -> Result { + let result = (|| { + self.rewind()?; + Ok(std::io::copy(self, output)?) + })(); + self.park(); + result + } + + /// Copies `len` bytes starting at `start` to `output`, which is how a + /// volume set takes one fragment of a member at a time. + pub(crate) fn copy_range_to( + &mut self, + start: u64, + len: u64, + output: &mut dyn Write, + ) -> Result { + let result = (|| { + self.seek_to(start)?; + let copied = std::io::copy(&mut self.take(len), output)?; + if copied != len { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "spooled range is shorter than expected", + ) + .into()); + } + Ok(copied) + })(); + self.park(); + result + } +} + +impl Spool { + // Keep reservation/reconciliation identical for native and memory stores. + // The operation also permits deterministic short-write regression tests. + fn write_with( + &mut self, + buffer: &[u8], + write: impl FnOnce(&mut SpoolStore, &[u8]) -> std::io::Result, + ) -> std::io::Result { + if buffer.is_empty() { + return Ok(0); + } + let end = self.pos.checked_add(buffer.len() as u64).ok_or_else(|| { + std::io::Error::other(Error::InvalidArgument("spool length overflows u64")) + })?; + if let Some(charge) = &mut self.charge { + charge + .grow_to(self.len.max(end)) + .map_err(std::io::Error::other)?; + } + // Keep the reservation while the backing store grows. Release only the + // unwritten allowance on errors or short writes; overwrites cost nothing. + let result = self.file().and_then(|file| write(file, buffer)); + match result { + Ok(written) if written != 0 => { + self.pos += written as u64; + self.len = self.len.max(self.pos); + } + _ => {} + } + if let Some(charge) = &mut self.charge { + charge.shrink_to(self.len); + } + result + } +} + +impl Write for Spool { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.write_with(buffer, |file, bytes| file.write(bytes)) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.file()?.flush() + } +} + +impl Read for Spool { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.file()?.read(buffer)?; + self.pos = self.pos.saturating_add(read as u64); + Ok(read) + } +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +impl Spool { + pub(crate) fn into_source(mut self) -> EntrySource { + self.park(); + let owner = Arc::new(self); + EntrySource::from_opener(owner.len(), move || { + Ok(Box::new(SpoolReader { + file: std::fs::File::open(&owner.path)?, + _owner: owner.clone(), + })) + }) + } +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +struct SpoolReader { + file: std::fs::File, + _owner: Arc, +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +impl Read for SpoolReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.file.read(buffer) + } +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +impl Seek for SpoolReader { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.file.seek(from) + } +} + +impl Seek for Spool { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.pos = self.file()?.seek(from)?; + Ok(self.pos) + } +} + +impl Drop for Spool { + fn drop(&mut self) { + self.file = None; + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + if let Err(error) = std::fs::remove_file(&self.path) { + if error.kind() != std::io::ErrorKind::NotFound { + // The backing file may still occupy storage. Keep that debt in + // the shared quota group instead of admitting replacement bytes. + if let Some(charge) = &mut self.charge { + charge.bytes = 0; + } + } + } + } +} + +impl Drop for MemoryPermit { + fn drop(&mut self) { + let mut used = self + .budget + .used + .lock() + .expect("memory budget lock poisoned"); + *used = used.saturating_sub(self.bytes); + self.budget.changed.notify_all(); + } +} + +#[cfg(test)] +mod tests { + #[test] + fn entry_source_length_helpers_handle_empty_data_and_length_errors() { + use super::*; + + let empty = EntrySource::from_bytes(Arc::<[u8]>::from(&b""[..])); + assert!(empty.is_empty().unwrap()); + assert!(format!("{empty:?}").contains("len: Some(0)")); + + let nonempty = EntrySource::from_bytes(Arc::<[u8]>::from(&b"x"[..])); + assert!(!nonempty.is_empty().unwrap()); + + let missing = + EntrySource::from_path(crate::scratch::case("missing-source").join("missing")); + assert!(missing.is_empty().is_err()); + assert!(format!("{missing:?}").contains("len: None")); + } + + #[test] + fn preparation_refusal_is_one_shot_and_releases_the_charge() { + use super::preparation::Bytes; + use super::*; + use std::sync::atomic::Ordering; + + let (resources, attempts) = WriterResources::default().refuse_preparation_growth_at(0); + assert_eq!( + Bytes::zeroed(1, &resources).unwrap_err(), + Error::WriterFailure("injected preparation admission failure") + ); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + assert_eq!( + *resources + .preparation_budget + .as_ref() + .unwrap() + .used + .lock() + .unwrap(), + 0 + ); + drop(Bytes::zeroed(1, &resources).unwrap()); + assert_eq!( + *resources + .preparation_budget + .as_ref() + .unwrap() + .used + .lock() + .unwrap(), + 0 + ); + } + #[test] + fn workspace_admission_reports_preexisting_cancellation_before_budget_checks() { + use super::*; + + let cancellation = WriteCancellation::new(); + cancellation.cancel(); + let resources = WriterResources::new(1).with_cancellation(cancellation); + assert!(matches!( + resources.acquire_cancellable(2, 0, &|| false), + Err(Error::Cancelled) + )); + assert!(matches!( + resources.acquire_cancellable(1, 0, &|| true), + Err(Error::Cancelled) + )); + assert!(matches!( + resources.acquire_serialising_cancellable(1, &|| { + panic!("pre-cancelled resources called the progress callback") + }), + Err(Error::Cancelled) + )); + assert_eq!(resources.workspace_in_use(), 0); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_rejects_length_overflow_before_writing_or_charging() { + use super::*; + + let root = crate::scratch::case("spool-length-overflow"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(16); + let mut spool = Spool::create(&resources).unwrap(); + spool.pos = u64::MAX; + let error = spool.write(b"ab").unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::Other); + assert_eq!(spool.len, 0); + assert_eq!(spool_used(&resources), 0); + } + + #[test] + fn writer_memory_reader_retains_payload_after_source_drop() { + use super::*; + let resources = WriterResources::default().with_max_memory_bytes(8192); + let source = EntrySource::copy_for_writer(&[42; 4096], &resources).unwrap(); + let retained = resources.managed_memory_in_use(); + assert!(retained >= 4096); + let mut reader = source.open().unwrap(); + drop(source); + assert!(resources.managed_memory_in_use() > retained); + let mut bytes = [0; 4096]; + reader.read_exact(&mut bytes).unwrap(); + assert_eq!(bytes, [42; 4096]); + drop(reader); + assert_eq!(resources.managed_memory_in_use(), 0); + } + + use super::*; + fn spool_used(resources: &WriterResources) -> u64 { + *resources + .spool_budget + .as_ref() + .unwrap() + .used + .lock() + .unwrap() + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_path_capacity_survives_parking_and_refuses_before_file_creation() { + let root = crate::scratch::case("spool-path-capacity"); + let limited = crate::codec::workspace::Allowance::limited(1); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_execution_allowance(limited.clone()); + assert_eq!( + Spool::create(&resources).err().unwrap().kind(), + crate::ErrorKind::ResourceLimit + ); + assert_eq!(limited.used(), 0); + assert_eq!(std::fs::read_dir(&*root).unwrap().count(), 0); + let ledger = crate::codec::workspace::Allowance::limited(4096); + let resources = resources.with_execution_allowance(ledger.clone()); + let mut spool = Spool::create(&resources).unwrap(); + let capacity = spool.path.capacity() as u64; + assert_eq!(ledger.used(), capacity); + spool.write_all(b"payload").unwrap(); + spool.park(); + assert_eq!(ledger.used(), capacity); + let mut copied = Vec::new(); + spool.copy_to(&mut copied).unwrap(); + assert_eq!(copied, b"payload"); + assert_eq!(ledger.used(), capacity); + drop(spool); + assert_eq!(ledger.used(), 0); + assert_eq!(std::fs::read_dir(&*root).unwrap().count(), 0); + } + + #[test] + fn spool_reservations_refuse_arithmetic_overflow() { + let resources = WriterResources::default().with_max_spool_bytes(u64::MAX); + let mut first = StorageCharge::new(&resources).unwrap(); + let mut second = StorageCharge::new(&resources).unwrap(); + first.grow_to(u64::MAX).unwrap(); + assert_eq!( + second.grow_to(1).unwrap_err(), + Error::WriterSpoolLimitExceeded { + limit: u64::MAX, + required: u64::MAX, + used: u64::MAX, + } + ); + drop(first); + second.grow_to(1).unwrap(); + assert_eq!(spool_used(&resources), 1); + } + + #[test] + fn spool_quota_counts_live_extents_across_parking_and_overwrites() { + let root = crate::scratch::case("spool-quota-extents"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(10); + let mut first = Spool::create(&resources).unwrap(); + first.write_all(b"abcdef").unwrap(); + first.park(); + let mut second = Spool::create(&resources.clone()).unwrap(); + second.write_all(b"1234").unwrap(); + assert_eq!(spool_used(&resources), 10); + let error = Error::from(second.write(b"x").unwrap_err()); + assert_eq!( + error, + Error::WriterSpoolLimitExceeded { + limit: 10, + required: 11, + used: 10 + } + ); + assert_eq!(second.len(), 4); + first.seek_to(1).unwrap(); + first.write_all(b"XY").unwrap(); + assert_eq!(spool_used(&resources), 10); + drop(second); + assert_eq!(spool_used(&resources), 6); + first.seek_to(9).unwrap(); + assert_eq!(spool_used(&resources), 6); + first.write_all(b"Z").unwrap(); // charge the hole as well + assert_eq!(spool_used(&resources), 10); + let mut bytes = Vec::new(); + first.copy_to(&mut bytes).unwrap(); + assert_eq!(bytes, b"aXYdef\0\0\0Z"); + drop(first); + assert_eq!(spool_used(&resources), 0); + } + + #[test] + fn spool_short_writes_hold_then_reconcile_the_growth_reservation() { + let root = crate::scratch::case("spool-quota-short-write"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(10); + let mut spool = Spool::create(&resources).unwrap(); + let written = spool + .write_with(b"0123456789", |file, bytes| { + assert_eq!(spool_used(&resources), 10); + file.write(&bytes[..3]) + }) + .unwrap(); + assert_eq!(written, 3); + assert_eq!(spool.len(), 3); + assert_eq!(spool_used(&resources), 3); + spool.seek_to(9).unwrap(); + assert_eq!( + spool + .write_with(b"x", |_, _| { + assert_eq!(spool_used(&resources), 10); + Ok(0) + }) + .unwrap(), + 0 + ); + assert_eq!(spool.len(), 3); + assert_eq!(spool_used(&resources), 3); + assert!(spool + .write_with(b"xx", |_, _| panic!("refusal must precede backing I/O")) + .is_err()); + assert_eq!(spool_used(&resources), 3); + } + + #[test] + fn spool_zero_limit_allows_empty_writes_but_no_growth() { + let root = crate::scratch::case("spool-quota-zero"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(0); + let mut spool = Spool::create(&resources).unwrap(); + spool.seek_to(100).unwrap(); + assert_eq!(spool.write(b"").unwrap(), 0); + assert_eq!(spool.len(), 0); + assert_eq!(spool_used(&resources), 0); + assert!(matches!( + Error::from(spool.write(b"x").unwrap_err()), + Error::WriterSpoolLimitExceeded { + limit: 0, + required: 101, + used: 0 + } + )); + assert_eq!(spool.len(), 0); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn failed_backing_write_releases_only_its_reservation() { + let root = crate::scratch::case("spool-quota-io"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(10); + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"abc").unwrap(); + spool.park(); + let mut read_only = File::open(&spool.path).unwrap(); + read_only.seek(SeekFrom::Start(3)).unwrap(); + spool.file = Some(read_only); + assert!(spool.write_all(b"1234567").is_err()); + assert_eq!(spool_used(&resources), 3); + assert_eq!(spool.len(), 3); + spool.park(); + spool.write_all(b"1234567").unwrap(); + assert_eq!(spool_used(&resources), 10); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_sources_and_readers_retain_their_charge() { + let root = crate::scratch::case("spool-quota-source"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(3); + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"abc").unwrap(); + let source = spool.into_source(); + let mut reader = source.open().unwrap(); + drop(source); + assert_eq!(spool_used(&resources), 3); + let mut another = Spool::create(&resources).unwrap(); + assert!(another.write(b"x").is_err()); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"abc"); + drop(reader); + another.write_all(b"xyz").unwrap(); + assert_eq!(spool_used(&resources), 3); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn concurrent_spool_growth_cannot_overbook_shared_resources() { + let root = crate::scratch::case("spool-quota-concurrent"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(10); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + for _ in 0..4 { + let tx = tx.clone(); + let resources = resources.clone(); + scope.spawn(move || { + let mut spool = Spool::create(&resources).unwrap(); + let result = spool.write_all(b"1234567").map_err(Error::from); + tx.send((spool, result)).unwrap(); + }); + } + }); + drop(tx); + let outcomes: Vec<_> = rx.into_iter().collect(); + assert_eq!( + outcomes.iter().filter(|(_, result)| result.is_ok()).count(), + 1 + ); + assert_eq!(spool_used(&resources), 7); + for (_, result) in &outcomes { + if let Err(error) = result { + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + } + } + drop(outcomes); + assert_eq!(spool_used(&resources), 0); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_cleanup_failure_retains_debt_in_the_quota_group() { + let root = crate::scratch::case("spool-quota-cleanup"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(3); + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"abc").unwrap(); + spool.park(); + let original = spool.path.clone(); + // Force removal to fail without relying on root/Windows permissions. + std::fs::rename(&original, root.join("orphan")).unwrap(); + std::fs::create_dir(&original).unwrap(); + drop(spool); + assert_eq!(spool_used(&resources), 3); + let mut another = Spool::create(&resources).unwrap(); + assert!(another.write_all(b"x").is_err()); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_cleanup_failure_without_quota_does_not_panic() { + let root = crate::scratch::case("spool-unlimited-cleanup"); + let resources = WriterResources::default().with_temp_dir(&*root); + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"abc").unwrap(); + let original = spool.path.clone(); + std::fs::rename(&original, root.join("orphan")).unwrap(); + std::fs::create_dir(&original).unwrap(); + drop(spool); + assert!(original.is_dir()); + assert_eq!(std::fs::read(root.join("orphan")).unwrap(), b"abc"); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_creation_accepts_the_maximum_sequence() { + let root = crate::scratch::case("spool-max-sequence"); + let resources = WriterResources::default().with_temp_dir(&*root); + let spool = Spool::create_with_sequence(&resources, || u64::MAX).unwrap(); + let path = root.join(format!( + ".rars-spool-{}-ffffffffffffffff", + std::process::id() + )); + assert_eq!(spool.path, path); + assert!(path.is_file()); + drop(spool); + assert!(!path.exists()); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_creation_retries_collisions_and_reports_exhaustion() { + let root = crate::scratch::case("spool-name-collision"); + let name = |sequence| format!(".rars-spool-{}-{sequence:016x}", std::process::id()); + std::fs::write(root.join(name(0)), b"occupied").unwrap(); + let resources = WriterResources::default().with_temp_dir(&*root); + let mut sequence = 0; + let spool = Spool::create_with_sequence(&resources, || { + let value = sequence; + sequence += 1; + value + }) + .unwrap(); + assert_eq!(sequence, 2); + assert_eq!(spool.path, root.join(name(1))); + drop(spool); + + for sequence in 1..128 { + std::fs::write(root.join(name(sequence)), b"occupied").unwrap(); + } + let mut sequence = 0; + let error = Spool::create_with_sequence(&resources, || { + let value = sequence; + sequence += 1; + value + }) + .err() + .unwrap(); + assert_eq!(sequence, 128); + assert_eq!(error.kind(), crate::ErrorKind::Io); + assert_eq!(std::fs::read(root.join(name(0))).unwrap(), b"occupied"); + + let missing = WriterResources::default().with_temp_dir(root.join("missing")); + let error = Spool::create_with_sequence(&missing, || 0).err().unwrap(); + assert_eq!(error.kind(), crate::ErrorKind::Io); + } + + #[test] + fn spool_quota_is_released_on_unwind() { + let root = crate::scratch::case("spool-quota-unwind"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(3); + let result = std::panic::catch_unwind(|| { + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"abc").unwrap(); + panic!("injected unwind"); + }); + assert!(result.is_err()); + assert_eq!(spool_used(&resources), 0); + } + + #[test] + fn byte_sources_are_rewindable_without_copying() { + let source = EntrySource::from_bytes(Arc::<[u8]>::from(&b"hello"[..])); + let mut first = source.open().unwrap(); + let mut second = source.open().unwrap(); + let mut a = Vec::new(); + let mut b = Vec::new(); + first.read_to_end(&mut a).unwrap(); + second.read_to_end(&mut b).unwrap(); + assert_eq!(a, b"hello"); + assert_eq!(b, b"hello"); + } + + #[test] + fn spool_tracks_length_across_seeks_and_overwrites() { + let scratch = crate::scratch::case("rars-spool"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"0123456789").unwrap(); + assert_eq!(spool.len(), 10); + + // Rewriting earlier bytes must not extend the spool. + spool.seek_to(4).unwrap(); + spool.write_all(b"ab").unwrap(); + assert_eq!(spool.len(), 10); + + // Writing past the end does extend it. + spool.seek_to(9).unwrap(); + spool.write_all(b"xyz").unwrap(); + assert_eq!(spool.len(), 12); + + let mut copied = Vec::new(); + spool.copy_to(&mut copied).unwrap(); + assert_eq!(copied, b"0123ab678xyz"); + } + + #[test] + fn spool_reads_from_the_seeked_position() { + let scratch = crate::scratch::case("rars-spool"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"abcdefgh").unwrap(); + spool.seek(SeekFrom::Start(3)).unwrap(); + + let mut buffer = [0u8; 4]; + spool.read_exact(&mut buffer).unwrap(); + assert_eq!(&buffer, b"defg"); + assert_eq!(spool.stream_position().unwrap(), 7); + } + + #[test] + fn parked_spools_preserve_cursor_and_release_handles_after_copy_errors() { + let scratch = crate::scratch::case("parked-spool"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let mut spool = Spool::create_parked(&resources).unwrap(); + spool.write_all(b"abcdef").unwrap(); + spool.seek_to(2).unwrap(); + spool.park(); + spool.write_all(b"XY").unwrap(); + spool.park(); + spool.write_all(b"Z").unwrap(); + assert_eq!(spool.len(), 6); + let mut bytes = Vec::new(); + spool.copy_range_to(1, 4, &mut bytes).unwrap(); + assert_eq!(bytes, b"bXYZ"); + struct FailingSink; + impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + assert!(spool.copy_to(&mut FailingSink).is_err()); + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + assert!(spool.file.is_none()); + assert!(spool.copy_range_to(0, 7, &mut Vec::new()).is_err()); + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + assert!(spool.file.is_none()); + bytes.clear(); + spool.copy_to(&mut bytes).unwrap(); + assert_eq!(bytes, b"abXYZf"); + drop(spool); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn spool_cleanup_owns_both_active_and_parked_files() { + let scratch = crate::scratch::case("spool-cleanup"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + for parked in [false, true] { + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"private compressed payload").unwrap(); + let path = spool.path.clone(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o077, + 0 + ); + } + if parked { + spool.park(); + } + assert!(path.exists()); + drop(spool); + assert!(!path.exists()); + } + let unwind = std::panic::catch_unwind(|| { + let mut spool = Spool::create(&resources).unwrap(); + spool.write_all(b"unfinished").unwrap(); + panic!("injected failure"); + }); + assert!(unwind.is_err()); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + #[test] + fn admission_waits_can_be_cancelled_without_releasing_the_held_permit() { + use std::sync::mpsc; + use std::time::Duration; + for via_callback in [false, true] { + let cancel = WriteCancellation::new(); + let resources = WriterResources::new(100).with_cancellation(if via_callback { + WriteCancellation::new() + } else { + cancel.clone() + }); + let held = resources.acquire(100, 0).unwrap(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let callback_cancel = cancel.clone(); + let worker = std::thread::spawn(move || { + let polls = AtomicU64::new(0); + let result = resources.acquire_cancellable(1, 0, &|| { + // Admission checks once before entering the budget and + // again before locking. The third poll proves a timed + // wait occurred while the permit was still held. + if polls.fetch_add(1, Ordering::Relaxed) >= 2 { + let _ = entered_tx.send(()); + } + via_callback && callback_cancel.is_cancelled() + }); + done_tx.send(result.map(|_| ())).unwrap(); + resources + }); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + cancel.cancel(); + assert_eq!( + done_rx.recv_timeout(Duration::from_secs(2)).unwrap(), + Err(Error::Cancelled) + ); + let resources = worker.join().unwrap(); + assert_eq!(*resources.budget.used.lock().unwrap(), 100); + drop(held); + assert_eq!(*resources.budget.used.lock().unwrap(), 0); + } + } + + #[test] + fn oversized_workspace_is_rejected_before_waiting() { + let resources = WriterResources::new(1024); + assert!(matches!( + resources.acquire(1025, 512), + Err(Error::MemoryLimitExceeded { .. }) + )); + } +} diff --git a/arca-rar/vendor/rars/src/streaming/memory_spool.rs b/arca-rar/vendor/rars/src/streaming/memory_spool.rs new file mode 100644 index 00000000..795e4f53 --- /dev/null +++ b/arca-rar/vendor/rars/src/streaming/memory_spool.rs @@ -0,0 +1,513 @@ +//! Optional bounded storage for bare-WASM spools. +//! Payload blocks and index replacements have known sizes before allocation. +use super::{CapacityCharge, WriterResources}; +use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; + +const BLOCK_BYTES: usize = 4096; +type Block = Box<[u8; BLOCK_BYTES]>; +const INDEX_ENTRY_BYTES: usize = std::mem::size_of::>(); + +pub(super) struct MemorySpool { + store: Store, +} + +enum Store { + Unbounded(Cursor>), + Bounded(BoundedSpool), +} + +struct BoundedSpool { + // Only index slots move during growth; payload allocations stay in place. + blocks: Box<[Option]>, + block_count: usize, + len: usize, + pos: u64, + // Drop payload and index allocations before releasing their shared charge. + charge: CapacityCharge, +} + +impl MemorySpool { + pub(super) fn new(resources: &WriterResources) -> Self { + let store = match resources.spool_capacity_charge() { + Some(charge) => Store::Bounded(BoundedSpool { + blocks: Box::default(), + block_count: 0, + len: 0, + pos: 0, + charge, + }), + None => Store::Unbounded(Cursor::new(Vec::new())), + }; + Self { store } + } +} + +impl BoundedSpool { + fn grow(&mut self, end: usize) -> io::Result<()> { + let count = end.div_ceil(BLOCK_BYTES); + if count <= self.block_count { + return Ok(()); + } + let overflow = || { + io::Error::new( + io::ErrorKind::InvalidInput, + "memory spool capacity overflow", + ) + }; + let slots = if count > self.blocks.len() { + count.checked_next_power_of_two().ok_or_else(overflow)? + } else { + self.blocks.len() + }; + let payload_bytes = count.checked_mul(BLOCK_BYTES).ok_or_else(overflow)?; + let index_bytes = slots.checked_mul(INDEX_ENTRY_BYTES).ok_or_else(overflow)?; + let retained = payload_bytes + .checked_add(index_bytes) + .ok_or_else(overflow)?; + let old_index_bytes = if slots != self.blocks.len() { + self.blocks.len() * INDEX_ENTRY_BYTES + } else { + 0 + }; + let peak = retained.checked_add(old_index_bytes).ok_or_else(overflow)?; + self.charge.grow_to(peak as u64).map_err(io::Error::other)?; + if slots != self.blocks.len() { + // vec![value; n] requests exactly n elements. The boxed slice keeps + // that layout; no unspecified Vec growth capacity enters the quota. + let mut replacement = vec![None; slots].into_boxed_slice(); + for (target, source) in replacement.iter_mut().zip(self.blocks.iter_mut()) { + *target = source.take(); + } + self.blocks = replacement; + // Assignment freed the old index before its allowance is released. + self.charge.shrink_to(retained as u64); + } + while self.block_count < count { + self.blocks[self.block_count] = Some(Box::new([0; BLOCK_BYTES])); + self.block_count += 1; + } + Ok(()) + } + + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let pos = usize::try_from(self.pos).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "memory spool position overflow", + ) + })?; + let end = pos.checked_add(bytes.len()).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "memory spool length overflow") + })?; + self.grow(end)?; + let mut copied = 0; + while copied < bytes.len() { + let offset = pos + copied; + let within = offset % BLOCK_BYTES; + let count = (BLOCK_BYTES - within).min(bytes.len() - copied); + self.blocks[offset / BLOCK_BYTES].as_mut().unwrap()[within..within + count] + .copy_from_slice(&bytes[copied..copied + count]); + copied += count; + } + self.pos = end as u64; + self.len = self.len.max(end); + Ok(bytes.len()) + } + + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.pos >= self.len as u64 { + return Ok(0); + } + let pos = self.pos as usize; + let size = bytes.len().min(self.len - pos); + let mut copied = 0; + while copied < size { + let offset = pos + copied; + let within = offset % BLOCK_BYTES; + let count = (BLOCK_BYTES - within).min(size - copied); + bytes[copied..copied + count].copy_from_slice( + &self.blocks[offset / BLOCK_BYTES].as_ref().unwrap()[within..within + count], + ); + copied += count; + } + self.pos += size as u64; + Ok(size) + } + + fn seek(&mut self, from: SeekFrom) -> io::Result { + let pos = match from { + SeekFrom::Start(pos) => Some(pos), + SeekFrom::End(offset) => (self.len as u64).checked_add_signed(offset), + SeekFrom::Current(offset) => self.pos.checked_add_signed(offset), + } + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid memory spool seek"))?; + self.pos = pos; + Ok(pos) + } +} + +impl Write for MemorySpool { + fn write(&mut self, bytes: &[u8]) -> io::Result { + match &mut self.store { + Store::Unbounded(cursor) => cursor.write(bytes), + Store::Bounded(spool) => spool.write(bytes), + } + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +impl Read for MemorySpool { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + match &mut self.store { + Store::Unbounded(cursor) => cursor.read(bytes), + Store::Bounded(spool) => spool.read(bytes), + } + } +} +impl Seek for MemorySpool { + fn seek(&mut self, from: SeekFrom) -> io::Result { + match &mut self.store { + Store::Unbounded(cursor) => cursor.seek(from), + Store::Bounded(spool) => spool.seek(from), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Error, ErrorKind}; + + #[test] + fn impossible_spool_growth_is_rejected_before_allocation() { + let resources = WriterResources::default().with_max_spool_memory_bytes(u64::MAX); + let mut spool = MemorySpool::new(&resources); + let Store::Bounded(bounded) = &mut spool.store else { + panic!("memory limit must select bounded storage"); + }; + let error = bounded.grow(usize::MAX).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(used(&resources), 0); + spool.flush().unwrap(); + } + + #[test] + fn impossible_write_extent_keeps_position_and_capacity() { + let resources = WriterResources::default().with_max_spool_memory_bytes(u64::MAX); + let mut spool = MemorySpool::new(&resources); + spool.seek(SeekFrom::Start(usize::MAX as u64)).unwrap(); + let error = spool.write(b"x").unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(error.to_string(), "memory spool length overflow"); + assert_eq!(spool.stream_position().unwrap(), usize::MAX as u64); + assert_eq!(used(&resources), 0); + spool.seek(SeekFrom::Start(0)).unwrap(); + assert_eq!(spool.read(&mut [0]).unwrap(), 0); + spool.write_all(b"valid").unwrap(); + spool.seek(SeekFrom::Start(0)).unwrap(); + let mut payload = [0; 5]; + spool.read_exact(&mut payload).unwrap(); + assert_eq!(&payload, b"valid"); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn position_wider_than_address_space_is_rejected_before_allocation() { + let resources = WriterResources::default().with_max_spool_memory_bytes(u64::MAX); + let mut spool = MemorySpool::new(&resources); + let position = u64::from(u32::MAX) + 1; + spool.seek(SeekFrom::Start(position)).unwrap(); + let error = spool.write(b"x").unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(error.to_string(), "memory spool position overflow"); + assert_eq!(spool.stream_position().unwrap(), position); + assert_eq!(used(&resources), 0); + // Empty writes need no addressable extent. + assert_eq!(spool.write(&[]).unwrap(), 0); + assert_eq!(spool.stream_position().unwrap(), position); + } + + #[test] + fn admitted_spool_retains_its_capacity_after_worker_retirement() { + use crate::codec::workspace::{Allowance, Buffer, RESERVATION_BYTES}; + use crate::streaming::preparation::Bytes; + let ledger = Allowance::limited(20000); + let resources = WriterResources::default().with_execution_allowance(ledger.clone()); + let prepared = Bytes::zeroed(32, &resources).unwrap(); + let mut reservation = ledger.reserve(capacity(2)).unwrap(); + let allowance = reservation.allowance(); + let worker = resources + .clone() + .with_execution_allowance(allowance.clone()); + reservation.start(); + let scratch = Buffer::filled(64, 0u8, &allowance).unwrap(); + let mut spool = MemorySpool::new(&worker); + spool.write_all(b"payload").unwrap(); + assert_eq!(ledger.used(), 32 + capacity(2) + RESERVATION_BYTES); + reservation.retire(); + assert_eq!(ledger.used(), 32 + 64 + capacity(1) + RESERVATION_BYTES); + drop(scratch); + drop(prepared); + assert_eq!(ledger.used(), capacity(1) + RESERVATION_BYTES); + spool.seek(SeekFrom::Start(BLOCK_BYTES as u64)).unwrap(); + let error = spool.write_all(b"growth after retirement").unwrap_err(); + assert_eq!( + error.downcast::().unwrap().kind(), + ErrorKind::ResourceLimit + ); + assert_eq!(ledger.used(), capacity(1) + RESERVATION_BYTES); + spool.seek(SeekFrom::Start(0)).unwrap(); + let mut bytes = [0; 7]; + spool.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"payload"); + drop(spool); + assert_eq!(ledger.used(), RESERVATION_BYTES); + drop((worker, allowance)); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn copying_retired_worker_output_counts_source_and_spool_together() { + use crate::codec::workspace::{Allowance, Buffer, RESERVATION_BYTES}; + let payload = 32; + let ledger = Allowance::limited(capacity(1) + payload + RESERVATION_BYTES); + let resources = WriterResources::default().with_execution_allowance(ledger.clone()); + let mut reservation = ledger.reserve(payload).unwrap(); + reservation.start(); + let bytes = Buffer::filled(payload as usize, 7u8, &reservation.allowance()).unwrap(); + reservation.retire(); + let mut spool = MemorySpool::new(&resources); + spool.write_all(&bytes).unwrap(); + assert_eq!(ledger.used(), capacity(1) + payload + RESERVATION_BYTES); + // A second destination must not borrow the still-live source charge. + assert!(MemorySpool::new(&resources).write_all(&bytes).is_err()); + drop(bytes); + assert_eq!(ledger.used(), capacity(1)); + spool.seek(SeekFrom::Start(0)).unwrap(); + let mut actual = [0; 32]; + spool.read_exact(&mut actual).unwrap(); + assert_eq!(actual, [7; 32]); + drop(spool); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn class_quota_refusal_rolls_back_execution_spool_growth() { + use crate::codec::workspace::Allowance; + let ledger = Allowance::limited(20000); + let resources = WriterResources::default() + .with_max_spool_memory_bytes(capacity(1)) + .with_execution_allowance(ledger.clone()); + let mut spool = MemorySpool::new(&resources); + spool.write_all(b"payload").unwrap(); + spool.seek(SeekFrom::Start(BLOCK_BYTES as u64)).unwrap(); + assert!(spool.write_all(b"overflow").is_err()); + assert_eq!(ledger.used(), capacity(1)); + drop(spool); + assert_eq!(ledger.used(), 0); + MemorySpool::new(&resources).write_all(b"retry").unwrap(); + assert_eq!(ledger.used(), 0); + } + + fn capacity(blocks: usize) -> u64 { + (blocks * BLOCK_BYTES + blocks.next_power_of_two() * INDEX_ENTRY_BYTES) as u64 + } + + fn used(resources: &WriterResources) -> u64 { + *resources + .spool_memory_budget + .as_ref() + .unwrap() + .used + .lock() + .unwrap() + } + + #[test] + fn quota_admits_payload_capacity_before_allocating_any_blocks() { + for limit in [0, BLOCK_BYTES as u64, capacity(1) - 1] { + let resources = WriterResources::default().with_max_spool_memory_bytes(limit); + let mut spool = MemorySpool::new(&resources); + let error = Error::from(spool.write(b"x").unwrap_err()); + assert_eq!( + error, + Error::WriterSpoolMemoryLimitExceeded { + limit, + required: capacity(1), + used: 0, + } + ); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!(used(&resources), 0); + let Store::Bounded(bounded) = &spool.store else { + panic!("bounded storage required") + }; + assert!(bounded.blocks.is_empty()); + assert_eq!(bounded.len, 0); + assert_eq!(bounded.pos, 0); + } + } + + #[test] + fn spare_payload_capacity_is_shared_and_stays_charged_until_drop() { + let resources = WriterResources::default().with_max_spool_memory_bytes(capacity(1)); + let mut first = MemorySpool::new(&resources); + first.write_all(b"x").unwrap(); + assert_eq!(used(&resources), capacity(1)); + let mut second = MemorySpool::new(&resources.clone()); + assert!(second.write(b"y").is_err()); + first.rewind().unwrap(); + first.write_all(b"replacement").unwrap(); + assert_eq!(used(&resources), capacity(1)); + first.seek(SeekFrom::Start(BLOCK_BYTES as u64 - 1)).unwrap(); + assert!(first.write_all(b"ab").is_err()); + assert_eq!(used(&resources), capacity(1)); + let mut bytes = Vec::new(); + first.rewind().unwrap(); + first.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"replacement"); + drop(first); + second.write_all(b"y").unwrap(); + assert_eq!(used(&resources), capacity(1)); + drop(second); + assert_eq!(used(&resources), 0); + } + + #[test] + fn index_replacement_requires_both_indexes_before_growth() { + let peak = capacity(2) + INDEX_ENTRY_BYTES as u64; + for limit in [peak - 1, peak] { + let resources = WriterResources::default().with_max_spool_memory_bytes(limit); + let mut spool = MemorySpool::new(&resources); + spool.write_all(b"original").unwrap(); + spool.seek(SeekFrom::Start(BLOCK_BYTES as u64)).unwrap(); + let result = spool.write(b"next"); + if limit < peak { + assert_eq!( + Error::from(result.unwrap_err()), + Error::WriterSpoolMemoryLimitExceeded { + limit, + required: peak, + used: capacity(1), + } + ); + assert_eq!(used(&resources), capacity(1)); + assert_eq!(spool.stream_position().unwrap(), BLOCK_BYTES as u64); + assert_eq!(spool.seek(SeekFrom::End(0)).unwrap(), 8); + } else { + assert_eq!(result.unwrap(), 4); + // The replaced index is no longer charged after growth. + assert_eq!(used(&resources), capacity(2)); + } + spool.rewind().unwrap(); + let mut original = [0; 8]; + spool.read_exact(&mut original).unwrap(); + assert_eq!(&original, b"original"); + drop(spool); + assert_eq!(used(&resources), 0); + } + } + + #[test] + fn unused_index_slots_remain_charged_and_reused() { + let resources = WriterResources::default().with_max_spool_memory_bytes(capacity(4)); + let mut spool = MemorySpool::new(&resources); + // Three payload blocks allocate four index slots. + spool.write_all(&vec![7; BLOCK_BYTES * 3]).unwrap(); + assert_eq!(used(&resources), capacity(3)); + spool.write_all(b"fourth").unwrap(); + assert_eq!(used(&resources), capacity(4)); + // A second spool cannot use the retained capacity. + assert!(MemorySpool::new(&resources).write(b"x").is_err()); + assert_eq!(used(&resources), capacity(4)); + drop(spool); + assert_eq!(used(&resources), 0); + } + + #[test] + fn block_boundaries_overwrites_and_holes_match_cursor_contents() { + let resources = + WriterResources::default().with_max_spool_memory_bytes((BLOCK_BYTES * 8) as u64); + let mut actual = MemorySpool::new(&resources); + let mut expected = Cursor::new(Vec::new()); + for (position, data) in [ + (0, b"start".as_slice()), + (BLOCK_BYTES as u64 - 3, b"cross a block boundary".as_slice()), + ( + (BLOCK_BYTES * 3) as u64 + 1, + b"after a zero-filled hole".as_slice(), + ), + (2, b"overwrite".as_slice()), + ] { + actual.seek(SeekFrom::Start(position)).unwrap(); + expected.seek(SeekFrom::Start(position)).unwrap(); + actual.write_all(data).unwrap(); + expected.write_all(data).unwrap(); + } + assert_eq!(used(&resources), capacity(4)); + assert_eq!( + actual.seek(SeekFrom::End(-5)).unwrap(), + expected.seek(SeekFrom::End(-5)).unwrap() + ); + actual.write_all(b"tail!").unwrap(); + expected.write_all(b"tail!").unwrap(); + actual.rewind().unwrap(); + let mut bytes = Vec::new(); + let mut buffer = [0; 73]; + loop { + let count = actual.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + bytes.extend_from_slice(&buffer[..count]); + } + assert_eq!(bytes, expected.into_inner()); + } + + #[test] + fn seek_and_empty_io_do_not_allocate_or_change_length() { + let resources = WriterResources::default().with_max_spool_memory_bytes(0); + let mut spool = MemorySpool::new(&resources); + spool.seek(SeekFrom::Start(u64::MAX)).unwrap(); + assert_eq!(spool.write(b"").unwrap(), 0); + assert_eq!(spool.read(&mut [0; 1]).unwrap(), 0); + assert!(spool.write(b"x").is_err()); + assert_eq!(used(&resources), 0); + assert!(spool.seek(SeekFrom::Current(1)).is_err()); + assert_eq!(spool.stream_position().unwrap(), u64::MAX); + assert!(spool.seek(SeekFrom::End(-1)).is_err()); + assert_eq!(spool.seek(SeekFrom::End(0)).unwrap(), 0); + } + + #[test] + fn unwinding_releases_payload_blocks_and_their_charge() { + let resources = WriterResources::default().with_max_spool_memory_bytes(capacity(1)); + let failure = std::panic::catch_unwind(|| { + let mut spool = MemorySpool::new(&resources); + spool.write_all(b"payload").unwrap(); + panic!("injected failure"); + }); + assert!(failure.is_err()); + assert_eq!(used(&resources), 0); + MemorySpool::new(&resources) + .write_all(b"replacement") + .unwrap(); + assert_eq!(used(&resources), 0); + } + + #[test] + fn default_storage_remains_the_unbounded_cursor() { + let resources = WriterResources::default(); + let mut spool = MemorySpool::new(&resources); + assert!(matches!(spool.store, Store::Unbounded(_))); + spool.write_all(b"payload").unwrap(); + spool.rewind().unwrap(); + let mut bytes = Vec::new(); + spool.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"payload"); + } +} diff --git a/arca-rar/vendor/rars/src/streaming/output.rs b/arca-rar/vendor/rars/src/streaming/output.rs new file mode 100644 index 00000000..4468acfc --- /dev/null +++ b/arca-rar/vendor/rars/src/streaming/output.rs @@ -0,0 +1,253 @@ +//! Final collectors retain capacity until transfer, including adapter copies. +use super::{ + preparation::{Bytes, Records}, + CapacityCharge, WriterResources, +}; +use crate::{Error, Result}; +use std::{ + io::{self, Write}, + sync::{Arc, Mutex}, +}; + +/// Encoded bytes whose managed capacity remains charged until dropped or handed +/// to the caller with `into_vec`. Bindings use `copy_with` for a counted copy. +#[derive(Debug)] +pub struct WriterOutput { + bytes: Bytes, + resources: WriterResources, +} +impl WriterOutput { + pub(crate) fn new(resources: &WriterResources) -> Self { + Self { + bytes: Bytes::output(resources), + resources: resources.clone(), + } + } + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + pub fn into_vec(mut self) -> Vec { + self.bytes.take_vec() + } + /// Reserve the destination payload while this source remains live. The + /// callback must make one copy of the bytes; its result belongs to the caller. + pub fn copy_with(&self, copy: impl FnOnce(&[u8]) -> T) -> Result { + let mut charge = self.resources.execution_charge(); + if let Some(charge) = &mut charge { + charge.grow_to(self.bytes.len() as u64)?; + } + Ok(copy(&self.bytes)) + } +} +impl Write for WriterOutput { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes + .extend_from_slice(bytes) + .map_err(io::Error::other)?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +/// A collected volume set retaining its aggregate charge through adapter copies. +pub struct WriterVolumes { + outputs: Records, + resources: WriterResources, +} +impl WriterVolumes { + pub fn volumes(&self) -> &[WriterOutput] { + &self.outputs + } + pub fn into_vec(mut self) -> Result>> { + let mut charge = self.resources.execution_charge(); + let bytes = self + .outputs + .len() + .checked_mul(std::mem::size_of::>()) + .ok_or(Error::InvalidArgument("volume output capacity overflows"))?; + if let Some(charge) = &mut charge { + charge.grow_to(bytes as u64)?; + } + let mut out = Vec::with_capacity(self.outputs.len()); + // Keep all source charges in the old array until the handoff, although + // the buffers have moved to the destination array without copying. + for output in self.outputs.iter_mut() { + out.push(output.bytes.take_vec()); + } + Ok(out) + } + /// Count all destination payloads while the full source set stays retained. + pub fn copy_with(&self, copy: impl FnOnce(&[WriterOutput]) -> T) -> Result { + let bytes = self + .outputs + .iter() + .try_fold(0u64, |total, out| total.checked_add(out.bytes.len() as u64)) + .ok_or(Error::InvalidArgument("volume copy capacity overflows"))?; + let mut charge = self.resources.execution_charge(); + if let Some(charge) = &mut charge { + charge.grow_to(bytes)?; + } + Ok(copy(&self.outputs)) + } +} + +struct Shared { + outputs: Mutex>, + _charge: Option, +} +pub(crate) struct VolumeCollector { + shared: Arc, + resources: WriterResources, +} +struct VolumeOutput { + shared: Arc, + index: usize, + _charge: Option, +} +impl VolumeCollector { + pub(crate) fn new(resources: &WriterResources) -> Result { + let mut charge = resources.execution_charge(); + if let Some(charge) = &mut charge { + charge.grow_to(std::mem::size_of::() as u64)?; + } + Ok(Self { + shared: Arc::new(Shared { + outputs: Mutex::new(Records::with_charge(0, resources.execution_charge())?), + _charge: charge, + }), + resources: resources.clone(), + }) + } + pub(crate) fn finish(self) -> Result { + let shared = Arc::try_unwrap(self.shared) + .map_err(|_| Error::WriterFailure("volume output still open"))?; + Ok(WriterVolumes { + outputs: shared.outputs.into_inner().unwrap(), + resources: self.resources, + }) + } +} +impl crate::rar50::VolumeSink for VolumeCollector { + fn start_volume(&mut self, index: u64) -> Result> { + let index = + usize::try_from(index).map_err(|_| Error::InvalidArgument("volume index overflows"))?; + let mut outputs = self.shared.outputs.lock().unwrap(); + if index != outputs.len() { + return Err(Error::WriterFailure("nonsequential volume output")); + } + let mut charge = self.resources.execution_charge(); + if let Some(charge) = &mut charge { + charge.grow_to(std::mem::size_of::() as u64)?; + } + outputs.push_growing(WriterOutput::new(&self.resources))?; + Ok(Box::new(VolumeOutput { + shared: self.shared.clone(), + index, + _charge: charge, + })) + } +} +impl Write for VolumeOutput { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.shared.outputs.lock().unwrap()[self.index].write(bytes) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rar50::VolumeSink; + + #[test] + fn refused_adapter_copy_keeps_source_charged_and_never_calls_adapter() { + let resources = WriterResources::default().with_max_memory_bytes(4096); + let mut output = WriterOutput::new(&resources); + output.write_all(&[42; 4096]).unwrap(); + assert_eq!(resources.managed_memory_in_use(), 4096); + let error = output + .copy_with(|_| panic!("copy must be refused before allocation")) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::ResourceLimit); + assert_eq!(output.as_bytes(), &[42; 4096]); + assert_eq!(resources.managed_memory_in_use(), 4096); + drop(output); + assert_eq!(resources.managed_memory_in_use(), 0); + } + + #[test] + fn adapter_copy_charges_source_and_destination_until_handoff() { + let resources = WriterResources::default().with_max_memory_bytes(8192); + let mut output = WriterOutput::new(&resources); + output.write_all(&[42; 4096]).unwrap(); + let copied = output + .copy_with(|bytes| { + assert_eq!(resources.managed_memory_in_use(), 8192); + bytes.to_vec() + }) + .unwrap(); + assert_eq!(resources.managed_memory_in_use(), 4096); + assert_eq!(output.into_vec(), copied); + assert_eq!(resources.managed_memory_in_use(), 0); + } + + #[test] + fn unbounded_output_copy_and_flush_preserve_bytes() { + let mut output = WriterOutput::new(&WriterResources::default()); + output.write_all(b"payload").unwrap(); + output.flush().unwrap(); + assert_eq!( + output.copy_with(|bytes| bytes.to_vec()).unwrap(), + b"payload" + ); + assert_eq!(output.into_vec(), b"payload"); + } + + #[test] + fn volume_collector_requires_sequential_closed_outputs() { + let resources = WriterResources::default(); + let mut collector = VolumeCollector::new(&resources).unwrap(); + assert!(matches!( + collector.start_volume(1), + Err(Error::WriterFailure("nonsequential volume output")) + )); + let mut output = collector.start_volume(0).unwrap(); + output.write_all(b"first").unwrap(); + output.flush().unwrap(); + assert!(matches!( + collector.start_volume(0), + Err(Error::WriterFailure("nonsequential volume output")) + )); + assert!(matches!( + collector.finish(), + Err(Error::WriterFailure("volume output still open")) + )); + drop(output); + + let mut collector = VolumeCollector::new(&resources).unwrap(); + let mut first = collector.start_volume(0).unwrap(); + first.write_all(b"first").unwrap(); + drop(first); + let mut second = collector.start_volume(1).unwrap(); + second.write_all(b"second").unwrap(); + drop(second); + let volumes = collector.finish().unwrap(); + assert_eq!( + volumes + .copy_with(|parts| parts + .iter() + .map(|part| part.as_bytes().to_vec()) + .collect::>()) + .unwrap(), + [b"first".to_vec(), b"second".to_vec()] + ); + assert_eq!( + volumes.into_vec().unwrap(), + [b"first".to_vec(), b"second".to_vec()] + ); + } +} diff --git a/arca-rar/vendor/rars/src/streaming/preparation.rs b/arca-rar/vendor/rars/src/streaming/preparation.rs new file mode 100644 index 00000000..6c74aad8 --- /dev/null +++ b/arca-rar/vendor/rars/src/streaming/preparation.rs @@ -0,0 +1,503 @@ +//! Ownership of preparation capacity, including temporary replacement peaks. +use super::{CapacityCharge, WriterResources}; +use crate::{Error, Result}; + +#[derive(Debug)] +pub(crate) struct Bytes { + bytes: Vec, + len: usize, + charge: Option, +} +impl Bytes { + pub(crate) fn new(resources: &WriterResources) -> Self { + Self { + bytes: Vec::new(), + len: 0, + charge: resources.preparation_charge(), + } + } + pub(crate) fn output(resources: &WriterResources) -> Self { + Self { + bytes: Vec::new(), + len: 0, + charge: resources.execution_charge(), + } + } + pub(crate) fn take_vec(&mut self) -> Vec { + self.bytes.truncate(self.len); + self.len = 0; + std::mem::take(&mut self.bytes) + } + pub(crate) fn zeroed(len: usize, resources: &WriterResources) -> Result { + let mut out = Self::new(resources); + out.grow(len)?; + out.len = len; + Ok(out) + } + fn grow(&mut self, capacity: usize) -> Result<()> { + if capacity <= self.bytes.len() { + return Ok(()); + } + if capacity > isize::MAX as usize { + return Err(Error::InvalidArgument("preparation capacity overflows")); + } + let peak = self + .bytes + .len() + .checked_add(capacity) + .ok_or(Error::InvalidArgument("preparation capacity overflows"))?; + if let Some(charge) = &mut self.charge { + charge.grow_to(peak as u64)?; + } + let mut replacement = vec![0; capacity]; + replacement[..self.len].copy_from_slice(&self.bytes[..self.len]); + self.bytes = replacement; + if let Some(charge) = &mut self.charge { + charge.shrink_to(capacity as u64); + } + Ok(()) + } + pub(crate) fn extend_from_slice(&mut self, bytes: &[u8]) -> Result<()> { + if self.charge.is_none() { + self.bytes.truncate(self.len); + self.bytes.extend_from_slice(bytes); + self.len = self.bytes.len(); + return Ok(()); + } + let end = self + .len + .checked_add(bytes.len()) + .ok_or(Error::InvalidArgument("preparation length overflows"))?; + if end > self.bytes.len() { + let capacity = end + .checked_next_power_of_two() + .ok_or(Error::InvalidArgument("preparation capacity overflows"))?; + self.grow(capacity)?; + } + self.bytes[self.len..end].copy_from_slice(bytes); + self.len = end; + Ok(()) + } + pub(crate) fn vint(&mut self, mut value: u64) -> Result<()> { + let mut bytes = [0; 10]; + let mut len = 0; + loop { + bytes[len] = (value as u8 & 0x7f) | if value >= 128 { 128 } else { 0 }; + len += 1; + value >>= 7; + if value == 0 { + break; + } + } + self.extend_from_slice(&bytes[..len]) + } + #[cfg(test)] + pub(crate) fn as_slice(&self) -> &[u8] { + self + } + #[cfg(test)] + pub(crate) fn capacity(&self) -> usize { + self.bytes.capacity() + } +} +impl std::ops::Deref for Bytes { + type Target = [u8]; + fn deref(&self) -> &[u8] { + &self.bytes[..self.len] + } +} +impl std::ops::DerefMut for Bytes { + fn deref_mut(&mut self) -> &mut [u8] { + &mut self.bytes[..self.len] + } +} + +/// Fixed-capacity descriptor array, admitted before allocating. Consuming its +/// elements keeps the backing allocation charged until the iterator is dropped. +#[derive(Debug)] +pub(crate) struct Records { + values: Vec, + limit: usize, + charge: Option, +} +#[cfg(test)] +impl Clone for Records { + fn clone(&self) -> Self { + assert!( + self.charge.is_none(), + "charged owners require fallible copying" + ); + self.values.clone().into() + } +} +#[cfg(test)] +impl From> for Records { + fn from(values: Vec) -> Self { + Self { + limit: values.capacity(), + values, + charge: None, + } + } +} +impl Records { + pub(crate) fn new(limit: usize, resources: &WriterResources) -> Result { + Self::with_charge(limit, resources.preparation_charge()) + } + pub(crate) fn with_charge(limit: usize, mut charge: Option) -> Result { + let bytes = limit + .checked_mul(std::mem::size_of::()) + .filter(|bytes| *bytes <= isize::MAX as usize) + .ok_or(Error::InvalidArgument( + "preparation records capacity overflows", + ))?; + if let Some(charge) = &mut charge { + charge.grow_to(bytes as u64)?; + } + // with_capacity requests exactly this layout. No subsequent growth is allowed. + Ok(Self { + values: Vec::with_capacity(limit), + limit, + charge, + }) + } + pub(crate) fn collect(values: I, resources: &WriterResources) -> Result + where + I: ExactSizeIterator>, + { + let mut out = Self::new(values.len(), resources)?; + for value in values { + out.push(value?)?; + } + Ok(out) + } + + /// Grow only on explicit request, reserving the old and replacement arrays + /// together before moving elements. Fixed-admission callers use `push`. + pub(crate) fn push_growing(&mut self, value: T) -> Result<()> { + if self.values.len() == self.limit { + let limit = self + .limit + .max(1) + .checked_mul(2) + .ok_or(Error::InvalidArgument( + "preparation records capacity overflows", + ))?; + let bytes = limit + .checked_mul(std::mem::size_of::()) + .filter(|bytes| *bytes <= isize::MAX as usize) + .ok_or(Error::InvalidArgument( + "preparation records capacity overflows", + ))?; + if let Some(charge) = &mut self.charge { + charge.grow_to((bytes as u64).checked_add(charge.bytes()).ok_or( + Error::InvalidArgument("preparation records capacity overflows"), + )?)?; + } + let mut replacement = Vec::with_capacity(limit); + replacement.append(&mut self.values); + self.values = replacement; + self.limit = limit; + if let Some(charge) = &mut self.charge { + charge.shrink_to(bytes as u64); + } + } + self.push(value) + } + + pub(crate) fn pop(&mut self) -> Option { + self.values.pop() + } + + pub(crate) fn push(&mut self, value: T) -> Result<()> { + if self.values.len() == self.limit { + return Err(Error::WriterFailure( + "preparation record count exceeded admission", + )); + } + self.values.push(value); + Ok(()) + } +} +impl std::ops::Deref for Records { + type Target = [T]; + fn deref(&self) -> &[T] { + &self.values + } +} +impl std::ops::DerefMut for Records { + fn deref_mut(&mut self) -> &mut [T] { + &mut self.values + } +} +pub(crate) struct RecordIter { + iter: std::vec::IntoIter, + _charge: Option, +} +impl Iterator for RecordIter { + type Item = T; + fn next(&mut self) -> Option { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} +impl ExactSizeIterator for RecordIter {} + +impl IntoIterator for Records { + type Item = T; + type IntoIter = RecordIter; + fn into_iter(self) -> Self::IntoIter { + RecordIter { + iter: self.values.into_iter(), + _charge: self.charge, + } + } +} +impl<'a, T> IntoIterator for &'a Records { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + fn into_iter(self) -> Self::IntoIter { + self.values.iter() + } +} +impl<'a, T> IntoIterator for &'a mut Records { + type Item = &'a mut T; + type IntoIter = std::slice::IterMut<'a, T>; + fn into_iter(self) -> Self::IntoIter { + self.values.iter_mut() + } +} + +pub(crate) struct Owned { + value: Box, + _charge: Option, +} +impl Owned { + pub(crate) fn new(value: T, resources: &WriterResources) -> Result { + let mut charge = resources.preparation_charge(); + if let Some(charge) = &mut charge { + charge.grow_to(std::mem::size_of::() as u64)?; + } + Ok(Self { + value: Box::new(value), + _charge: charge, + }) + } + pub(crate) fn into_inner(self) -> T { + *self.value + } +} +impl std::ops::Deref for Owned { + type Target = T; + fn deref(&self) -> &T { + &self.value + } +} +impl std::ops::DerefMut for Owned { + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} + +impl AsRef for Owned { + fn as_ref(&self) -> &T { + self + } +} +impl PartialEq for Bytes { + fn eq(&self, other: &Self) -> bool { + **self == **other + } +} +impl PartialEq> for Bytes { + fn eq(&self, other: &Vec) -> bool { + &**self == other.as_slice() + } +} +impl PartialEq for Vec { + fn eq(&self, other: &Bytes) -> bool { + self.as_slice() == &**other + } +} + +#[cfg(test)] +mod tests { + #[test] + fn preparation_vint_encodes_single_and_multiple_byte_boundaries() { + use super::*; + + let mut bytes = Bytes::new(&WriterResources::default()); + bytes.vint(127).unwrap(); + bytes.vint(128).unwrap(); + bytes.vint(u64::MAX).unwrap(); + let mut expected = vec![0x7f, 0x80, 0x01]; + expected.extend_from_slice(&[0xff; 9]); + expected.push(0x01); + assert_eq!(bytes.as_slice(), expected); + } + + #[test] + fn fixed_preparation_capacity_refuses_extra_records_and_impossible_growth() { + use super::*; + + let resources = WriterResources::default().with_max_preparation_bytes(64); + let mut bytes = Bytes::zeroed(4, &resources).unwrap(); + bytes.grow(4).unwrap(); + assert_eq!( + bytes.grow(isize::MAX as usize + 1).unwrap_err().kind(), + crate::ErrorKind::InvalidArgument + ); + assert_eq!(&*bytes, &[0; 4]); + assert_eq!(bytes, vec![0; 4]); + assert_eq!(vec![0; 4], bytes); + assert_eq!(bytes, Bytes::zeroed(4, &resources).unwrap()); + + let mut records = Records::new(1, &resources).unwrap(); + records.push(7u8).unwrap(); + assert!(matches!( + records.push(9), + Err(Error::WriterFailure( + "preparation record count exceeded admission" + )) + )); + assert_eq!(&*records, &[7]); + } + + #[test] + fn preparation_growth_satisfies_both_ledgers_or_leaves_both_unchanged() { + use crate::codec::workspace::Allowance; + let ledger = Allowance::limited(1024); + let resources = WriterResources::default() + .with_max_preparation_bytes(64) + .with_execution_allowance(ledger.clone()); + let mut bytes = Bytes::zeroed(32, &resources).unwrap(); + assert!(bytes.extend_from_slice(&[1; 40]).is_err()); + assert_eq!(&*bytes, &[0; 32]); + assert_eq!(ledger.used(), 32); + drop(bytes); + assert_eq!(ledger.used(), 0); + drop(Bytes::zeroed(64, &resources).unwrap()); + let small = Allowance::limited(16); + let constrained = resources.clone().with_execution_allowance(small.clone()); + assert!(Bytes::zeroed(32, &constrained).is_err()); + assert_eq!(small.used(), 0); + drop(Bytes::zeroed(64, &resources).unwrap()); + assert_eq!(ledger.used(), 0); + } + + #[test] + fn descriptor_growth_counts_replacement_peak_and_preserves_refused_input() { + let resources = WriterResources::default().with_max_preparation_bytes(23); + let mut records = Records::new(1, &resources).unwrap(); + records.push(7u64).unwrap(); + assert!(matches!( + records.push_growing(9), + Err(Error::WriterPreparationLimitExceeded { required: 24, .. }) + )); + assert_eq!(&*records, &[7]); + assert_eq!(used(&resources), 8); + drop(records); + assert_eq!(used(&resources), 0); + let resources = resources.with_max_preparation_bytes(24); + let mut records = Records::new(1, &resources).unwrap(); + records.push(7u64).unwrap(); + records.push_growing(9).unwrap(); + assert_eq!(&*records, &[7, 9]); + assert_eq!(used(&resources), 16); + drop(records); + assert_eq!(used(&resources), 0); + } + + use super::*; + fn used(resources: &WriterResources) -> u64 { + *resources + .preparation_budget + .as_ref() + .unwrap() + .used + .lock() + .unwrap() + } + + #[test] + fn byte_growth_admits_replacement_peak_and_preserves_refused_contents() { + let resources = WriterResources::default().with_max_preparation_bytes(11); + let mut bytes = Bytes::new(&resources); + bytes.extend_from_slice(b"abcd").unwrap(); + assert_eq!(used(&resources), 4); + assert_eq!( + bytes.extend_from_slice(b"e").unwrap_err(), + Error::WriterPreparationLimitExceeded { + limit: 11, + required: 12, + used: 4 + } + ); + assert_eq!(&*bytes, b"abcd"); + assert_eq!(used(&resources), 4); + drop(bytes); + assert_eq!(used(&resources), 0); + let resources = resources.with_max_preparation_bytes(12); + let mut bytes = Bytes::new(&resources); + bytes.extend_from_slice(b"abcd").unwrap(); + bytes.extend_from_slice(b"e").unwrap(); + assert_eq!(bytes.capacity(), 8); + assert_eq!(used(&resources), 8); + let mut other = Bytes::new(&resources.clone()); + assert!(other.extend_from_slice(b"12345").is_err()); + drop(bytes); + other.extend_from_slice(b"12345").unwrap(); + drop(other); + assert_eq!(used(&resources), 0); + } + + #[test] + fn records_and_boxed_state_keep_their_allocation_charged_until_freed() { + struct Probe(WriterResources); + impl Drop for Probe { + fn drop(&mut self) { + assert!(used(&self.0) > 0); + } + } + let resources = WriterResources::default().with_max_preparation_bytes(4096); + let mut records = Records::new(2, &resources).unwrap(); + records.push(Probe(resources.clone())).unwrap(); + records.push(Probe(resources.clone())).unwrap(); + let bytes = 2 * std::mem::size_of::() as u64; + assert_eq!(used(&resources), bytes); + let mut iter = records.into_iter(); + drop(iter.next()); + assert_eq!(used(&resources), bytes); + drop(iter); + assert_eq!(used(&resources), 0); + let failure = std::panic::catch_unwind(|| { + let _owned = Owned::new(Probe(resources.clone()), &resources).unwrap(); + let _bytes = Bytes::zeroed(100, &resources).unwrap(); + panic!("injected failure"); + }); + assert!(failure.is_err()); + assert_eq!(used(&resources), 0); + } + + #[test] + fn concurrent_preparation_shares_one_admission_boundary() { + let resources = WriterResources::default().with_max_preparation_bytes(128); + std::thread::scope(|scope| { + let hold = Bytes::zeroed(128, &resources).unwrap(); + scope + .spawn(|| assert!(Bytes::zeroed(1, &resources.clone()).is_err())) + .join() + .unwrap(); + drop(hold); + scope + .spawn(|| { + Bytes::zeroed(128, &resources.clone()).unwrap(); + }) + .join() + .unwrap(); + }); + assert_eq!(used(&resources), 0); + } +} diff --git a/arca-rar/vendor/rars/src/temp_file.rs b/arca-rar/vendor/rars/src/temp_file.rs new file mode 100644 index 00000000..2caa0b8f --- /dev/null +++ b/arca-rar/vendor/rars/src/temp_file.rs @@ -0,0 +1,181 @@ +//! Temporary creation shared by private spools and archive publication. + +use crate::atomic64::AtomicU64; +use crate::{Error, Result}; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; + +static SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum TemporaryKind { + Spool, + #[cfg(any(feature = "write", feature = "recovery"))] + Archive, +} + +pub(crate) fn next_sequence() -> u64 { + SEQUENCE.fetch_add(1, Ordering::Relaxed) +} + +pub(crate) fn create_with_sequence( + directory: &Path, + admit: impl FnMut(usize) -> Result, + next_sequence: impl FnMut() -> u64, +) -> Result<(PathBuf, File, C)> { + create_named_with_sequence(TemporaryKind::Spool, directory, admit, next_sequence) +} + +pub(crate) fn create_named_with_sequence( + kind: TemporaryKind, + directory: &Path, + mut admit: impl FnMut(usize) -> Result, + mut next_sequence: impl FnMut() -> u64, +) -> Result<(PathBuf, File, C)> { + let (prefix, overflow, collision) = match kind { + TemporaryKind::Spool => ( + ".rars-spool", + "spool path capacity overflows", + "could not allocate a unique rars spool file", + ), + #[cfg(any(feature = "write", feature = "recovery"))] + TemporaryKind::Archive => ( + ".rars-writing", + "temporary path capacity overflows", + "could not allocate a unique archive temporary file", + ), + }; + for _ in 0..128 { + let sequence = next_sequence(); + // Both prefixes, a process ID and u64 hex sequence fit this buffer. + let mut name = [0u8; 64]; + let mut name_writer = std::io::Cursor::new(&mut name[..]); + write!( + name_writer, + "{prefix}-{}-{sequence:016x}", + std::process::id() + ) + .expect("temporary name fits its fixed buffer"); + let name_len = name_writer.position() as usize; + let name = std::str::from_utf8(&name[..name_len]).expect("ASCII temporary name"); + let capacity = directory + .as_os_str() + .len() + .checked_add(1 + name_len) + .ok_or(Error::InvalidArgument(overflow))?; + let path_charge = admit(capacity)?; + let mut path = PathBuf::with_capacity(capacity); + path.push(directory); + path.push(name); + let mut options = File::options(); + options.write(true).create_new(true); + if kind == TemporaryKind::Spool { + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // Temporary storage can hold plaintext before encryption or after + // decryption, so keep its creation mode private. + options.mode(0o600); + } + } + match options.open(&path) { + Ok(file) => { + return Ok((path, file, path_charge)); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } + } + Err(std::io::Error::new(std::io::ErrorKind::AlreadyExists, collision).into()) +} + +/// An uncharged file owner. The reader applies its own logical disk quota. +pub(crate) struct TemporaryFile { + path: PathBuf, + file: Option, +} + +impl TemporaryFile { + pub(crate) fn create(directory: &Path) -> Result { + let (path, file, ()) = create_with_sequence(directory, |_| Ok(()), next_sequence)?; + Ok(Self { + path, + file: Some(file), + }) + } +} + +impl Read for TemporaryFile { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + self.file + .as_mut() + .expect("temporary file is present") + .read(bytes) + } +} +impl Write for TemporaryFile { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.file + .as_mut() + .expect("temporary file is present") + .write(bytes) + } + fn flush(&mut self) -> std::io::Result<()> { + self.file + .as_mut() + .expect("temporary file is present") + .flush() + } +} +impl Seek for TemporaryFile { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.file + .as_mut() + .expect("temporary file is present") + .seek(from) + } +} +impl Drop for TemporaryFile { + fn drop(&mut self) { + self.file = None; + let _ = std::fs::remove_file(&self.path); + } +} + +#[cfg(all(test, unix, any(feature = "write", feature = "recovery")))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn creation_permissions_preserve_private_spools_and_default_archives() { + let root = crate::scratch::case("temporary-creation-permissions"); + let reference = File::create(root.join("reference")).unwrap(); + let default_mode = reference.metadata().unwrap().permissions().mode() & 0o777; + for kind in [TemporaryKind::Spool, TemporaryKind::Archive] { + let (_, mut file, ()) = + create_named_with_sequence(kind, &root, |_| Ok(()), || 0).unwrap(); + let mode = file.metadata().unwrap().permissions().mode() & 0o777; + if kind == TemporaryKind::Spool { + assert_eq!(mode & 0o077, 0, "plaintext spool must remain private"); + file.write_all(b"payload").unwrap(); + file.rewind().unwrap(); + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"payload"); + } else { + assert_eq!( + mode, default_mode, + "publication retains ordinary archive permissions" + ); + assert!( + file.read(&mut [0]).is_err(), + "publication file remains write-only" + ); + } + } + } +} diff --git a/arca-rar/vendor/rars/src/test_features.rs b/arca-rar/vendor/rars/src/test_features.rs new file mode 100644 index 00000000..edd86c13 --- /dev/null +++ b/arca-rar/vendor/rars/src/test_features.rs @@ -0,0 +1,283 @@ +//! Unit-test fixtures track independently selected Cargo capabilities. + +#[cfg(feature = "write")] +pub(crate) fn encryption_cases() -> impl Iterator { + std::iter::once(false).chain(cfg!(feature = "encryption").then_some(true)) +} + +#[test] +fn plain_readers_and_fixed_key_comments_remain_available() { + use crate::ArchiveReader; + let cases: &[(&[u8], &[&[u8]])] = &[ + ( + include_bytes!("../tests/fixtures/rar13/README.RAR"), + &[include_bytes!("../tests/fixtures/rar13/README")], + ), + ( + include_bytes!("../tests/fixtures/rar15_40/rar300/solid_simple_rar300.rar"), + &[ + b"shared prefix shared prefix shared prefix alpha\n", + b"shared prefix shared prefix shared prefix beta\n", + ], + ), + ( + include_bytes!("../tests/fixtures/rar50/stored.rar"), + &[b"Hello, RAR 5.0 fixture world.\n"], + ), + ]; + for &(data, expected) in cases { + let archive = ArchiveReader::read(data).unwrap(); + assert_eq!(archive.members().count(), expected.len()); + for (index, bytes) in expected.iter().enumerate() { + assert_eq!( + &archive + .read_member_at(index, None) + .unwrap() + .expect("indexed member"), + bytes + ); + } + } + let archive = + ArchiveReader::read(include_bytes!("../tests/fixtures/rar13/COMMENT.RAR")).unwrap(); + assert_eq!( + archive.comment(None).unwrap().unwrap(), + b"This is the archive comment.\r\n" + ); +} + +#[cfg(not(feature = "encryption"))] +#[test] +fn disabled_crypto_retains_metadata_and_typed_payload_and_header_refusals() { + use crate::{ArchiveReadOptions, ArchiveReader, Error}; + for data in [ + include_bytes!("../tests/fixtures/rar13/README_password=password.rar").as_slice(), + include_bytes!("../tests/fixtures/rar15_40/rar154/readme_154_password.rar").as_slice(), + include_bytes!("../tests/fixtures/rar15_40/encrypted/per_file_rar300_password.rar") + .as_slice(), + include_bytes!("../tests/fixtures/rar50/password_aes.rar").as_slice(), + ] { + for password in [None, Some(b"password".as_slice())] { + let archive = ArchiveReader::read_with_options( + data, + ArchiveReadOptions::with_optional_password(password), + ) + .unwrap(); + let index = archive.members().position(|m| m.meta.is_encrypted).unwrap(); + assert_eq!( + archive + .read_member_at(index, password) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "encryption" + } + ); + } + } + for data in [ + include_bytes!("../tests/fixtures/rar15_40/encrypted/header_rar300_password.rar") + .as_slice(), + include_bytes!("../tests/fixtures/rar50/header_encrypted.rar").as_slice(), + ] { + for password in [None, Some(b"password".as_slice())] { + assert_eq!( + ArchiveReader::read_with_options( + data, + ArchiveReadOptions::with_optional_password(password) + ) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "encryption" + } + ); + } + } + assert!(matches!( + crate::crypto::rar50::Rar50Keys::derive(b"password", [0; 16], 255), + Err(crate::crypto::rar50::Error::FeatureDisabled) + )); +} + +#[cfg(not(feature = "recovery"))] +#[test] +fn disabled_recovery_keeps_record_metadata_and_preserves_output() { + use crate::{ArchiveReadOptions, ArchiveReader, Error, ReadCancellation}; + let data = include_bytes!("../tests/fixtures/rar50/with_recovery.rar"); + let archive = ArchiveReader::read(data).unwrap(); + assert!(archive + .as_rar50() + .unwrap() + .services() + .any(|s| s.recovery_record().unwrap().is_some())); + let mut output = b"untouched".to_vec(); + assert_eq!( + archive + .repair_recovery_to(&mut output) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "recovery" + } + ); + assert_eq!(output, b"untouched"); + let token = ReadCancellation::new(); + token.cancel(); + assert_eq!( + archive + .repair_recovery_with_options(ArchiveReadOptions::new().with_cancellation(&token)) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "recovery" + } + ); +} + +#[cfg(all(feature = "write", not(feature = "encryption")))] +#[test] +fn disabled_encrypted_writers_refuse_before_source_access_or_output() { + use crate::{ArchiveVersion, Builder, EntrySource, Error, WriterResources}; + for version in ArchiveVersion::ALL { + let mut builder = Builder::new(version) + .store(true) + .password(Some(b"password".to_vec())); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(1, || panic!("crypto refusal must precede source access")), + None, + None, + ) + .unwrap(); + let mut output = b"untouched".to_vec(); + assert_eq!( + builder + .write_to(&mut output, &WriterResources::default(), None) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "encryption" + } + ); + assert_eq!(output, b"untouched"); + } +} + +#[cfg(all(feature = "write", not(feature = "recovery")))] +#[test] +fn disabled_recovery_writers_refuse_before_source_access_or_output() { + use crate::{ArchiveVersion, Builder, EntrySource, Error, WriterResources}; + let mut builder = Builder::new(ArchiveVersion::Rar50).recovery_percent(Some(1)); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(1, || panic!("recovery refusal must precede source access")), + None, + None, + ) + .unwrap(); + let mut output = b"untouched".to_vec(); + assert_eq!( + builder + .write_to(&mut output, &WriterResources::default(), None) + .unwrap_err() + .root_cause(), + &Error::FeatureDisabled { + feature: "recovery" + } + ); + assert_eq!(output, b"untouched"); +} + +#[test] +fn crypto_key_equality_and_debug_do_not_require_derivation() { + use crate::crypto::rar50::Rar50Keys; + let keys = Rar50Keys { + key: [1; 32], + hash_key: [2; 32], + password_check: [3; 8], + }; + assert_eq!(keys, keys.clone()); + for field in 0..3 { + let mut different = keys.clone(); + match field { + 0 => different.key[31] ^= 1, + 1 => different.hash_key[31] ^= 1, + _ => different.password_check[7] ^= 1, + } + assert_ne!(keys, different); + } + assert_eq!(format!("{keys:?}"), "Rar50Keys { .. }"); +} + +#[cfg(feature = "write")] +#[test] +fn small_explicit_delta_decodes_without_crypto_or_recovery() { + use crate::rar50::{ArchiveEntry, Rar50Writer, WriterOptions}; + use crate::{ + ArchiveReadOptions, ArchiveReader, ArchiveVersion, EntrySource, Error, FeatureSet, + FilterKind, FilterPolicy, FilterSpec, + }; + let data: Vec = (0..512).map(|i| (i / 4) as u8).collect(); + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1) + .with_dictionary_size(128 * 1024), + ) + .entry(ArchiveEntry::new( + b"small-delta".to_vec(), + EntrySource::from_bytes(data.clone()), + )) + .filter_policy(FilterPolicy::Explicit(FilterSpec::whole( + FilterKind::Delta { channels: 4 }, + ))) + .finish() + .unwrap(); + let archive = ArchiveReader::read(&bytes).unwrap(); + assert!(!archive + .as_rar50() + .unwrap() + .files() + .next() + .unwrap() + .is_stored()); + assert_eq!(archive.read_member_at(0, None).unwrap().unwrap(), data); + // An unfiltered compressed member succeeds on this route. This exact + // refusal proves the produced archive actually contains a filter command. + let error = archive + .test_with_options(ArchiveReadOptions::new().with_rar50_buffered_decode_limit(0)) + .unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::Rar50BufferedDecodeLimitExceeded { + limit: 0, + required: 512 + } + ); + let plain_bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1) + .with_dictionary_size(128 * 1024), + ) + .entry(ArchiveEntry::new( + b"small-plain".to_vec(), + EntrySource::from_bytes(data.clone()), + )) + .filter_policy(FilterPolicy::None) + .finish() + .unwrap(); + let plain = ArchiveReader::read(&plain_bytes).unwrap(); + assert!(!plain + .as_rar50() + .unwrap() + .files() + .next() + .unwrap() + .is_stored()); + plain + .test_with_options(ArchiveReadOptions::new().with_rar50_buffered_decode_limit(0)) + .unwrap(); + assert_eq!(plain.read_member_at(0, None).unwrap().unwrap(), data); +} diff --git a/arca-rar/vendor/rars/src/timestamp.rs b/arca-rar/vendor/rars/src/timestamp.rs new file mode 100644 index 00000000..85e464a6 --- /dev/null +++ b/arca-rar/vendor/rars/src/timestamp.rs @@ -0,0 +1,342 @@ +//! Timestamp conversions shared by archive consumers. +//! Legacy DOS values follow the established extracting-machine local-zone policy. + +use crate::tzif::TimeZone; +use crate::{ArchiveFamily, TimeRefinement}; +use std::fs; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// A stored timestamp with its interpretation attached, before refinements. +/// +/// This is a view of archive metadata, not necessarily a valid instant. +/// Presence is represented separately by `Option`: +/// `DosLocal(0)` is an invalid stored date, while `UnixSeconds(0)` is the Unix +/// epoch. Odd-second and subsecond refinements remain separate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum StoredTimestamp { + /// Packed DOS date/time fields with no stored timezone and two-second + /// resolution. Obtaining an instant requires a timezone policy. + DosLocal(u32), + /// Absolute whole seconds since the Unix epoch. + UnixSeconds(u32), +} + +impl StoredTimestamp { + pub(crate) fn from_family(family: ArchiveFamily, raw: u32) -> Self { + match family { + ArchiveFamily::Rar13 | ArchiveFamily::Rar15To40 => Self::DosLocal(raw), + ArchiveFamily::Rar50Plus => Self::UnixSeconds(raw), + } + } + + /// Calendar fields `(year, month, day, hour, minute, second)` for display. + /// + /// DOS values expose the stored wall-clock fields without timezone + /// conversion; Unix values use UTC. Refinements are not applied. For + /// compatibility with Python date tuples, a zero DOS month or day returns + /// `None`; other field values are exposed without validation. + /// Use [`crate::ArchiveMemberMeta::modification_time`] to obtain an instant + /// under the existing local-zone and refinement policy instead. + pub fn calendar_fields(self) -> Option<(u16, u8, u8, u8, u8, u8)> { + match self { + Self::UnixSeconds(seconds) => Some(unix_datetime(seconds)), + Self::DosLocal(raw) => { + let year = 1980 + ((raw >> 25) & 0x7f) as u16; + let month = ((raw >> 21) & 0x0f) as u8; + let day = ((raw >> 16) & 0x1f) as u8; + let hour = ((raw >> 11) & 0x1f) as u8; + let minute = ((raw >> 5) & 0x3f) as u8; + let second = ((raw & 0x1f) * 2) as u8; + (month != 0 && day != 0).then_some((year, month, day, hour, minute, second)) + } + } + } +} + +pub fn current_filetime() -> u64 { + const FILETIME_UNIX_EPOCH_SECONDS: u64 = 11_644_473_600; + const FILETIME_TICKS_PER_SECOND: u64 = 10_000_000; + + let duration = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let seconds = duration + .as_secs() + .saturating_add(FILETIME_UNIX_EPOCH_SECONDS) + .saturating_mul(FILETIME_TICKS_PER_SECOND); + seconds.saturating_add(u64::from(duration.subsec_nanos() / 100)) +} + +pub fn format_filetime_utc(filetime: u64) -> String { + filetime_to_system_time(filetime) + .and_then(format_system_time_utc) + .unwrap_or_else(|| format!("{filetime:#018x}")) +} + +pub fn source_unix_mtime(metadata: &fs::Metadata) -> Option { + metadata + .modified() + .ok()? + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|duration| u32::try_from(duration.as_secs()).ok()) +} + +/// UTC calendar fields for a RAR5 whole-second timestamp, including Unix epoch. +pub fn unix_datetime(seconds: u32) -> (u16, u8, u8, u8, u8, u8) { + let (year, month, day) = civil_from_days(i64::from(seconds / 86_400)); + let within_day = seconds % 86_400; + ( + year as u16, + month as u8, + day as u8, + (within_day / 3600) as u8, + ((within_day % 3600) / 60) as u8, + (within_day % 60) as u8, + ) +} + +pub fn source_dos_mtime(metadata: &fs::Metadata) -> u32 { + metadata + .modified() + .ok() + .and_then(system_time_to_dos_time) + .unwrap_or(0) +} + +/// Converts a present stored timestamp without treating Unix epoch as missing. +/// Legacy DOS zero remains invalid under the existing local-zone conversion. +pub fn extracted_system_time( + family: ArchiveFamily, + file_time: Option, + refinement: Option, +) -> Option { + let file_time = file_time?; + let base = match family { + ArchiveFamily::Rar13 | ArchiveFamily::Rar15To40 => dos_time_to_system_time(file_time), + ArchiveFamily::Rar50Plus => Some(UNIX_EPOCH + Duration::from_secs(u64::from(file_time))), + }?; + // A DOS timestamp counts in two-second steps, so an odd second and any + // sub-second detail arrive separately and have to be added back on. + let Some(refinement) = refinement else { + return Some(base); + }; + let extra = Duration::from_secs(u64::from(refinement.add_second)) + + Duration::from_nanos(u64::from(refinement.nanoseconds)); + base.checked_add(extra).or(Some(base)) +} + +/// A DOS timestamp is a wall-clock reading with no zone, so both directions go +/// through the local zone rather than UTC. See [`crate::tzif`] for why, and for +/// what that costs. +fn system_time_to_dos_time(time: SystemTime) -> Option { + let instant = i64::try_from(time.duration_since(UNIX_EPOCH).ok()?.as_secs()).ok()?; + let local = instant.checked_add(i64::from(TimeZone::local().offset_at(instant)))?; + let seconds = u64::try_from(local).ok()?; + let days = i64::try_from(seconds / 86_400).ok()?; + let seconds_of_day = seconds % 86_400; + let (year, month, day) = civil_from_days(days); + if !(1980..=2107).contains(&year) { + return None; + } + let hour = seconds_of_day / 3_600; + let minute = (seconds_of_day % 3_600) / 60; + let second = seconds_of_day % 60; + Some( + ((u32::try_from(year - 1980).ok()?) << 25) + | (month << 21) + | (day << 16) + | ((hour as u32) << 11) + | ((minute as u32) << 5) + | ((second as u32) / 2), + ) +} + +fn dos_time_to_system_time(time: u32) -> Option { + if time == 0 { + return None; + } + let second = (time & 0x1f) * 2; + let minute = (time >> 5) & 0x3f; + let hour = (time >> 11) & 0x1f; + let day = (time >> 16) & 0x1f; + let month = (time >> 21) & 0x0f; + let year = 1980 + i32::try_from((time >> 25) & 0x7f).ok()?; + // The five-bit day field cannot exceed 31. + if month == 0 || month > 12 || day == 0 || hour > 23 || minute > 59 || second > 59 { + return None; + } + let days = days_from_civil(year, month, day); + let local = days + .checked_mul(86_400)? + .checked_add(i64::from(hour) * 3_600 + i64::from(minute) * 60 + i64::from(second))?; + let zone = TimeZone::local(); + let instant = local.checked_sub(i64::from(zone.offset_for_local(local)))?; + Some(UNIX_EPOCH + Duration::from_secs(u64::try_from(instant).ok()?)) +} + +fn filetime_to_system_time(filetime: u64) -> Option { + const FILETIME_UNIX_EPOCH_SECONDS: u64 = 11_644_473_600; + const FILETIME_TICKS_PER_SECOND: u64 = 10_000_000; + + let seconds = filetime / FILETIME_TICKS_PER_SECOND; + let ticks = filetime % FILETIME_TICKS_PER_SECOND; + let unix_seconds = seconds.checked_sub(FILETIME_UNIX_EPOCH_SECONDS)?; + Some(UNIX_EPOCH + Duration::from_secs(unix_seconds) + Duration::from_nanos(ticks * 100)) +} + +fn format_system_time_utc(time: SystemTime) -> Option { + let duration = time.duration_since(UNIX_EPOCH).ok()?; + let days = i64::try_from(duration.as_secs() / 86_400).ok()?; + let seconds_of_day = duration.as_secs() % 86_400; + let (year, month, day) = civil_from_days(days); + let hour = seconds_of_day / 3_600; + let minute = (seconds_of_day % 3_600) / 60; + let second = seconds_of_day % 60; + Some(format!( + "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}Z" + )) +} + +fn civil_from_days(days_since_unix_epoch: i64) -> (i32, u32, u32) { + // All callers derive days from nonnegative Unix durations. + let z = days_since_unix_epoch + 719_468; + let era = z / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = doy - (153 * mp + 2) / 5 + 1; + let month = mp + if mp < 10 { 3 } else { -9 }; + let year = y + i64::from(month <= 2); + (year as i32, month as u32, day as u32) +} + +fn days_from_civil(year: i32, month: u32, day: u32) -> i64 { + // The sole production caller extracts DOS years in 1980..=2107. + let year = i64::from(year) - i64::from(month <= 2); + let era = year / 400; + let yoe = year - era * 400; + let month = i64::from(month); + let day = i64::from(day); + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +#[cfg(test)] +mod tests { + use super::StoredTimestamp; + + #[test] + fn calendar_conversions_match_every_day_in_the_dos_range() { + // Independent Gregorian reference: count days from the Unix epoch, + // using month lengths rather than the production March-based formula. + let mut days = 0; + for year in 1970..=2107 { + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let months = [ + 31, + if leap { 29 } else { 28 }, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31, + ]; + for (index, length) in months.into_iter().enumerate() { + let month = index as u32 + 1; + for day in 1..=length { + assert_eq!(super::civil_from_days(days), (year, month, day)); + if year >= 1980 { + assert_eq!(super::days_from_civil(year, month, day), days); + } + days += 1; + } + } + } + assert_eq!(super::unix_datetime(u32::MAX), (2106, 2, 7, 6, 28, 15)); + } + + #[test] + fn source_dos_conversion_refuses_dates_outside_its_year_range() { + use std::time::{Duration, UNIX_EPOCH}; + assert!(super::system_time_to_dos_time(UNIX_EPOCH - Duration::from_secs(1)).is_none()); + // These dates remain outside the DOS range in every real timezone. + for seconds in [0, 4_386_528_000] { + // 1970-01-01 and 2109-01-02 UTC + assert!( + super::system_time_to_dos_time(UNIX_EPOCH + Duration::from_secs(seconds)).is_none() + ); + } + // Midyear is safely separated from a timezone-dependent year boundary. + for (seconds, year) in [(331_344_000, 1980), (4_339_852_800, 2107)] { + let packed = + super::system_time_to_dos_time(UNIX_EPOCH + Duration::from_secs(seconds)).unwrap(); + assert_eq!(1980 + (packed >> 25), year); + let decoded = super::dos_time_to_system_time(packed).unwrap(); + let original = UNIX_EPOCH + Duration::from_secs(seconds); + assert!(original.duration_since(decoded).unwrap() < Duration::from_secs(2)); + } + } + + #[test] + fn filetime_format_boundaries_keep_epoch_and_far_future_dates() { + let epoch = 116_444_736_000_000_000; + assert_eq!(super::format_filetime_utc(epoch - 1), "0x019db1ded53e7fff"); + assert_eq!(super::format_filetime_utc(epoch), "1970-01-01T00:00:00Z"); + assert_eq!( + super::format_filetime_utc(u64::MAX), + "60056-05-28T05:36:10Z" + ); + } + + #[test] + fn invalid_dos_fields_do_not_become_extracted_instants() { + // Packed DOS fields have independent ranges, with two-second units. + let year = 40u32 << 25; + let valid_date = year | (1 << 21) | (1 << 16); + for raw in [ + year | (1 << 16), // month zero + year | (13 << 21) | (1 << 16), + year | (1 << 21), // day zero + valid_date | (24 << 11), + valid_date | (60 << 5), + valid_date | 30, // second 60 + ] { + for family in [crate::ArchiveFamily::Rar13, crate::ArchiveFamily::Rar15To40] { + assert!(super::extracted_system_time(family, Some(raw), None).is_none()); + } + } + } + + #[test] + fn stored_calendar_fields_keep_wall_clock_separate_from_utc() { + assert_eq!(StoredTimestamp::DosLocal(0).calendar_fields(), None); + assert_eq!( + StoredTimestamp::UnixSeconds(0).calendar_fields(), + Some((1970, 1, 1, 0, 0, 0)) + ); + assert_eq!( + StoredTimestamp::DosLocal(0x5022_1882).calendar_fields(), + Some((2020, 1, 2, 3, 4, 4)) + ); + assert_eq!( + StoredTimestamp::UnixSeconds(1_582_934_400).calendar_fields(), + Some((2020, 2, 29, 0, 0, 0)) + ); + // The display accessor retains historically exposed raw DOS fields, + // rather than silently normalizing malformed dates into another date. + assert_eq!( + StoredTimestamp::DosLocal(u32::MAX).calendar_fields(), + Some((2107, 15, 31, 31, 63, 62)) + ); + } +} diff --git a/arca-rar/vendor/rars/src/tzif.rs b/arca-rar/vendor/rars/src/tzif.rs new file mode 100644 index 00000000..47355354 --- /dev/null +++ b/arca-rar/vendor/rars/src/tzif.rs @@ -0,0 +1,654 @@ +//! The local time zone, read from the system's TZif database. +//! +//! RAR 1.3 through 4.x store a modification time as MS-DOS fields: a wall-clock +//! reading with no zone and no offset. Every reader in the wild resolves that +//! against the *extracting* machine's local zone, so a file archived at 15:00 +//! shows 15:00 wherever it is unpacked. Reading those fields as UTC instead is +//! self-consistent and round-trips, which is how the mistake survives, but it +//! disagrees with every other tool by the local offset. +//! +//! Doing better needs the zone's offset *for the timestamp in question*, since +//! a summer archive extracted in winter still needs summer's rules. That means +//! the zone database. The C library would answer it in two calls, but both are +//! `unsafe`, and this workspace forbids that; a TZif file is a binary format, +//! and parsing those is what the rest of this project does anyway. +//! +//! Only what a timestamp conversion needs is read: transition instants, the +//! offset in force after each, and whether that offset is daylight saving. +//! Designations, leap seconds and the standard/wall indicators are skipped. +//! +//! The trailing POSIX rule string is skipped too, which is the one deliberate +//! gap. It governs dates after the final transition, and the tables run to +//! 2037. Before then the transitions answer everything, including for zones +//! that have none at all: a fixed-offset zone still records its offset in the +//! type table, so `Etc/GMT+5` resolves correctly with an empty transition list. +//! After 2037 a daylight-saving zone freezes at its final offset, which needs a +//! file stamped past 2038 *and* stored in a pre-RAR-5 archive to notice. + +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +/// One local time type: an offset from UTC and whether it is daylight saving. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct LocalType { + offset: i32, + is_dst: bool, +} + +/// A time zone's transitions, enough to convert in both directions. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TimeZone { + /// Transition instants in Unix seconds, ascending. + transitions: Vec, + /// The local type in force from the matching transition onward. + types: Vec, + /// Which entry of `types` each transition selects. + indices: Vec, +} + +impl TimeZone { + /// The zone where every offset is zero, used when nothing else resolves. + pub(crate) fn utc() -> Self { + Self { + transitions: Vec::new(), + types: vec![LocalType { + offset: 0, + is_dst: false, + }], + indices: Vec::new(), + } + } + + /// The machine's zone, parsed once and reused. + /// + /// Anything unreadable or unparsable falls back to UTC, which is what this + /// did before the database was consulted at all. A timestamp is not worth + /// failing an extraction over. + pub(crate) fn local() -> &'static Self { + static LOCAL: OnceLock = OnceLock::new(); + LOCAL.get_or_init(|| { + local_zone_path() + .and_then(|path| std::fs::read(path).ok()) + .and_then(|bytes| Self::parse(&bytes)) + .unwrap_or_else(Self::utc) + }) + } + + /// Reads a TZif file, preferring its 64-bit block when there is one. + pub(crate) fn parse(bytes: &[u8]) -> Option { + let header = Header::parse(bytes, 0)?; + // A version 2 or later file repeats everything in a second block with + // 64-bit transition times. That block is the authoritative one; the + // first exists only for readers that predate it. + if header.version >= b'2' { + let second = header.data_end(TimeWidth::Bits32)?; + let header = Header::parse(bytes, second)?; + Self::parse_block(bytes, &header, second + Header::LEN, TimeWidth::Bits64) + } else { + Self::parse_block(bytes, &header, Header::LEN, TimeWidth::Bits32) + } + } + + fn parse_block(bytes: &[u8], header: &Header, start: usize, width: TimeWidth) -> Option { + // Check every declared field, including skipped data, before reserving + // arrays from the header's counts. + bytes.get(start..start.checked_add(header.data_len(width)?)?)?; + let stride = width.bytes(); + let mut pos = start; + + let mut transitions = Vec::with_capacity(header.timecnt); + for _ in 0..header.timecnt { + let raw = bytes.get(pos..pos.checked_add(stride)?)?; + transitions.push(match width { + TimeWidth::Bits32 => i64::from(i32::from_be_bytes(raw.try_into().ok()?)), + TimeWidth::Bits64 => i64::from_be_bytes(raw.try_into().ok()?), + }); + pos += stride; + } + + let indices = bytes.get(pos..pos.checked_add(header.timecnt)?)?.to_vec(); + pos += header.timecnt; + + let mut types = Vec::with_capacity(header.typecnt); + for _ in 0..header.typecnt { + let record = bytes.get(pos..pos.checked_add(6)?)?; + types.push(LocalType { + offset: i32::from_be_bytes(record[..4].try_into().ok()?), + is_dst: record[4] != 0, + }); + pos += 6; + } + + if types.is_empty() || indices.iter().any(|&i| usize::from(i) >= types.len()) { + return None; + } + if transitions.windows(2).any(|pair| pair[0] >= pair[1]) { + return None; + } + Some(Self { + transitions, + types, + indices, + }) + } + + /// The offset in force at an instant. + fn type_at(&self, unix_seconds: i64) -> LocalType { + let after = self + .transitions + .partition_point(|&transition| transition <= unix_seconds); + match after.checked_sub(1) { + Some(index) => self.types[usize::from(self.indices[index])], + // RFC 8536 section 3.2 specifies type 0 before the first transition. + None => self.types[0], + } + } + + /// The offset in force at an instant, in seconds east of UTC. + pub(crate) fn offset_at(&self, unix_seconds: i64) -> i32 { + self.type_at(unix_seconds).offset + } + + /// The offset that turns a wall-clock reading back into an instant. + /// + /// A reading is not always one instant. In the hour a zone repeats when + /// daylight saving ends it names two, and in the hour it skips when + /// daylight saving starts it names none. An offset is accepted here when + /// applying it lands on an instant whose own offset is the same one, so + /// only genuinely reachable readings are accepted; where two qualify the + /// standard-time one wins, and where none do the search still returns the + /// standard-time candidate rather than failing. + /// + /// The reference readers disagree with each other in exactly these hours, + /// so there is nothing to match: RAR 7.12 resolves the repeated hour to + /// standard time under `Europe/London` and to daylight saving under + /// `America/New_York`. Everything outside those two hours a year is + /// unambiguous and agrees. + pub(crate) fn offset_for_local(&self, local_seconds: i64) -> i32 { + let mut best: Option = None; + for candidate in self.candidates(local_seconds) { + if self.type_at(local_seconds - i64::from(candidate.offset)) != candidate { + continue; + } + let better = match best { + None => true, + Some(current) => current.is_dst && !candidate.is_dst, + }; + if better { + best = Some(candidate); + } + } + best.or_else(|| { + // A reading in the skipped hour matches nothing. Resolve it with + // the standard-time offset, which moves it forward into the hour + // that does exist rather than backward into the one that does not. + self.candidates(local_seconds) + .into_iter() + .find(|local| !local.is_dst) + }) + .unwrap_or_else(|| self.type_at(local_seconds)) + .offset + } + + /// The types plausibly in force around a wall-clock reading. + /// + /// A day either side covers every real transition, which never move a + /// clock by anything close to that. + fn candidates(&self, local_seconds: i64) -> Vec { + const DAY: i64 = 86_400; + let mut found: Vec = Vec::new(); + for probe in [local_seconds - DAY, local_seconds, local_seconds + DAY] { + let local = self.type_at(probe); + if !found.contains(&local) { + found.push(local); + } + } + found + } +} + +/// Which file holds the machine's zone. +/// +/// `TZ` names a zone under the database directory; unset, the symlink at +/// `/etc/localtime` is the answer. A `TZ` holding a POSIX rule string rather +/// than a name has no file, and resolves to nothing here. +fn local_zone_path() -> Option { + zone_path( + std::env::var_os("TZ").as_deref(), + std::env::var_os("TZDIR").as_deref(), + ) +} + +fn zone_path( + name: Option<&std::ffi::OsStr>, + directory: Option<&std::ffi::OsStr>, +) -> Option { + const DEFAULT_DIR: &str = "/usr/share/zoneinfo"; + + let Some(name) = name else { + return Some(PathBuf::from("/etc/localtime")); + }; + let name = name.to_str()?; + // A leading colon is allowed and means the rest is a file name. + let name = name.strip_prefix(':').unwrap_or(name); + if name.is_empty() { + return None; + } + if Path::new(name).is_absolute() { + return Some(PathBuf::from(name)); + } + // TZ comes from the environment, so refuse anything that could walk out of + // the database directory. + if name + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + { + return None; + } + let dir = directory.map_or_else(|| PathBuf::from(DEFAULT_DIR), PathBuf::from); + Some(dir.join(name)) +} + +#[derive(Clone, Copy)] +enum TimeWidth { + Bits32, + Bits64, +} + +impl TimeWidth { + fn bytes(self) -> usize { + match self { + Self::Bits32 => 4, + Self::Bits64 => 8, + } + } +} + +struct Header { + version: u8, + isutcnt: usize, + isstdcnt: usize, + leapcnt: usize, + timecnt: usize, + typecnt: usize, + charcnt: usize, +} + +impl Header { + const LEN: usize = 44; + + fn parse(bytes: &[u8], at: usize) -> Option { + let header = bytes.get(at..at.checked_add(Self::LEN)?)?; + if &header[..4] != b"TZif" { + return None; + } + let counts: Vec = header[20..44] + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| u32::from_be_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) as usize) + .collect(); + Some(Self { + version: header[4], + isutcnt: counts[0], + isstdcnt: counts[1], + leapcnt: counts[2], + timecnt: counts[3], + typecnt: counts[4], + charcnt: counts[5], + }) + } + + /// Where this block's data ends, which is where the next header begins. + fn data_end(&self, width: TimeWidth) -> Option { + Self::LEN.checked_add(self.data_len(width)?) + } + + fn data_len(&self, width: TimeWidth) -> Option { + self.timecnt + .checked_mul(width.bytes() + 1)? + .checked_add(self.typecnt.checked_mul(6)?)? + .checked_add(self.charcnt)? + .checked_add(self.leapcnt.checked_mul(width.bytes() + 4)?)? + .checked_add(self.isstdcnt)? + .checked_add(self.isutcnt) + } +} + +#[cfg(test)] +mod tests { + use super::{LocalType, TimeZone}; + + /// Zone files are shipped rather than read from the machine, so these test + /// the parser and not whatever the build box is set to. + const LONDON: &[u8] = include_bytes!("../tests/fixtures/tz/Europe_London"); + const NEW_YORK: &[u8] = include_bytes!("../tests/fixtures/tz/America_New_York"); + const KOLKATA: &[u8] = include_bytes!("../tests/fixtures/tz/Asia_Kolkata"); + const LORD_HOWE: &[u8] = include_bytes!("../tests/fixtures/tz/Australia_Lord_Howe"); + const GMT_MINUS_5: &[u8] = include_bytes!("../tests/fixtures/tz/Etc_GMT_5"); + + fn zone(name: &str) -> TimeZone { + let bytes = match name { + "Europe/London" => LONDON, + "America/New_York" => NEW_YORK, + "Asia/Kolkata" => KOLKATA, + "Australia/Lord_Howe" => LORD_HOWE, + "Etc/GMT+5" => GMT_MINUS_5, + other => panic!("no fixture for {other}"), + }; + TimeZone::parse(bytes).expect("fixture parses") + } + + /// Expected offsets come from Python's `zoneinfo`, so the arithmetic here + /// is checked against another implementation rather than against itself. + #[test] + fn offset_at_matches_the_zone_database() { + let cases = [ + ("Europe/London", 1_768_478_400, 0), // 2026-01-15 GMT + ("Europe/London", 1_784_116_800, 3_600), // 2026-07-15 BST + ("America/New_York", 1_768_478_400, -18_000), // EST + ("America/New_York", 1_784_116_800, -14_400), // EDT + ("Asia/Kolkata", 1_784_116_800, 19_800), // +05:30, no DST + ("Australia/Lord_Howe", 1_768_478_400, 39_600), // +11 in their summer + ("Australia/Lord_Howe", 1_784_116_800, 37_800), // +10:30, a 30-minute step + ("Etc/GMT+5", 1_784_116_800, -18_000), // no transitions at all + ("Europe/London", 486_475_200, 3_600), // 1985, well before the tables end + ("America/New_York", 946_681_200, -18_000), // 1999 + ]; + for (name, instant, expected) in cases { + assert_eq!( + zone(name).offset_at(instant), + expected, + "{name} at {instant}" + ); + } + } + + /// The reverse direction, on readings that name exactly one instant. + #[test] + fn offset_for_local_matches_the_zone_database() { + let cases = [ + ("Europe/London", 1_768_478_400, 0), // 12:00 GMT + ("Europe/London", 1_784_127_600, 3_600), // 15:00 BST + ("America/New_York", 1_784_109_600, -14_400), // 10:00 EDT + ("Asia/Kolkata", 1_784_143_800, 19_800), // 19:30 IST + ("Australia/Lord_Howe", 1_784_154_600, 37_800), // 22:30 +10:30 + ("Etc/GMT+5", 1_784_098_800, -18_000), // 07:00 -05 + ("Europe/London", 2_131_272_000, 3_600), // 2037, the last year the tables cover + ]; + for (name, reading, expected) in cases { + assert_eq!( + zone(name).offset_for_local(reading), + expected, + "{name} reading {reading}" + ); + } + } + + /// Both directions have to agree, or a timestamp written and read back on + /// one machine would drift. + #[test] + fn the_two_directions_round_trip() { + for name in [ + "Europe/London", + "America/New_York", + "Asia/Kolkata", + "Australia/Lord_Howe", + "Etc/GMT+5", + ] { + let zone = zone(name); + // Every six hours across 2026, skipping nothing. + let mut instant = 1_767_225_600; // 2026-01-01 + while instant < 1_798_761_600 { + let reading = instant + i64::from(zone.offset_at(instant)); + let back = reading - i64::from(zone.offset_for_local(reading)); + // The instant itself need not come back. In the hour a zone + // repeats, two instants share one reading and either is a + // correct answer. What must hold is that the answer still + // reads the same on the clock, which is the whole point of + // storing a reading. + let back_reading = back + i64::from(zone.offset_at(back)); + assert_eq!(back_reading, reading, "{name} at {instant}"); + instant += 21_600; + } + } + } + + /// The hour a zone repeats names two instants and the hour it skips names + /// none. Both resolve to standard time here, deterministically. + /// + /// The reference readers are not consistent in these hours, so there is + /// nothing to match: RAR 7.12 resolves London's repeated hour to standard + /// time and New York's to daylight saving. This pins our own choice. + #[test] + fn ambiguous_and_skipped_readings_resolve_to_standard_time() { + // 2026-10-25 01:30 in London happens twice, as BST then as GMT. + assert_eq!(zone("Europe/London").offset_for_local(1_792_891_800), 0); + // 2026-03-29 01:30 in London never happens; the clock jumps 01:00→02:00. + assert_eq!(zone("Europe/London").offset_for_local(1_774_747_800), 0); + // 2026-11-01 01:30 in New York happens twice, as EDT then as EST. + assert_eq!( + zone("America/New_York").offset_for_local(1_793_496_600), + -18_000 + ); + } + + #[test] + fn a_zone_with_no_transitions_still_has_its_offset() { + let zone = zone("Etc/GMT+5"); + assert!(zone.transitions.is_empty()); + assert_eq!(zone.offset_at(0), -18_000); + assert_eq!(zone.offset_at(i64::MAX / 2), -18_000); + } + + #[test] + fn utc_is_the_fallback_and_has_no_offset() { + let zone = TimeZone::utc(); + assert_eq!(zone.offset_at(1_784_116_800), 0); + assert_eq!(zone.offset_for_local(1_784_116_800), 0); + assert_eq!( + zone.types, + vec![LocalType { + offset: 0, + is_dst: false + }] + ); + } + + // RFC 8536 sections 3.1 and 3.2: a version-1 header, signed 32-bit + // transition instants, type indices, six-byte type records and designations. + fn version_one(transitions: &[(i32, u8)], types: &[(i32, bool)]) -> Vec { + let mut bytes = b"TZif\0".to_vec(); + bytes.resize(20, 0); + for count in [0, 0, 0, transitions.len() as u32, types.len() as u32, 4] { + bytes.extend(count.to_be_bytes()); + } + for &(instant, _) in transitions { + bytes.extend(instant.to_be_bytes()); + } + bytes.extend(transitions.iter().map(|&(_, index)| index)); + for &(offset, daylight) in types { + bytes.extend(offset.to_be_bytes()); + bytes.extend([u8::from(daylight), 0]); + } + bytes.extend(b"STD\0"); + bytes + } + + #[test] + fn version_one_signed_transitions_apply_at_the_exact_instant() { + let bytes = version_one(&[(-3600, 1), (3600, 0)], &[(1800, false), (3600, true)]); + let zone = TimeZone::parse(&bytes).unwrap(); + for (instant, offset) in [(-3601, 1800), (-3600, 3600), (3599, 3600), (3600, 1800)] { + assert_eq!(zone.offset_at(instant), offset); + } + } + + #[test] + fn before_first_transition_uses_type_zero_even_when_it_is_daylight_time() { + let bytes = version_one(&[(2_000_000_000, 1)], &[(3600, true), (0, false)]); + let zone = TimeZone::parse(&bytes).unwrap(); + assert_eq!(zone.offset_at(1_000_000_000), 3600); + assert_eq!(zone.offset_at(1_999_999_999), 3600); + assert_eq!(zone.offset_at(2_000_000_000), 0); + assert_eq!(zone.offset_for_local(1_000_000_000), 3600); + } + + #[test] + fn rejects_empty_types_invalid_indices_and_nonascending_transitions() { + for bytes in [ + version_one(&[], &[]), + version_one(&[(0, 1)], &[(0, false)]), + version_one(&[(3600, 0), (0, 0)], &[(0, false)]), + version_one(&[(0, 0), (0, 0)], &[(0, false)]), + ] { + assert!(TimeZone::parse(&bytes).is_none()); + } + let bytes = version_one(&[(0, 0)], &[(0, false)]); + // Every prefix missing transition/type data is incomplete. + for cut in 0..bytes.len() - 4 { + assert!(TimeZone::parse(&bytes[..cut]).is_none(), "cut {cut}"); + } + let mut foreign = bytes; + foreign[..4].copy_from_slice(b"NOPE"); + assert!(TimeZone::parse(&foreign).is_none()); + } + + #[test] + fn rejects_truncated_designations_and_other_declared_block_data() { + let bytes = version_one(&[(0, 0)], &[(0, false)]); + for cut in bytes.len() - 4..bytes.len() { + assert!(TimeZone::parse(&bytes[..cut]).is_none(), "cut {cut}"); + } + // Counts for leap records and standard/UT indicators also contribute + // to the block extent, even though conversion does not interpret them. + for count_at in [20, 24, 28] { + let mut incomplete = bytes.clone(); + incomplete[count_at..count_at + 4].copy_from_slice(&1u32.to_be_bytes()); + assert!( + TimeZone::parse(&incomplete).is_none(), + "count at {count_at}" + ); + } + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn declared_block_sizes_reject_overflow_on_native_32_bit_hosts() { + use super::{Header, TimeWidth}; + let empty = Header { + version: 0, + isutcnt: 0, + isstdcnt: 0, + leapcnt: 0, + timecnt: 0, + typecnt: 0, + charcnt: 0, + }; + let wire_maximum = u32::MAX as usize; + for header in [ + Header { + timecnt: wire_maximum, + ..empty + }, + Header { + typecnt: wire_maximum, + ..empty + }, + Header { + leapcnt: wire_maximum, + ..empty + }, + Header { + isstdcnt: wire_maximum, + isutcnt: 1, + ..empty + }, + ] { + assert!(header.data_len(TimeWidth::Bits32).is_none()); + assert!(header.data_len(TimeWidth::Bits64).is_none()); + } + let header = Header { + charcnt: wire_maximum, + ..empty + }; + assert_eq!(header.data_len(TimeWidth::Bits32), Some(wire_maximum)); + assert!(header.data_end(TimeWidth::Bits32).is_none()); + } + + #[test] + fn zone_paths_honor_colons_and_directories_without_parent_traversal() { + use std::ffi::OsStr; + use std::path::PathBuf; + assert_eq!( + super::zone_path(None, None), + Some(PathBuf::from("/etc/localtime")) + ); + for name in [ + "", + ":", + "Europe//London", + "Europe/../London", + "./Europe/London", + "Europe/./London", + "Europe/", + ] { + assert!( + super::zone_path(Some(OsStr::new(name)), None).is_none(), + "{name}" + ); + } + for name in ["Europe/London", ":Europe/London"] { + assert_eq!( + super::zone_path(Some(OsStr::new(name)), None), + Some(PathBuf::from("/usr/share/zoneinfo/Europe/London")), + ); + assert_eq!( + super::zone_path(Some(OsStr::new(name)), Some(OsStr::new("custom-zones"))), + Some(PathBuf::from("custom-zones/Europe/London")), + ); + } + let absolute = std::env::current_dir().unwrap().join("chosen-zone"); + for name in [ + absolute.as_os_str().to_owned(), + format!(":{}", absolute.display()).into(), + ] { + assert_eq!(super::zone_path(Some(&name), None), Some(absolute.clone())); + } + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + assert!(super::zone_path(Some(OsStr::from_bytes(&[0xff])), None).is_none()); + } + } + + #[test] + fn daylight_only_gap_uses_the_offset_at_the_reading_when_no_standard_type_exists() { + let bytes = version_one(&[(1_000_000_000, 1)], &[(3600, true), (7200, true)]); + let zone = TimeZone::parse(&bytes).unwrap(); + assert_eq!(zone.offset_for_local(1_000_005_000), 7200); + } + + #[test] + fn repeated_reading_retains_first_candidate_when_daylight_status_is_equal() { + // A political offset change can repeat a reading without changing + // daylight status. Both instants are valid; preserve the first match. + for daylight in [false, true] { + let bytes = version_one(&[(1_000_000_000, 1)], &[(7200, daylight), (3600, daylight)]); + let zone = TimeZone::parse(&bytes).unwrap(); + let reading = 1_000_005_000; + assert_eq!(zone.offset_at(reading - 7200), 7200); + assert_eq!(zone.offset_at(reading - 3600), 3600); + assert_eq!(zone.offset_for_local(reading), 7200); + } + } + + /// A damaged or foreign file must not panic or be believed. + #[test] + fn rejects_files_that_are_not_a_usable_zone() { + assert!(TimeZone::parse(b"").is_none()); + assert!(TimeZone::parse(b"not a tzif file at all").is_none()); + assert!(TimeZone::parse(&LONDON[..20]).is_none(), "truncated header"); + assert!(TimeZone::parse(&LONDON[..200]).is_none(), "truncated body"); + } +} diff --git a/arca-rar/vendor/rars/src/version.rs b/arca-rar/vendor/rars/src/version.rs new file mode 100644 index 00000000..504fbcad --- /dev/null +++ b/arca-rar/vendor/rars/src/version.rs @@ -0,0 +1,139 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ArchiveFamily { + Rar13, + Rar15To40, + Rar50Plus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ArchiveVersion { + Rar13, + Rar14, + Rar15, + Rar20, + Rar29, + Rar30, + Rar40, + Rar50, + Rar70, +} + +impl ArchiveVersion { + pub const fn family(self) -> ArchiveFamily { + match self { + Self::Rar13 | Self::Rar14 => ArchiveFamily::Rar13, + Self::Rar15 | Self::Rar20 | Self::Rar29 | Self::Rar30 | Self::Rar40 => { + ArchiveFamily::Rar15To40 + } + Self::Rar50 | Self::Rar70 => ArchiveFamily::Rar50Plus, + } + } + + pub const fn is_rar13_family(self) -> bool { + matches!(self, Self::Rar13 | Self::Rar14) + } + + /// Every version, so a caller can ask what each one supports without + /// keeping its own list that drifts. + pub const ALL: [Self; 9] = [ + Self::Rar13, + Self::Rar14, + Self::Rar15, + Self::Rar20, + Self::Rar29, + Self::Rar30, + Self::Rar40, + Self::Rar50, + Self::Rar70, + ]; + + /// The inverse of [`as_str`](Self::as_str), for callers that take a format + /// name from outside the process. Separators and case are ignored, and the + /// short spellings `rar4`, `rar5` and `rar7` name the newest version in + /// their family, so `rar-5.0` and `RAR5` both reach [`Self::Rar50`]. + pub fn from_name(name: &str) -> Option { + match name + .to_ascii_lowercase() + .replace(['_', '-', '.', ' '], "") + .as_str() + { + "rar13" => Some(Self::Rar13), + "rar14" => Some(Self::Rar14), + "rar15" => Some(Self::Rar15), + "rar20" => Some(Self::Rar20), + "rar29" => Some(Self::Rar29), + "rar30" => Some(Self::Rar30), + "rar40" | "rar4" => Some(Self::Rar40), + "rar50" | "rar5" => Some(Self::Rar50), + "rar70" | "rar7" => Some(Self::Rar70), + _ => None, + } + } + + /// The spelling used by the `--format` argument, and by every message a + /// user reads. It is the only spelling they can type back into the tool. + pub const fn as_str(self) -> &'static str { + match self { + Self::Rar13 => "rar13", + Self::Rar14 => "rar14", + Self::Rar15 => "rar15", + Self::Rar20 => "rar20", + Self::Rar29 => "rar29", + Self::Rar30 => "rar30", + Self::Rar40 => "rar40", + Self::Rar50 => "rar50", + Self::Rar70 => "rar70", + } + } +} + +impl std::fmt::Display for ArchiveVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_versions_to_archive_families() { + let cases = [ + (ArchiveVersion::Rar13, ArchiveFamily::Rar13), + (ArchiveVersion::Rar14, ArchiveFamily::Rar13), + (ArchiveVersion::Rar15, ArchiveFamily::Rar15To40), + (ArchiveVersion::Rar20, ArchiveFamily::Rar15To40), + (ArchiveVersion::Rar29, ArchiveFamily::Rar15To40), + (ArchiveVersion::Rar30, ArchiveFamily::Rar15To40), + (ArchiveVersion::Rar40, ArchiveFamily::Rar15To40), + (ArchiveVersion::Rar50, ArchiveFamily::Rar50Plus), + (ArchiveVersion::Rar70, ArchiveFamily::Rar50Plus), + ]; + + for (version, family) in cases { + assert_eq!(version.family(), family); + } + } + + #[test] + fn identifies_rar13_family_versions() { + let cases = [ + (ArchiveVersion::Rar13, true), + (ArchiveVersion::Rar14, true), + (ArchiveVersion::Rar15, false), + (ArchiveVersion::Rar20, false), + (ArchiveVersion::Rar29, false), + (ArchiveVersion::Rar30, false), + (ArchiveVersion::Rar40, false), + (ArchiveVersion::Rar50, false), + (ArchiveVersion::Rar70, false), + ]; + + for (version, expected) in cases { + assert_eq!(version.is_rar13_family(), expected); + } + } +} diff --git a/arca-rar/vendor/rars/src/volume_extract.rs b/arca-rar/vendor/rars/src/volume_extract.rs new file mode 100644 index 00000000..cb91f839 --- /dev/null +++ b/arca-rar/vendor/rars/src/volume_extract.rs @@ -0,0 +1,183 @@ +use crate::codec::workspace::{Allowance, Budget, Buffer}; +use std::io::{Read, Result}; + +pub(crate) struct SplitVolumeState

{ + pending: Option

, +} + +impl

SplitVolumeState

{ + pub(crate) fn new() -> Self { + Self { pending: None } + } + + pub(crate) fn advance( + &mut self, + split_before: bool, + split_after: bool, + ) -> SplitVolumeStep<'_, P> { + match (self.pending.is_some(), split_before, split_after) { + (false, false, false) => SplitVolumeStep::Regular, + (false, false, true) => SplitVolumeStep::Start, + // The match arm's first field proves a pending split exists. + (true, true, true) => { + SplitVolumeStep::Continue(self.pending.as_mut().expect("pending split")) + } + // The match arm's first field proves a pending split exists. + (true, true, false) => { + SplitVolumeStep::Finish(self.pending.take().expect("pending split")) + } + // Error states leave pending untouched; callers currently return + // the error immediately rather than attempting recovery. + (false, true, _) => SplitVolumeStep::MissingFirst, + (true, false, _) => SplitVolumeStep::Interrupted, + } + } + + pub(crate) fn begin(&mut self, pending: P) { + debug_assert!(self.pending.is_none()); + self.pending = Some(pending); + } + + pub(crate) fn is_pending(&self) -> bool { + self.pending.is_some() + } +} + +pub(crate) enum SplitVolumeStep<'a, P> { + Regular, + Start, + Continue(&'a mut P), + Finish(P), + MissingFirst, + Interrupted, +} + +pub(crate) struct ChainedReader { + readers: Buffer, + index: usize, +} + +impl ChainedReader { + pub(crate) fn with_readers(readers: Buffer) -> Self { + Self { readers, index: 0 } + } +} +#[cfg(test)] +impl ChainedReader { + pub(crate) fn new(readers: Vec) -> Self { + Self { + readers: Buffer::from_vec(readers), + index: 0, + } + } +} + +impl Read for ChainedReader { + fn read(&mut self, out: &mut [u8]) -> Result { + if out.is_empty() { + return Ok(0); + } + while let Some(reader) = self.readers.get_mut(self.index) { + let read = reader.read(out)?; + if read != 0 { + return Ok(read); + } + self.index += 1; + } + Ok(0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn split_volume_state_reports_regular_member_without_pending_split() { + let mut state = SplitVolumeState::::new(); + + assert!(matches!( + state.advance(false, false), + SplitVolumeStep::Regular + )); + assert!(!state.is_pending()); + } + + #[test] + fn split_volume_state_tracks_start_continue_and_finish() { + let mut state = SplitVolumeState::new(); + + assert!(matches!(state.advance(false, true), SplitVolumeStep::Start)); + state.begin(vec![1]); + assert!(state.is_pending()); + + match state.advance(true, true) { + SplitVolumeStep::Continue(parts) => parts.push(2), + _ => panic!("expected split continuation"), + } + assert!(state.is_pending()); + + match state.advance(true, false) { + SplitVolumeStep::Finish(parts) => assert_eq!(parts, vec![1, 2]), + _ => panic!("expected split finish"), + } + assert!(!state.is_pending()); + } + + #[test] + fn split_volume_state_reports_orphan_continuation() { + let mut state = SplitVolumeState::::new(); + + assert!(matches!( + state.advance(true, false), + SplitVolumeStep::MissingFirst + )); + assert!(matches!( + state.advance(true, true), + SplitVolumeStep::MissingFirst + )); + assert!(!state.is_pending()); + } + + #[test] + fn split_volume_state_reports_regular_entry_interrupting_pending_split() { + let mut state = SplitVolumeState::new(); + assert!(matches!(state.advance(false, true), SplitVolumeStep::Start)); + state.begin(7u8); + + assert!(matches!( + state.advance(false, false), + SplitVolumeStep::Interrupted + )); + assert!(state.is_pending()); + } + + #[test] + fn chained_reader_empty_reads_do_not_consume_fragments() { + let mut reader = ChainedReader::new(vec![Cursor::new(b"one"), Cursor::new(b"two")]); + assert_eq!(reader.read(&mut []).unwrap(), 0); + let mut byte = [0]; + reader.read_exact(&mut byte).unwrap(); + assert_eq!(&byte, b"o"); + assert_eq!(reader.read(&mut []).unwrap(), 0); + let mut rest = Vec::new(); + reader.read_to_end(&mut rest).unwrap(); + assert_eq!(rest, b"netwo"); + } + + #[test] + fn chained_reader_reads_all_fragments_in_order() { + let readers: Vec> = vec![ + Box::new(Cursor::new(b"one".to_vec())), + Box::new(Cursor::new(Vec::new())), + Box::new(Cursor::new(b"two".to_vec())), + ]; + let mut reader = ChainedReader::new(readers); + let mut out = Vec::new(); + + reader.read_to_end(&mut out).unwrap(); + + assert_eq!(out, b"onetwo"); + } +} diff --git a/arca-rar/vendor/rars/src/write_plan.rs b/arca-rar/vendor/rars/src/write_plan.rs new file mode 100644 index 00000000..6d4bc9ef --- /dev/null +++ b/arca-rar/vendor/rars/src/write_plan.rs @@ -0,0 +1,446 @@ +//! What each format's writer can be asked for, and one place that says no. +//! +//! Every writer used to answer this for itself, in about thirty functions that +//! nearly all ended the same way: build the set of features this shape of +//! archive is allowed, compare it with what the caller asked for, and on a +//! mismatch return a fixed string that never said which option was the problem. +//! A caller who asked for two things and could only have one was told "writer +//! feature is not supported" and left to work out which. +//! +//! So capability is data here, keyed by format and by the shape of the archive +//! being built, and a refusal names the option, the format, and why. + +use crate::error::{Error, Result}; +use crate::features::{Feature, FeatureSet}; +use crate::filter::FilterPolicy; +use crate::version::{ArchiveFamily, ArchiveVersion}; + +/// How the members of an archive are coded. +/// +/// The three arms are the three questions a writer has to answer before it +/// touches a member, and they used to be three entry points per format that +/// differed in a handful of lines each. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[non_exhaustive] +pub enum MemberCoding { + /// Store every member as it is. + Stored, + /// Compress, letting the writer decide how. The RAR 2.9 family looks for a + /// filter unless PPMd was asked for; RAR 1.3 to 2.0 have none to look for. + #[default] + Compressed, + /// Compress under a filter policy the caller named. RAR 2.9 and later only. + Filtered(FilterPolicy), +} + +impl MemberCoding { + pub fn compresses(&self) -> bool { + !matches!(self, Self::Stored) + } + + pub fn is_filtered(&self) -> bool { + matches!(self, Self::Filtered(_)) + } + + /// The shape of archive this coding produces, so validation and the + /// capability table are asked the same question the writer will answer. + pub fn shape(&self) -> PlanShape { + PlanShape::new() + .compressed(self.compresses()) + .filtered(self.is_filtered()) + } +} + +/// What kind of archive is being built. Capability answers depend on this as +/// well as on the format: RAR 5 will write a file comment into a stored archive +/// but not a compressed one, for instance. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub struct PlanShape { + pub compressed: bool, + pub volumes: bool, + pub filtered: bool, +} + +impl PlanShape { + pub const fn new() -> Self { + Self { + compressed: false, + volumes: false, + filtered: false, + } + } + + pub const fn compressed(mut self, compressed: bool) -> Self { + self.compressed = compressed; + self + } + + pub const fn volumes(mut self, volumes: bool) -> Self { + self.volumes = volumes; + self + } + + pub const fn filtered(mut self, filtered: bool) -> Self { + self.filtered = filtered; + self + } + + /// Every combination, for exhaustively checking capability answers. + pub fn all() -> Vec { + let mut shapes = Vec::with_capacity(8); + for compressed in [false, true] { + for volumes in [false, true] { + for filtered in [false, true] { + shapes.push(Self { + compressed, + volumes, + filtered, + }); + } + } + } + shapes + } +} + +pub use crate::writer_option::WriterOption; + +/// The features this format can honour for an archive of this shape. +/// +/// Deliberately blind to what the caller asked for. The check this replaced +/// built its "allowed" set partly out of the request itself, which in several +/// places decayed into assigning a flag to itself, so the flag was accepted +/// whatever it said. +pub fn supported_features(target: ArchiveVersion, shape: PlanShape) -> FeatureSet { + FeatureSet { + solid: supports(target, WriterOption::Feature(Feature::Solid), shape), + header_encryption: supports( + target, + WriterOption::Feature(Feature::HeaderEncryption), + shape, + ), + quick_open: supports(target, WriterOption::Feature(Feature::QuickOpen), shape), + } +} + +/// Whether this format's writer can honour `option` for an archive of this +/// shape. +pub fn supports(target: ArchiveVersion, option: WriterOption, shape: PlanShape) -> bool { + if !cfg!(feature = "encryption") + && matches!( + option, + WriterOption::Password | WriterOption::Feature(Feature::HeaderEncryption) + ) + { + return false; + } + + let family = target.family(); + match option { + // Solid means compressing members against each other, so there has to + // be compression for it to mean anything. + WriterOption::Feature(Feature::Solid) => shape.compressed, + WriterOption::Feature(Feature::HeaderEncryption) => matches!( + target, + ArchiveVersion::Rar30 + | ArchiveVersion::Rar40 + | ArchiveVersion::Rar50 + | ArchiveVersion::Rar70 + ), + // Quick-open indexes headers, so it cannot cover encrypted ones, and + // no volume writer emits the index. + WriterOption::Feature(Feature::QuickOpen) => { + family == ArchiveFamily::Rar50Plus && !shape.volumes + } + WriterOption::CompressionLevel => true, + // PPMd arrived with RAR 2.9 and left again with RAR 5. The legacy + // volume writer takes a packed payload rather than a method, so a + // split set is always LZ. + WriterOption::CompressionMethod => { + matches!( + target, + ArchiveVersion::Rar29 | ArchiveVersion::Rar30 | ArchiveVersion::Rar40 + ) && !shape.volumes + } + WriterOption::DictionarySize => !family_is(family, ArchiveFamily::Rar13), + // RAR 1.3 and 1.5 have no filters at all; RAR 2.0 predates the VM. The + // legacy volume writer takes a payload rather than a policy, so a + // filter cannot be asked for across a split set. + WriterOption::Filter => { + !matches!( + target, + ArchiveVersion::Rar13 + | ArchiveVersion::Rar14 + | ArchiveVersion::Rar15 + | ArchiveVersion::Rar20 + ) && (family == ArchiveFamily::Rar50Plus || !shape.volumes) + } + WriterOption::RecoveryRecord => { + cfg!(feature = "recovery") && family == ArchiveFamily::Rar50Plus + } + WriterOption::VolumeSize => true, + // No volume writer carries a comment: the legacy one emits the split + // member and nothing else, and the RAR 5 one refuses comments outright + // because a volume set has no single place to put one. + WriterOption::ArchiveComment => !shape.volumes, + // Embedded legacy file comments and RAR5 service comments are supported + // for single archives; volume writers cannot retain their association. + WriterOption::FileComment => !shape.volumes, + WriterOption::ArchiveMetadata => family == ArchiveFamily::Rar50Plus && !shape.volumes, + WriterOption::Password => true, + // Every writer compresses to a budget now: RAR 5 sizes its blocks by + // it, and the older ones use it to decide how many members to have in + // flight. A legacy volume set is the exception, still built in memory. + WriterOption::MemoryLimit => family == ArchiveFamily::Rar50Plus || !shape.volumes, + // Only the RAR 5 engine spills to disk. + WriterOption::TempDir => family == ArchiveFamily::Rar50Plus, + } +} + +const fn family_is(family: ArchiveFamily, other: ArchiveFamily) -> bool { + matches!( + (family, other), + (ArchiveFamily::Rar13, ArchiveFamily::Rar13) + | (ArchiveFamily::Rar15To40, ArchiveFamily::Rar15To40) + | (ArchiveFamily::Rar50Plus, ArchiveFamily::Rar50Plus) + ) +} + +/// Every format whose writer can honour `option` for an archive of this shape. +/// +/// Computed rather than stored, so a writer that gains a feature does not leave +/// a stale list behind in an error message. +pub fn formats_supporting(option: WriterOption, shape: PlanShape) -> Vec { + ArchiveVersion::ALL + .into_iter() + .filter(|&target| supports(target, option, shape)) + .collect() +} + +/// Refuses the first feature this format cannot honour. +pub fn validate_features( + target: ArchiveVersion, + asked: FeatureSet, + shape: PlanShape, +) -> Result<()> { + if asked.header_encryption { + crate::crypto::require_encryption()?; + } + + match asked.first_unsupported(supported_features(target, shape)) { + Some(feature) => Err(Error::UnsupportedWriterOption { + target, + option: WriterOption::Feature(feature), + because: None, + }), + None => Ok(()), + } +} + +/// Refuses a compression level outside the range every format shares. +pub fn validate_compression_level(target: ArchiveVersion, level: Option) -> Result<()> { + match level { + None | Some(0..=5) => Ok(()), + Some(_) => Err(Error::UnsupportedWriterOption { + target, + option: WriterOption::CompressionLevel, + because: Some("levels run from 0 to 5"), + }), + } +} + +/// Refuses an option this format cannot honour for this shape of archive. +pub fn validate_option( + target: ArchiveVersion, + option: WriterOption, + shape: PlanShape, +) -> Result<()> { + if !cfg!(feature = "encryption") + && matches!( + option, + WriterOption::Password | WriterOption::Feature(Feature::HeaderEncryption) + ) + { + return Err(Error::FeatureDisabled { + feature: "encryption", + }); + } + if !cfg!(feature = "recovery") && matches!(option, WriterOption::RecoveryRecord) { + return Err(Error::FeatureDisabled { + feature: "recovery", + }); + } + if supports(target, option, shape) { + Ok(()) + } else { + Err(Error::UnsupportedWriterOption { + target, + option, + because: None, + }) + } +} + +/// Whether a compressed member should be written as stored bytes instead. +/// +/// Compression that does not shrink a member costs the reader time for nothing. +/// The formats disagree about when that trade is worth making, so what differs +/// between them is parameters here rather than three separate rules. +#[derive(Debug, Clone, Copy)] +pub(crate) struct StoreFallback { + /// Members below this size stay compressed. Where a stored member costs + /// enough header of its own, swapping a small one can lose more than the + /// compression was wasting. + pub(crate) min_size: usize, + /// Whether a solid member may be swapped. RAR 5 runs one dictionary across + /// every member and cannot go back on what it has already fed in; the + /// RAR 1.5 family rebuilds its encoder at that point instead. + pub(crate) allow_solid: bool, + /// A filter the caller named is not discarded because the result did not + /// shrink. They asked for it. + pub(crate) filter_requested: bool, +} + +impl StoreFallback { + pub(crate) const fn new() -> Self { + Self { + min_size: 0, + allow_solid: false, + filter_requested: false, + } + } + + pub(crate) const fn min_size(mut self, size: usize) -> Self { + self.min_size = size; + self + } + + pub(crate) const fn allow_solid(mut self, allow: bool) -> Self { + self.allow_solid = allow; + self + } + + pub(crate) const fn filter_requested(mut self, requested: bool) -> Self { + self.filter_requested = requested; + self + } + + pub(crate) fn applies(&self, solid: bool, unpacked: usize, packed: usize) -> bool { + if packed < unpacked || self.filter_requested { + return false; + } + if solid && !self.allow_solid { + return false; + } + unpacked >= self.min_size + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The capability table and the validator have to agree everywhere, or a + /// caller can be told an option is available and then refused it. + #[test] + fn asking_and_validating_agree_for_every_combination() { + for target in ArchiveVersion::ALL { + for option in WriterOption::ALL { + for shape in PlanShape::all() { + let supported = supports(target, option, shape); + let validated = validate_option(target, option, shape); + assert_eq!( + supported, + validated.is_ok(), + "{target} {option:?} {shape:?}: supports={supported} validate={validated:?}" + ); + if let Err(Error::UnsupportedWriterOption { option: named, .. }) = validated { + assert_eq!(named, option, "the refusal named the wrong option"); + } + } + } + } + } + + /// A refusal is only useful if it can point somewhere, so every option has + /// to be available on at least one format. + #[test] + fn every_option_is_supported_by_some_format() { + for option in WriterOption::ALL { + assert!(!option.name().is_empty()); + let disabled = match option { + WriterOption::Password | WriterOption::Feature(Feature::HeaderEncryption) => { + !cfg!(feature = "encryption") + } + WriterOption::RecoveryRecord => !cfg!(feature = "recovery"), + _ => false, + }; + assert_eq!( + PlanShape::all() + .into_iter() + .any(|shape| !formats_supporting(option, shape).is_empty()), + !disabled, + "{option:?} must have a supporting format exactly when its capability is enabled" + ); + } + } + + #[test] + fn feature_capability_matches_the_per_option_answer() { + for target in ArchiveVersion::ALL { + for shape in PlanShape::all() { + let features = supported_features(target, shape); + for feature in Feature::ALL { + assert_eq!( + feature_of(features, feature), + supports(target, WriterOption::Feature(feature), shape), + "{target} {feature:?} {shape:?}" + ); + } + } + } + } + + fn feature_of(set: FeatureSet, feature: Feature) -> bool { + match feature { + Feature::Solid => set.solid, + Feature::HeaderEncryption => set.header_encryption, + Feature::QuickOpen => set.quick_open, + } + } + + #[test] + fn a_member_is_only_stored_when_compressing_it_gained_nothing() { + let plain = StoreFallback::new(); + assert!(plain.applies(false, 100, 100)); + assert!(plain.applies(false, 100, 120)); + assert!(!plain.applies(false, 100, 99)); + + // A named filter survives a result that did not shrink. + assert!(!plain.filter_requested(true).applies(false, 100, 200)); + + // Solid is refused unless the format can rebuild its chain. + assert!(!plain.applies(true, 100, 100)); + assert!(plain.allow_solid(true).applies(true, 100, 100)); + + // The floor keeps small members compressed. + let floored = plain.min_size(1024); + assert!(!floored.applies(false, 1023, 1023)); + assert!(floored.applies(false, 1024, 1024)); + } + + #[test] + fn a_level_above_five_is_refused_by_name() { + let refused = validate_compression_level(ArchiveVersion::Rar50, Some(9)); + assert!(matches!( + refused, + Err(Error::UnsupportedWriterOption { + option: WriterOption::CompressionLevel, + .. + }) + )); + assert!(validate_compression_level(ArchiveVersion::Rar50, Some(5)).is_ok()); + assert!(validate_compression_level(ArchiveVersion::Rar50, None).is_ok()); + } +} diff --git a/arca-rar/vendor/rars/src/write_progress.rs b/arca-rar/vendor/rars/src/write_progress.rs new file mode 100644 index 00000000..8a6169eb --- /dev/null +++ b/arca-rar/vendor/rars/src/write_progress.rs @@ -0,0 +1,304 @@ +pub(crate) use crate::progress::ProgressReporter; +use crate::progress::{WriteOperation, WriteProgress, WriteProgressEvent}; + +pub(crate) struct WorkTracker<'a> { + progress: Option>, + operation: WriteOperation, + total: u64, + state: Mutex, +} + +#[derive(Default)] +struct WorkState { + completed: u64, +} + +impl<'a> WorkTracker<'a> { + pub(crate) fn reporter(&self) -> Option> { + self.progress + } + + pub(crate) fn check(&self) -> crate::Result<()> { + check_cancelled(self.progress) + } + pub(crate) fn new( + progress: Option>, + operation: WriteOperation, + total: u64, + ) -> Self { + Self { + progress, + operation, + total, + state: Mutex::new(WorkState::default()), + } + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.progress.is_some_and(ProgressReporter::is_cancelled) + } + + pub(crate) fn advance(&self, amount: u64) -> bool { + let Some(progress) = self.progress else { + return true; + }; + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + state.completed = state.completed.saturating_add(amount).min(self.total); + progress.report(WriteProgressEvent::Advanced { + operation: self.operation, + completed_bytes: state.completed, + total_bytes: self.total, + pass: 1, + }); + !progress.is_cancelled() + } + + pub(crate) fn finish(&self) -> bool { + let completed = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .completed; + self.advance(self.total.saturating_sub(completed)) + } + + pub(crate) fn entry_started( + &self, + index: usize, + total_entries: usize, + name: &[u8], + input_bytes: u64, + ) { + if let Some(progress) = self.progress { + progress.report(WriteProgressEvent::EntryStarted { + operation: self.operation, + index, + total_entries, + name, + input_bytes, + }); + } + } + + pub(crate) fn entry_finished( + &self, + index: usize, + total_entries: usize, + name: &[u8], + input_bytes: u64, + ) { + if let Some(progress) = self.progress { + progress.report(WriteProgressEvent::EntryFinished { + operation: self.operation, + index, + total_entries, + name, + input_bytes, + }); + } + } +} +use std::sync::Mutex; + +/// Combine cancellation policy with optional presentation for one write. +pub(crate) struct ResourceProgress<'a> { + resources: &'a crate::WriterResources, + inner: Option>, + cancelled: std::sync::atomic::AtomicBool, +} +impl<'a> ResourceProgress<'a> { + pub(crate) fn new( + resources: &'a crate::WriterResources, + inner: Option>, + ) -> Self { + Self { + resources, + inner, + cancelled: std::sync::atomic::AtomicBool::new(false), + } + } +} +impl WriteProgress for ResourceProgress<'_> { + fn report(&self, event: WriteProgressEvent<'_>) { + if let Some(inner) = self.inner { + inner.report(event); + } + } + fn is_cancelled(&self) -> bool { + use std::sync::atomic::Ordering; + if self.cancelled.load(Ordering::Relaxed) + || self.resources.is_cancelled() + || self.inner.is_some_and(ProgressReporter::is_cancelled) + { + self.cancelled.store(true, Ordering::Relaxed); + true + } else { + false + } + } +} + +pub(crate) fn check_cancelled(progress: Option>) -> crate::Result<()> { + if progress.is_some_and(ProgressReporter::is_cancelled) { + Err(crate::Error::Cancelled) + } else { + Ok(()) + } +} + +/// Preserve typed cancellation through APIs which can only return I/O errors. +/// Never use ErrorKind::Interrupted: write_all/read_exact would retry forever. +pub(crate) struct CancellableIo<'a, T> { + pub(crate) inner: T, + pub(crate) progress: Option>, +} +impl CancellableIo<'_, T> { + fn check(&self) -> std::io::Result<()> { + check_cancelled(self.progress).map_err(std::io::Error::other) + } +} +impl std::io::Read for CancellableIo<'_, T> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.check()?; + let len = if self.progress.is_some() { + buffer.len().min(64 * 1024) + } else { + buffer.len() + }; + self.inner.read(&mut buffer[..len]) + } +} +impl std::io::Write for CancellableIo<'_, T> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.check()?; + let len = if self.progress.is_some() { + buffer.len().min(64 * 1024) + } else { + buffer.len() + }; + self.inner.write(&buffer[..len]) + } + fn flush(&mut self) -> std::io::Result<()> { + self.check()?; + self.inner.flush() + } +} +impl std::io::Seek for CancellableIo<'_, T> { + fn seek(&mut self, from: std::io::SeekFrom) -> std::io::Result { + self.check()?; + self.inner.seek(from) + } +} + +#[cfg(test)] +#[test] +fn callback_panic_keeps_peer_progress_and_completion_usable() { + use std::sync::atomic::{AtomicBool, Ordering}; + let panic_once = AtomicBool::new(true); + let reported = Mutex::new(Vec::new()); + let callback = |event: WriteProgressEvent<'_>| { + if let WriteProgressEvent::Advanced { + completed_bytes, .. + } = event + { + if panic_once.swap(false, Ordering::Relaxed) { + panic!("injected progress callback panic"); + } + reported.lock().unwrap().push(completed_bytes); + } + }; + let work = WorkTracker::new( + Some(ProgressReporter(&callback)), + WriteOperation::Compression, + 10, + ); + std::thread::scope(|scope| { + let worker = scope.spawn(|| work.advance(3)); + assert!(worker.join().is_err()); + let peer = scope.spawn(|| work.advance(2)); + assert!(peer.join().unwrap()); + }); + assert!(work.finish()); + assert_eq!(*reported.lock().unwrap(), [5, 10]); +} + +/// Count accepted output without buffering an archive or guessing its final size. +pub(crate) struct EmissionWriter<'a> { + output: &'a mut dyn std::io::Write, + progress: Option>, + completed: u64, + reported: u64, + started: bool, +} +impl<'a> EmissionWriter<'a> { + pub(crate) fn new( + output: &'a mut dyn std::io::Write, + progress: Option>, + ) -> Self { + Self { + output, + progress, + completed: 0, + reported: 0, + started: false, + } + } + fn start(&mut self) -> crate::Result<()> { + check_cancelled(self.progress)?; + if !self.started { + self.started = true; + if let Some(progress) = self.progress { + progress.report(WriteProgressEvent::OperationStarted { + operation: WriteOperation::Emission, + total_bytes: None, + total_entries: None, + pass: 1, + }); + } + } + check_cancelled(self.progress) + } + fn advance(&mut self) -> crate::Result<()> { + if self.completed != self.reported { + self.reported = self.completed; + if let Some(progress) = self.progress { + progress.report(WriteProgressEvent::BytesWritten { + completed_bytes: self.completed, + }); + } + } + check_cancelled(self.progress) + } + pub(crate) fn finish(&mut self) -> crate::Result<()> { + self.start()?; + self.advance()?; + if let Some(progress) = self.progress { + progress.report(WriteProgressEvent::OperationFinished { + operation: WriteOperation::Emission, + total_bytes: Some(self.completed), + total_entries: None, + pass: 1, + }); + } + check_cancelled(self.progress) + } +} +impl std::io::Write for EmissionWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.start().map_err(std::io::Error::other)?; + let written = self.output.write(&bytes[..bytes.len().min(64 * 1024)])?; + self.completed += written as u64; + if self.completed - self.reported >= 64 * 1024 { + self.advance().map_err(std::io::Error::other)?; + } + check_cancelled(self.progress).map_err(std::io::Error::other)?; + Ok(written) + } + fn flush(&mut self) -> std::io::Result<()> { + check_cancelled(self.progress).map_err(std::io::Error::other)?; + self.output.flush() + } +} diff --git a/arca-rar/vendor/rars/src/write_stream.rs b/arca-rar/vendor/rars/src/write_stream.rs new file mode 100644 index 00000000..0f77db45 --- /dev/null +++ b/arca-rar/vendor/rars/src/write_stream.rs @@ -0,0 +1,452 @@ +//! Reading and writing archive members one at a time. +//! +//! The RAR 1.3 to 4.x codecs compress a member as a unit, so those writers +//! cannot avoid having a member resident. What they can avoid is having *every* +//! member resident, which is what they used to do: the caller read all the +//! inputs, the writer built the whole archive in a `Vec`, and peak memory was a +//! multiple of the total input rather than of the largest member. + +use crate::error::{Error, Result}; +use crate::streaming::EntrySource; +use std::borrow::Cow; +use std::io::{Read, Write}; + +pub(crate) use crate::entropy::fill_entropy; + +/// Attribute a writer failure without blaming a member for cancellation. +pub(crate) fn member_error(error: Error, name: &[u8], operation: &'static str) -> Error { + if error.kind() == crate::ErrorKind::Cancelled || error.entry_context().is_some() { + error + } else { + error.at_entry(name.to_vec(), operation) + } +} + +/// How much of a member is read at a time when it is walked rather than held. +const WALK_CHUNK: usize = 256 * 1024; + +/// Where one member's bytes come from. +/// +/// A caller who already holds the whole input hands over a slice; one writing +/// from disk hands over a source that is opened when the member is coded and +/// closed again straight after. Everything downstream is the same either way. +pub(crate) enum MemberBytes<'a> { + Borrowed(&'a [u8]), + Source(&'a EntrySource), + /// Builder inputs retain their source-read context even when loading is + /// deferred into a writer's preparation phase. + NamedSource(&'a EntrySource, &'a [u8]), +} + +impl<'a> MemberBytes<'a> { + pub(crate) fn len(&self) -> Result { + let result = match self { + Self::Borrowed(data) => Ok(data.len() as u64), + Self::Source(source) | Self::NamedSource(source, _) => source.len(), + }; + result.map_err(|error| self.source_error(error)) + } + + fn source_error(&self, error: Error) -> Error { + match self { + Self::NamedSource(_, name) => member_error(error, name, "reading source"), + _ => error, + } + } + + /// The whole member, borrowed when the caller already had it. + #[cfg(test)] + pub(crate) fn load(&self) -> Result> { + self.load_with_progress(None) + } + + pub(crate) fn load_with_progress( + &self, + progress: Option>, + ) -> Result> { + crate::write_progress::check_cancelled(progress)?; + self.load_exact_with_progress(self.len()?, progress) + } + + /// Use the source size already captured for the header and workspace. + /// Requerying a changing path here could disagree with that header. Inline + /// bytes remain borrowed and immutable throughout the write. + pub(crate) fn load_exact_with_progress( + &self, + expected: u64, + progress: Option>, + ) -> Result> { + self.load_exact(expected, progress) + .map_err(|error| self.source_error(error)) + } + + fn load_exact( + &self, + expected: u64, + progress: Option>, + ) -> Result> { + crate::write_progress::check_cancelled(progress)?; + match self { + Self::Borrowed(data) => Ok(Cow::Borrowed(data)), + Self::Source(source) | Self::NamedSource(source, _) => { + let capacity = usize::try_from(expected).map_err(|_| { + Error::InvalidArgument("member is larger than this host can hold") + })?; + let mut data = Vec::with_capacity(capacity); + crate::write_progress::check_cancelled(progress)?; + let mut reader = crate::write_progress::CancellableIo { + inner: source.open()?, + progress, + }; + reader.by_ref().take(expected).read_to_end(&mut data)?; + crate::write_progress::check_cancelled(progress)?; + check_source_length( + &mut reader, + data.len() as u64, + expected, + "entry source size changed while compressing", + )?; + crate::write_progress::check_cancelled(progress)?; + Ok(Cow::Owned(data)) + } + } + } + + /// Walks the member in chunks without holding it, which is how a stored + /// one is checksummed on its way through. + #[cfg(test)] + pub(crate) fn walk(&self, visit: impl FnMut(&[u8])) -> Result<()> { + self.walk_with_progress(None, visit) + } + + #[cfg(test)] + fn walk_with_progress( + &self, + progress: Option>, + visit: impl FnMut(&[u8]), + ) -> Result<()> { + crate::write_progress::check_cancelled(progress)?; + self.walk_exact_with_progress(self.len()?, progress, visit) + } + + /// Walk against the same captured source size that the member header uses. + pub(crate) fn walk_exact_with_progress( + &self, + expected: u64, + progress: Option>, + visit: impl FnMut(&[u8]), + ) -> Result<()> { + self.walk_exact(expected, progress, visit) + .map_err(|error| self.source_error(error)) + } + + fn walk_exact( + &self, + expected: u64, + progress: Option>, + mut visit: impl FnMut(&[u8]), + ) -> Result<()> { + crate::write_progress::check_cancelled(progress)?; + match self { + Self::Borrowed(data) => { + for chunk in data.chunks(WALK_CHUNK) { + crate::write_progress::check_cancelled(progress)?; + visit(chunk); + } + } + Self::Source(source) | Self::NamedSource(source, _) => { + crate::write_progress::check_cancelled(progress)?; + let mut reader = crate::write_progress::CancellableIo { + inner: source.open()?, + progress, + }; + let mut limited = reader.by_ref().take(expected); + let mut observed = 0; + let mut buffer = vec![0u8; WALK_CHUNK]; + loop { + let read = limited.read(&mut buffer)?; + if read == 0 { + break; + } + observed += read as u64; + visit(&buffer[..read]); + } + crate::write_progress::check_cancelled(progress)?; + check_source_length( + &mut reader, + observed, + expected, + "entry source size changed while reading", + )?; + } + } + crate::write_progress::check_cancelled(progress)?; + Ok(()) + } + + /// The source behind this member, when there is one to copy from. + pub(crate) fn source(&self) -> Option<&'a EntrySource> { + match self { + Self::Borrowed(_) => None, + Self::Source(source) | Self::NamedSource(source, _) => Some(source), + } + } + + /// The member has been written and verified, so this write will not + /// reopen its session-owned staged payload. Public source factories keep + /// their usual reopenable behavior. + pub(crate) fn release(&self) { + if let Some(source) = self.source() { + source.release(); + } + } +} + +/// A member's bytes as they will appear in the archive. +pub(crate) enum MemberPayload<'a> { + /// Already in memory: everything compressed, and anything encrypted. + Packed(Vec), + /// Copied from the source as the archive is written, which keeps a stored + /// member off the heap however large it is. + Copied(&'a EntrySource), +} + +impl MemberPayload<'_> { + /// How many bytes this payload puts in the archive. A copied member is + /// stored verbatim, so its packed size is the size it came in at. + pub(crate) fn size(&self, unpacked_size: u64) -> u64 { + match self { + Self::Packed(packed) => packed.len() as u64, + Self::Copied(_) => unpacked_size, + } + } + + pub(crate) fn write_to(&self, output: &mut dyn Write, expected: u64) -> Result<()> { + match self { + Self::Packed(packed) => output.write_all(packed)?, + Self::Copied(source) => { + let mut reader = source.open()?; + let copied = std::io::copy(&mut reader.by_ref().take(expected), output)?; + check_source_length( + &mut *reader, + copied, + expected, + "entry source size changed while writing", + )?; + } + } + Ok(()) + } +} + +// Consume at most the advertised length plus one probe byte, including for a +// source that keeps growing. The extra byte is never passed to a codec or sink. +pub(crate) fn check_source_length( + reader: &mut dyn Read, + observed: u64, + expected: u64, + message: &'static str, +) -> Result<()> { + if observed != expected { + return Err(Error::SourceChanged(message)); + } + let mut probe = [0]; + loop { + match reader.read(&mut probe) { + Ok(0) => return Ok(()), + Ok(_) => return Err(Error::SourceChanged(message)), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error.into()), + } + } +} + +#[cfg(test)] +mod tests { + #[test] + fn captured_source_size_controls_loading_and_checksum_walks() { + struct Source; + impl crate::streaming::SourceFactory for Source { + fn len(&self) -> Result { + panic!("must use the size already captured for the member header") + } + fn open(&self) -> Result> { + Ok(Box::new(std::io::Cursor::new(b"abc".as_slice()))) + } + } + for expected in [0, 1, 3, 4] { + let source = EntrySource::from_factory(Source); + let member = MemberBytes::Source(&source); + let loaded = member.load_exact_with_progress(expected, None); + let mut observed = Vec::new(); + let walked = member.walk_exact_with_progress(expected, None, |chunk| { + observed.extend_from_slice(chunk) + }); + if expected == 3 { + assert_eq!(loaded.unwrap().as_ref(), b"abc"); + walked.unwrap(); + assert_eq!(observed, b"abc"); + } else { + assert_eq!(loaded.unwrap_err().kind(), crate::ErrorKind::SourceChanged); + assert_eq!(walked.unwrap_err().kind(), crate::ErrorKind::SourceChanged); + } + } + } + + #[test] + fn source_end_probe_retries_interruptions_and_reports_other_io_errors() { + struct Probe { + calls: usize, + fail: bool, + } + impl std::io::Read for Probe { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + self.calls += 1; + if self.calls == 1 { + Err(std::io::ErrorKind::Interrupted.into()) + } else if self.fail { + Err(std::io::ErrorKind::PermissionDenied.into()) + } else { + Ok(0) + } + } + } + let mut end = Probe { + calls: 0, + fail: false, + }; + super::check_source_length(&mut end, 7, 7, "changed").unwrap(); + assert_eq!(end.calls, 2); + let mut failed = Probe { + calls: 0, + fail: true, + }; + assert_eq!( + super::check_source_length(&mut failed, 7, 7, "changed") + .unwrap_err() + .kind(), + crate::ErrorKind::Io + ); + } + + #[test] + fn member_error_preserves_cancellation_and_existing_context() { + assert_eq!( + super::member_error(crate::Error::Cancelled, b"file", "writing"), + crate::Error::Cancelled + ); + let existing = + crate::Error::InvalidArgument("source failed").at_entry(b"file".to_vec(), "reading"); + let error = super::member_error(existing, b"file", "writing"); + assert_eq!(error.entry_context(), Some((b"file".as_slice(), "reading"))); + } + + use super::*; + + #[cfg(target_pointer_width = "32")] + #[test] + fn oversized_member_is_refused_before_opening_its_source() { + let length = u64::from(u32::MAX) + 1; + let source = EntrySource::from_opener(length, || { + panic!("an unrepresentable member must not open its source") + }); + let member = MemberBytes::Source(&source); + assert_eq!(member.len().unwrap(), length); + assert_eq!( + member.load().unwrap_err(), + Error::InvalidArgument("member is larger than this host can hold") + ); + } + + #[test] + fn changed_sources_are_bounded_and_rejected() { + use std::io::{Cursor, Seek, SeekFrom}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + struct Counted { + data: Cursor>, + read: Arc, + } + impl Read for Counted { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.data.read(buffer)?; + self.read.fetch_add(count, Ordering::Relaxed); + Ok(count) + } + } + impl Seek for Counted { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.data.seek(position) + } + } + + for expected in [0, 8] { + for actual in [0, 3, 8, 1024 * 1024] { + let read = Arc::new(AtomicUsize::new(0)); + let source = EntrySource::from_opener(expected, { + let read = Arc::clone(&read); + move || { + Ok(Box::new(Counted { + data: Cursor::new(vec![42; actual]), + read: Arc::clone(&read), + })) + } + }); + let bytes = MemberBytes::Source(&source); + let valid = actual as u64 == expected; + assert_eq!(bytes.load().is_ok(), valid); + assert!(read.swap(0, Ordering::Relaxed) as u64 <= expected + 1); + let mut visited = 0; + assert_eq!(bytes.walk(|chunk| visited += chunk.len()).is_ok(), valid); + assert!(visited as u64 <= expected); + assert!(read.swap(0, Ordering::Relaxed) as u64 <= expected + 1); + let mut output = Vec::new(); + assert_eq!( + MemberPayload::Copied(&source) + .write_to(&mut output, expected) + .is_ok(), + valid + ); + assert!(output.len() as u64 <= expected); + assert!(read.load(Ordering::Relaxed) as u64 <= expected + 1); + } + } + } + + #[test] + fn a_source_walks_in_chunks_and_loads_whole() { + let data: Vec = (0..WALK_CHUNK * 2 + 7).map(|index| index as u8).collect(); + let source = EntrySource::from_bytes(data.clone()); + let bytes = MemberBytes::Source(&source); + + assert_eq!(bytes.len().unwrap(), data.len() as u64); + assert_eq!(bytes.load().unwrap().as_ref(), data.as_slice()); + + let mut chunks = Vec::new(); + let mut seen = Vec::new(); + bytes + .walk(|chunk| { + chunks.push(chunk.len()); + seen.extend_from_slice(chunk); + }) + .unwrap(); + assert_eq!(seen, data); + assert!(chunks.len() > 1, "a large source should arrive in pieces"); + } + + #[test] + fn borrowed_bytes_are_never_copied() { + let data = b"already in memory".to_vec(); + let bytes = MemberBytes::Borrowed(&data); + assert!(matches!(bytes.load().unwrap(), Cow::Borrowed(_))); + assert!(bytes.source().is_none()); + let mut visited = Vec::new(); + bytes + .walk(|chunk| visited.extend_from_slice(chunk)) + .unwrap(); + assert_eq!(visited, data); + } +} diff --git a/arca-rar/vendor/rars/src/writer_option.rs b/arca-rar/vendor/rars/src/writer_option.rs new file mode 100644 index 00000000..cb226d57 --- /dev/null +++ b/arca-rar/vendor/rars/src/writer_option.rs @@ -0,0 +1,63 @@ +//! Writer option names retained for structured error diagnostics. + +use crate::features::Feature; + +/// Something a caller can ask a writer for, that a format may not be able to do. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum WriterOption { + Feature(Feature), + CompressionLevel, + /// An engine other than the format's default, which today means PPMd. + CompressionMethod, + DictionarySize, + Filter, + RecoveryRecord, + VolumeSize, + ArchiveComment, + FileComment, + ArchiveMetadata, + Password, + MemoryLimit, + TempDir, +} + +impl WriterOption { + pub const ALL: [Self; 15] = [ + Self::Feature(Feature::Solid), + Self::Feature(Feature::HeaderEncryption), + Self::Feature(Feature::QuickOpen), + Self::CompressionLevel, + Self::CompressionMethod, + Self::DictionarySize, + Self::Filter, + Self::RecoveryRecord, + Self::VolumeSize, + Self::ArchiveComment, + Self::FileComment, + Self::ArchiveMetadata, + Self::Password, + Self::MemoryLimit, + Self::TempDir, + ]; + + /// How this option is named in a library message. The command line + /// substitutes the flag the user actually typed. + pub const fn name(self) -> &'static str { + match self { + Self::Feature(feature) => feature.name(), + Self::CompressionLevel => "a compression level", + Self::CompressionMethod => "an alternative compression method", + Self::DictionarySize => "a dictionary size", + Self::Filter => "a data filter", + Self::RecoveryRecord => "a recovery record", + Self::VolumeSize => "splitting into volumes", + Self::ArchiveComment => "an archive comment", + Self::FileComment => "a per-file comment", + Self::ArchiveMetadata => "archive metadata", + Self::Password => "encryption", + Self::MemoryLimit => "a memory limit", + Self::TempDir => "a temporary directory", + } + } +} diff --git a/arca-rar/vendor/rars/src/x86_filter_scan.rs b/arca-rar/vendor/rars/src/x86_filter_scan.rs new file mode 100644 index 00000000..0075a95a --- /dev/null +++ b/arca-rar/vendor/rars/src/x86_filter_scan.rs @@ -0,0 +1,440 @@ +#[cfg(test)] +use crate::codec::workspace::Allowance; +use crate::codec::workspace::{Budget, Buffer}; +use crate::codec::Result; +use std::ops::Range; + +const AUTO_X86_CLUSTER_GAP: usize = 4096; +const AUTO_X86_TIGHT_CLUSTER_GAP: usize = 512; +const AUTO_X86_SPAN_CLUSTER_GAP: usize = 32768; +const AUTO_X86_RANGE_PADDING: usize = 16; +const AUTO_X86_MAX_RANGES: usize = 8; +const AUTO_X86_MAX_SPAN_RANGES: usize = 4; +const AUTO_X86_MIN_SPAN_OPCODES: usize = 4; + +#[cfg(test)] +pub(crate) fn auto_x86_filter_ranges(data: &[u8], include_e9: bool) -> Vec> { + ranges_with_allowance(data, include_e9, &Allowance::default()) + .expect("unlimited scanner") + .into_vec() +} + +pub(crate) fn ranges_with_allowance( + data: &[u8], + include_e9: bool, + allowance: &B, +) -> Result, B>> { + let mut ranges = + auto_x86_filter_ranges_with_cluster_gap(data, include_e9, AUTO_X86_CLUSTER_GAP, allowance)?; + for range in auto_x86_filter_ranges_with_cluster_gap( + data, + include_e9, + AUTO_X86_TIGHT_CLUSTER_GAP, + allowance, + )? { + if !ranges.contains(&range) { + ranges.try_push(range)?; + } + } + Ok(ranges) +} + +fn auto_x86_filter_ranges_with_cluster_gap( + data: &[u8], + include_e9: bool, + cluster_gap: usize, + allowance: &B, +) -> Result, B>> { + if data.len() <= 5 { + return Ok(Buffer::new(allowance)); + } + + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let mut clusters = Buffer::new(allowance); + let mut current: Option<(usize, usize, usize)> = None; + let mut scan_pos = 0usize; + while let Some(pos) = + crate::codec::fast::next_x86_opcode(data, scan_pos, data.len() - 4, cmp_mask) + { + match current { + Some((start, last, count)) if pos - last <= cluster_gap => { + current = Some((start, pos, count + 1)); + } + Some(cluster) => { + clusters.try_push(cluster)?; + current = Some((pos, pos, 1)); + } + None => current = Some((pos, pos, 1)), + } + scan_pos = pos + 1; + } + if let Some(cluster) = current { + clusters.try_push(cluster)?; + } + + clusters.retain(|&(_, _, count)| count >= 2); + let mut ranges = Buffer::new(allowance); + let mut span_count = 0; + let mut span: Option<(usize, usize, usize)> = None; + for &(start, last, count) in clusters.iter() { + match span { + Some((span_start, span_last, span_opcodes)) + if start.saturating_sub(span_last) <= AUTO_X86_SPAN_CLUSTER_GAP => + { + span = Some((span_start, last, span_opcodes + count)); + } + Some((span_start, span_last, span_opcodes)) => { + if span_opcodes >= AUTO_X86_MIN_SPAN_OPCODES + && span_count < AUTO_X86_MAX_SPAN_RANGES + { + push_range(&mut ranges, data.len(), span_start, span_last)?; + span_count += 1; + } + span = Some((start, last, count)); + } + None => span = Some((start, last, count)), + } + } + if let Some((span_start, span_last, span_opcodes)) = span { + if span_opcodes >= AUTO_X86_MIN_SPAN_OPCODES && span_count < AUTO_X86_MAX_SPAN_RANGES { + push_range(&mut ranges, data.len(), span_start, span_last)?; + } + } + + clusters.sort_unstable_by(|a, b| { + let a_len = a.1 - a.0 + 5; + let b_len = b.1 - b.0 + 5; + b.2.cmp(&a.2) + .then_with(|| a_len.cmp(&b_len)) + .then_with(|| a.0.cmp(&b.0)) + }); + clusters.truncate(AUTO_X86_MAX_RANGES); + + for (start, last, _) in clusters { + push_range(&mut ranges, data.len(), start, last)?; + } + Ok(ranges) +} + +fn push_range( + ranges: &mut Buffer, B>, + data_len: usize, + start: usize, + last: usize, +) -> Result<()> { + let range_start = start.saturating_sub(AUTO_X86_RANGE_PADDING); + let range_end = (last + 5 + AUTO_X86_RANGE_PADDING).min(data_len); + let range = range_start..range_end; + // Scanned clusters satisfy start <= last <= data_len - 5, so padding + // and clamping cannot turn their range into an empty one. + if !ranges.contains(&range) { + ranges.try_push(range)?; + } + Ok(()) +} + +#[cfg(test)] +fn push_x86_filter_range( + ranges: &mut Vec>, + data_len: usize, + start: usize, + last: usize, +) { + let range = start.saturating_sub(AUTO_X86_RANGE_PADDING) + ..(last + 5 + AUTO_X86_RANGE_PADDING).min(data_len); + if range.start < range.end && !ranges.contains(&range) { + ranges.push(range); + } +} +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn span_limit_keeps_first_four_disjoint_spans_and_best_clusters() { + let mut data = vec![0x90; 300_000]; + for span in 0..6 { + let start = span * 50_000; + for offset in [0, 1, 5_000, 5_001] { + data[start + offset] = 0xe8; + } + } + let mut expected = Vec::new(); + for span in 0..4usize { + let start = span * 50_000; + expected.push(start.saturating_sub(16)..start + 5_022); + } + for span in 0..4usize { + for offset in [0, 5_000] { + let start = span * 50_000 + offset; + expected.push(start.saturating_sub(16)..start + 22); + } + } + assert_eq!(auto_x86_filter_ranges(&data, true), expected); + assert_eq!(scalar_auto_x86_filter_ranges(&data, true), expected); + } + + #[test] + fn scanner_retains_sparse_clusters_without_emitting_a_sparse_span() { + let mut data = vec![0x90; 40_030]; + for pos in [0, 1, 40_000, 40_001, 40_002, 40_003] { + data[pos] = 0xe8; + } + let expected = vec![39_984..40_024, 0..22]; + assert_eq!(auto_x86_filter_ranges(&data, true), expected); + assert_eq!(scalar_auto_x86_filter_ranges(&data, true), expected); + assert!(auto_x86_filter_ranges(&vec![0x90; 40_030], true).is_empty()); + } + + #[test] + fn scanned_regions_in_members_large_enough_to_screen_are_never_short_samples() { + for len in 12..=64 { + for start in 0..len - 5 { + let mut data = vec![0; len]; + data[start] = 0xe8; + data[start + 1] = 0xe8; + for region in auto_x86_filter_ranges(&data, true) { + assert!(region.len() >= 12, "{len}, {start}, {region:?}"); + } + } + } + } + + fn scalar_auto_x86_filter_ranges(data: &[u8], include_e9: bool) -> Vec> { + let mut ranges = + scalar_auto_x86_filter_ranges_with_cluster_gap(data, include_e9, AUTO_X86_CLUSTER_GAP); + for range in scalar_auto_x86_filter_ranges_with_cluster_gap( + data, + include_e9, + AUTO_X86_TIGHT_CLUSTER_GAP, + ) { + if !ranges.contains(&range) { + ranges.push(range); + } + } + ranges + } + + fn scalar_auto_x86_filter_ranges_with_cluster_gap( + data: &[u8], + include_e9: bool, + cluster_gap: usize, + ) -> Vec> { + if data.len() <= 5 { + return Vec::new(); + } + + let cmp_mask = if include_e9 { 0xfe } else { 0xff }; + let mut clusters = Vec::new(); + let mut current: Option<(usize, usize, usize)> = None; + for (pos, &byte) in data.iter().take(data.len() - 4).enumerate() { + if byte & cmp_mask != 0xe8 { + continue; + } + + match current { + Some((start, last, count)) if pos - last <= cluster_gap => { + current = Some((start, pos, count + 1)); + } + Some(cluster) => { + clusters.push(cluster); + current = Some((pos, pos, 1)); + } + None => current = Some((pos, pos, 1)), + } + } + if let Some(cluster) = current { + clusters.push(cluster); + } + + clusters.retain(|&(_, _, count)| count >= 2); + let mut ranges = Vec::new(); + let mut span_count = 0; + let mut span: Option<(usize, usize, usize)> = None; + for &(start, last, count) in &clusters { + match span { + Some((span_start, span_last, span_opcodes)) + if start.saturating_sub(span_last) <= AUTO_X86_SPAN_CLUSTER_GAP => + { + span = Some((span_start, last, span_opcodes + count)); + } + Some((span_start, span_last, span_opcodes)) => { + if span_opcodes >= AUTO_X86_MIN_SPAN_OPCODES + && span_count < AUTO_X86_MAX_SPAN_RANGES + { + push_x86_filter_range(&mut ranges, data.len(), span_start, span_last); + span_count += 1; + } + span = Some((start, last, count)); + } + None => span = Some((start, last, count)), + } + } + if let Some((span_start, span_last, span_opcodes)) = span { + if span_opcodes >= AUTO_X86_MIN_SPAN_OPCODES && span_count < AUTO_X86_MAX_SPAN_RANGES { + push_x86_filter_range(&mut ranges, data.len(), span_start, span_last); + } + } + + clusters.sort_by(|a, b| { + let a_len = a.1 - a.0 + 5; + let b_len = b.1 - b.0 + 5; + b.2.cmp(&a.2).then_with(|| a_len.cmp(&b_len)) + }); + clusters.truncate(AUTO_X86_MAX_RANGES); + + for (start, last, _) in clusters { + push_x86_filter_range(&mut ranges, data.len(), start, last); + } + ranges + } + + #[test] + fn bounded_scanner_preserves_tied_clusters_and_releases_refused_growth() { + let mut data = vec![0x41; 500_000]; + for section in 0..40 { + let start = 1024 + section * 10_000; + for offset in [0, 32, 64, 96] { + data[start + offset] = 0xe8; + } + } + let expected = scalar_auto_x86_filter_ranges(&data, true); + let mut successes = 0; + let mut refusals = 0; + // Close limits distinguish tight-pass refusal while broad-pass ranges + // remain charged from refusal during the initial scan. + for limit in [0, 95, 128, 256, 512, 1024, 2048, 8192] + .into_iter() + .chain((2048..=3072).step_by(32)) + { + let allowance = Allowance::limited(limit); + match ranges_with_allowance(&data, true, &allowance) { + Ok(ranges) => { + successes += 1; + assert_eq!(&*ranges, expected); + assert!( + allowance.used() + >= (ranges.len() * std::mem::size_of::>()) as u64 + ); + drop(ranges); + } + Err(crate::codec::Error::WorkspaceLimitExceeded(_)) => refusals += 1, + Err(error) => panic!("{error}"), + } + assert_eq!(allowance.used(), 0); + } + assert!(successes > 0 && refusals >= 3); + } + + #[test] + fn returns_no_ranges_for_inputs_too_short_to_contain_a_call() { + for len in 0..=5 { + let data = vec![0xe8; len]; + assert!(auto_x86_filter_ranges(&data, false).is_empty()); + assert!(auto_x86_filter_ranges(&data, true).is_empty()); + } + } + + #[test] + fn auto_x86_filter_ranges_match_reference_scanner_at_chunk_boundaries() { + let mut data = vec![0x41u8; 150_000]; + for pos in [ + 31usize, 32, 33, 1024, 1088, 4096, 4160, 80_000, 80_032, 80_064, + ] { + data[pos] = 0xe8; + } + data[80_096] = 0xe9; + + assert_eq!( + auto_x86_filter_ranges(&data, false), + scalar_auto_x86_filter_ranges(&data, false) + ); + assert_eq!( + auto_x86_filter_ranges(&data, true), + scalar_auto_x86_filter_ranges(&data, true) + ); + } + + #[test] + fn drops_isolated_opcodes_that_never_form_a_cluster() { + let mut data = vec![0x41; 20_000]; + data[100] = 0xe8; + data[10_000] = 0xe8; + assert!(auto_x86_filter_ranges(&data, false).is_empty()); + } + + #[test] + fn clamps_padded_range_to_buffer_bounds_at_both_ends() { + let mut data = vec![0x41u8; 30]; + for pos in [0, 4, 8, 12] { + data[pos] = 0xe8; + } + + let ranges = auto_x86_filter_ranges(&data, false); + + assert_eq!(ranges, vec![0..30]); + } + + #[test] + fn does_not_duplicate_a_span_range_already_emitted_for_a_cluster() { + let mut data = vec![0x41u8; 20_000]; + for pos in [1024, 1050, 1090, 1130] { + data[pos] = 0xe8; + } + + let ranges = auto_x86_filter_ranges(&data, false); + + assert_eq!(ranges.len(), 1); + assert_eq!(ranges[0], 1008..1151); + } + + #[test] + fn includes_tighter_ranges_inside_sparse_code_spans() { + let mut data = vec![0x41u8; 8_000]; + for pos in [1024, 1088, 3600, 3664] { + data[pos] = 0xe8; + } + + let ranges = auto_x86_filter_ranges(&data, false); + + assert!( + ranges + .iter() + .any(|range| range.start <= 1024 && range.end > 3664 && range.len() > 2000), + "missing broad sparse-code span: {ranges:?}" + ); + assert!( + ranges.iter().any(|range| range.contains(&1024) + && range.contains(&1088) + && !range.contains(&3600)), + "missing first tight code cluster: {ranges:?}" + ); + assert!( + ranges.iter().any(|range| range.contains(&3600) + && range.contains(&3664) + && !range.contains(&1088)), + "missing second tight code cluster: {ranges:?}" + ); + } + + #[test] + fn keeps_more_disjoint_code_section_candidates() { + let mut data = vec![0x41u8; 700_000]; + for section in 0..8 { + let start = 16_384 + section * 80_000; + for index in 0..6 { + data[start + index * 64] = 0xe8; + } + } + + let ranges = auto_x86_filter_ranges(&data, false); + + for section in 0..8 { + let start = 16_384 + section * 80_000; + assert!( + ranges.iter().any(|range| range.contains(&start)), + "missing x86 section at {start}" + ); + } + } +} diff --git a/arca-rar/vendor/rars/tests/builder_directories.rs b/arca-rar/vendor/rars/tests/builder_directories.rs new file mode 100644 index 00000000..709074c8 --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_directories.rs @@ -0,0 +1,83 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReader, ArchiveVersion, Builder}; + +#[path = "support/scratch.rs"] +mod scratch; + +#[test] +fn explicit_directories_survive_all_rar5_writer_paths() { + let root = scratch::case("builder-directories"); + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for solid in [false, true] { + let mut builder = Builder::new(format).solid(solid); + builder + .add_directory(b"empty".to_vec(), Some(123), None) + .unwrap(); + builder + .add_directory(b"nested".to_vec(), Some(456), Some(0o750)) + .unwrap(); + builder + .set_mtime_nanoseconds(b"nested", 704_088_300) + .unwrap(); + builder + .add_bytes(b"nested/file".to_vec(), b"payload".repeat(20), None, None) + .unwrap(); + let path = root.join(format!("{format}-{solid}.rar")); + builder.write_to_path(&path, None).unwrap(); + let mut outputs = vec![builder.to_bytes().unwrap(), std::fs::read(path).unwrap()]; + outputs.extend(builder.volume_size(Some(4096)).build_volumes(None).unwrap()); + for bytes in outputs { + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert_eq!(members.len(), 3); + assert!(members[0].meta.is_directory); + assert_eq!(members[0].meta.file_attr, 0x10); + assert_eq!(members[0].meta.file_time, Some(123)); + assert!(members[1].meta.is_directory); + assert_eq!(members[1].meta.file_attr, 0o040750); + assert_eq!( + members[1].meta.mtime_refinement.unwrap().nanoseconds, + 704_088_300 + ); + assert!(!members[2].meta.is_directory); + assert_eq!( + archive.read_member(b"nested/file", None).unwrap().unwrap(), + b"payload".repeat(20) + ); + } + } + } +} + +#[test] +fn directory_validation_does_not_change_entry_kind_or_queued_names() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_directory(b"empty".to_vec(), None, None) + .unwrap(); + let before = builder.to_bytes().unwrap(); + assert!(builder.set_dos_attributes(b"empty", 0x20).is_err()); + assert!(builder + .add_directory(b"empty".to_vec(), None, None) + .is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + let mut legacy = Builder::new(ArchiveVersion::Rar15); + assert!(legacy + .add_directory(b"empty".to_vec(), None, Some(0o750)) + .is_err()); + assert!(legacy.is_empty()); +} + +#[test] +fn rar15_directory_uses_legacy_directory_attributes() { + let mut builder = Builder::new(ArchiveVersion::Rar15).store(true); + builder + .add_directory(b"empty".to_vec(), Some(123), None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + assert!(member.meta.is_directory); + assert_eq!(member.meta.file_attr, 0x10); + assert_eq!(member.meta.file_time, Some(123)); +} diff --git a/arca-rar/vendor/rars/tests/builder_file_comments.rs b/arca-rar/vendor/rars/tests/builder_file_comments.rs new file mode 100644 index 00000000..96ad06f4 --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_file_comments.rs @@ -0,0 +1,285 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReader, ArchiveVersion, Builder, ErrorKind}; + +#[test] +fn recovery_service_does_not_change_rar5_member_comments() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .recovery_percent(Some(5)); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"note".to_vec())) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let Archive::Rar50Plus(ref rar50) = archive else { + unreachable!() + }; + assert!(rar50.blocks.iter().any( + |block| matches!(block, rars::rar50::Block::Service(service) if service.name == b"RR") + )); + assert_eq!( + archive.member_comments(None).unwrap(), + vec![Some(b"note".to_vec())] + ); + assert_eq!(archive.member_comment_encryption(), [false]); +} + +#[test] +fn rar5_comment_before_first_file_belongs_to_archive() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"note".to_vec())) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let Archive::Rar50Plus(ref mut rar50) = archive else { + unreachable!() + }; + assert!(matches!(rar50.blocks[0], rars::rar50::Block::File(_))); + assert!(matches!(rar50.blocks[1], rars::rar50::Block::Service(_))); + rar50.blocks.swap(0, 1); + + assert_eq!(archive.comment(None).unwrap(), Some(b"note".to_vec())); + assert_eq!(archive.member_comments(None).unwrap(), vec![None]); + assert_eq!(archive.member_comment_encryption(), [false]); + assert!(archive.rewrite_preservation_issues().is_empty()); +} + +#[test] +fn comments_follow_renames_and_round_trip_through_supported_writers() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for stored in [true, false] { + let mut builder = Builder::new(version).store(stored); + builder + .add_bytes(b"first".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"second".to_vec(), vec![], None, None) + .unwrap(); + builder + .set_file_comment(b"first", Some(b"note\xff".to_vec())) + .unwrap(); + builder.set_file_comment(b"second", Some(vec![])).unwrap(); + builder.rename(b"first", b"renamed".to_vec()).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.member_comments(None).unwrap(), + vec![Some(b"note\xff".to_vec()), Some(vec![])], + "{version:?}" + ); + assert_eq!( + archive.member_comment_at(0, None).unwrap(), + Some(b"note\xff".to_vec()) + ); + assert_eq!( + archive.read_member(b"renamed", None).unwrap().unwrap(), + b"payload" + ); + assert_eq!( + archive.member_comment_at(2, None).unwrap_err().kind(), + ErrorKind::EntryNotFound + ); + } + } +} + +#[test] +fn encrypted_headers_reject_incompatible_embedded_comments() { + for version in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + let mut builder = Builder::new(version) + .password(Some(b"secret".to_vec())) + .header_encryption(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"note".to_vec())) + .unwrap(); + assert!(matches!( + builder.to_bytes(), + Err(rars::Error::UnsupportedWriterOption { + option: rars::write_plan::WriterOption::FileComment, + .. + }) + )); + builder.set_file_comment(b"file", None).unwrap(); + let builder = builder.header_encryption(false).password(None); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!(archive.member_comments(None).unwrap(), vec![None]); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } +} + +#[test] +fn volume_output_refuses_comments_even_when_configured_after_the_comment() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"note".to_vec())) + .unwrap(); + let mut builder = builder.volume_size(Some(4096)); + assert!(matches!( + builder.build_volumes(None), + Err(rars::Error::UnsupportedWriterOption { + option: rars::write_plan::WriterOption::FileComment, + .. + }) + )); + assert!(builder.set_file_comment(b"file", Some(vec![])).is_err()); + builder.set_file_comment(b"file", None).unwrap(); + assert!(!builder.build_volumes(None).unwrap().is_empty()); + } +} + +#[test] +fn rar5_file_comments_follow_output_encryption_and_can_be_removed() { + let mut builder = Builder::new(ArchiveVersion::Rar50).password(Some(b"secret".to_vec())); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"private note".to_vec())) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.member_comment_at(0, None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!( + archive.member_comment_at(0, Some(b"secret")).unwrap(), + Some(b"private note".to_vec()) + ); + builder.set_file_comment(b"file", Some(vec![])).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.member_comment_at(0, Some(b"secret")).unwrap(), + Some(vec![]) + ); + builder.set_file_comment(b"file", None).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!(archive.member_comments(None).unwrap(), vec![None]); +} + +#[test] +fn preflight_accepts_attached_comments_but_rejects_duplicates_and_read_checks_integrity() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .comment(Some(b"archive".to_vec())); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .set_file_comment(b"dir", Some(b"directory".to_vec())) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), vec![], None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"file note".to_vec())) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + assert_eq!( + archive.member_comments(None).unwrap(), + [Some(b"directory".to_vec()), Some(b"file note".to_vec())] + ); + let Archive::Rar50Plus(a) = &mut archive else { + unreachable!() + }; + let at = a + .blocks + .iter() + .rposition(|b| matches!(b, rars::rar50::Block::Service(_))) + .unwrap(); + let original = a.blocks[at].clone(); + if let rars::rar50::Block::Service(file) = &mut a.blocks[at] { + file.hash = None; + file.data_crc32 = Some(0); + } + assert_eq!( + archive.member_comment_at(1, None).unwrap_err().kind(), + ErrorKind::ChecksumMismatch + ); + let Archive::Rar50Plus(a) = &mut archive else { + unreachable!() + }; + a.blocks[at] = original.clone(); + a.blocks.insert(at, original); + assert!(!archive.rewrite_preservation_issues().is_empty()); + assert!(archive.member_comments(None).is_err()); + assert!(archive.member_comment_at(1, None).is_err()); +} + +#[test] +fn legacy_archive_comment_encryption_is_independent_of_member_passwords() { + for encrypted_headers in [false, true] { + let password = b"secret"; + let mut builder = Builder::new(ArchiveVersion::Rar40) + .password(Some(password.to_vec())) + .header_encryption(encrypted_headers) + .comment(Some(b"private comment".to_vec())) + .archive_comment_password(Some(password.to_vec())); + builder + .add_bytes(b"plain".to_vec(), b"public".to_vec(), None, None) + .unwrap(); + builder.set_entry_encryption(b"plain", None, None).unwrap(); + let source = ArchiveReader::read_owned_with_options( + builder.to_bytes().unwrap(), + rars::ArchiveReadOptions::with_password(password), + ) + .unwrap(); + assert_eq!( + source.comment(Some(password)).unwrap(), + Some(b"private comment".to_vec()) + ); + assert!(!source.members().next().unwrap().meta.is_encrypted); + assert!(source.rewrite_preservation_issues().is_empty()); + let mut editor = source + .preserving_builder(Some(password)) + .unwrap() + .comment(source.comment(Some(password)).unwrap()); + editor + .add_bytes(b"renamed".to_vec(), b"public".to_vec(), None, None) + .unwrap(); + editor.set_entry_encryption(b"renamed", None, None).unwrap(); + let output = ArchiveReader::read_owned_with_options( + editor.to_bytes().unwrap(), + rars::ArchiveReadOptions::with_password(password), + ) + .unwrap(); + assert_eq!( + output.comment(Some(password)).unwrap(), + Some(b"private comment".to_vec()) + ); + assert!(output.comment(None).is_err()); + assert!(!output.members().next().unwrap().meta.is_encrypted); + if let Archive::Rar15To40(archive) = output { + assert_eq!(archive.main.has_encrypted_headers(), encrypted_headers); + assert!(archive.new_subs().next().unwrap().file.is_encrypted()); + } else { + panic!("legacy output expected"); + } + } +} diff --git a/arca-rar/vendor/rars/tests/builder_identity.rs b/arca-rar/vendor/rars/tests/builder_identity.rs new file mode 100644 index 00000000..222aa79a --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_identity.rs @@ -0,0 +1,246 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReader, ArchiveVersion, Builder, EntrySource}; + +fn file_copy_member(name: &[u8], target: &[u8]) -> rars::ArchiveMember { + let mut seed = Builder::new(ArchiveVersion::Rar50).store(true); + seed.add_unix_symlink(name.to_vec(), target.to_vec(), false, None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(seed.to_bytes().unwrap()).unwrap(); + let mut copy = archive.members().next().unwrap(); + copy.meta.host_os = Some(0); + copy.meta.file_attr = 0x20; + copy.meta.unpacked_size = 7; + if let rars::ArchiveMemberDetail::Rar50Plus { + redirection: Some(link), + .. + } = &mut copy.detail + { + link.redirection_type = 5; + link.flags = 0; + } + copy +} + +#[test] +fn duplicate_payloads_and_metadata_follow_ids_in_every_family() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let mut builder = Builder::new(format).store(true).allow_duplicate_names(true); + builder + .add_bytes(b"same".to_vec(), b"first".to_vec(), None, None) + .unwrap(); + builder.add_directory(b"same".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"same".to_vec(), b"second".to_vec(), None, None) + .unwrap(); + assert!(builder.remove(b"same").is_err()); + assert_eq!(builder.len(), 3); + builder.rename_by_id(2, b"second".to_vec()).unwrap(); + builder.remove_by_id(1).unwrap(); + builder.rename_by_id(0, b"first".to_vec()).unwrap(); + assert_eq!(builder.member_ids().collect::>(), [0, 2]); + assert!(builder.remove_by_id(1).is_err()); + let bytes = builder.to_bytes().unwrap(); + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert_eq!(members[0].meta.name, b"first"); + assert_eq!(members[1].meta.name, b"second"); + assert_eq!(archive.read_member_at(0, None).unwrap().unwrap(), b"first"); + assert_eq!(archive.read_member_at(1, None).unwrap().unwrap(), b"second"); + assert_eq!(members[0].meta.unpacked_size, 5); + assert_eq!(members[1].meta.unpacked_size, 6); + } +} + +#[test] +fn removing_a_duplicate_cannot_retarget_a_file_copy() { + let copy = file_copy_member(b"copy", b"same"); + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .allow_duplicate_names(true); + builder + .add_source( + b"same".to_vec(), + EntrySource::from_bytes(b"payload".as_slice()), + None, + None, + ) + .unwrap(); + builder.add_archive_redirection(©).unwrap(); + builder + .add_bytes(b"same".to_vec(), b"another".to_vec(), None, None) + .unwrap(); + assert!(matches!( + builder.remove_by_id(0), + Err(rars::Error::InvalidArgument( + "removing this duplicate would retarget a hard link or file copy" + )) + )); + assert_eq!(builder.member_ids().collect::>(), [0, 1, 2]); + builder.remove_by_id(2).unwrap(); + let output = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(output.rewrite_preservation_issues().is_empty()); + assert_eq!(output.read_member_at(0, None).unwrap().unwrap(), b"payload"); + + let mut shadowed = Builder::new(ArchiveVersion::Rar50) + .store(true) + .allow_duplicate_names(true); + for data in [b"payload".as_slice(), b"another".as_slice()] { + shadowed + .add_bytes(b"same".to_vec(), data.to_vec(), None, None) + .unwrap(); + } + shadowed.add_archive_redirection(©).unwrap(); + shadowed.remove_by_id(0).unwrap(); + let output = ArchiveReader::read_owned(shadowed.to_bytes().unwrap()).unwrap(); + assert!(output.rewrite_preservation_issues().is_empty()); + assert_eq!(output.read_member_at(0, None).unwrap().unwrap(), b"another"); +} + +#[test] +fn removing_a_duplicate_ignores_directories_and_unrelated_copies() { + let copy = file_copy_member(b"copy", b"other"); + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .allow_duplicate_names(true); + builder + .add_bytes(b"same".to_vec(), b"first".to_vec(), None, None) + .unwrap(); + builder.add_directory(b"same".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"other".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.add_archive_redirection(©).unwrap(); + builder + .add_bytes(b"same".to_vec(), b"second".to_vec(), None, None) + .unwrap(); + + builder.remove_by_id(0).unwrap(); + let output = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let copy = output + .members() + .find(|member| member.meta.name == b"copy") + .unwrap(); + assert_eq!(copy.supported_redirection().unwrap().target_name, b"other"); + assert_eq!( + output.read_member(b"other", None).unwrap().unwrap(), + b"payload" + ); + assert_eq!( + output.read_member(b"same", None).unwrap().unwrap(), + b"second" + ); +} + +#[test] +fn renaming_a_duplicate_updates_only_copies_of_that_identity() { + let mut copy = file_copy_member(b"copy", b"same"); + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .allow_duplicate_names(true); + builder + .add_bytes(b"same".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.add_archive_redirection(©).unwrap(); + builder + .add_bytes(b"same".to_vec(), b"another".to_vec(), None, None) + .unwrap(); + copy.meta.name = b"later-copy".to_vec(); + builder.add_archive_redirection(©).unwrap(); + builder + .add_bytes(b"other".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + copy.meta.name = b"other-copy".to_vec(); + if let rars::ArchiveMemberDetail::Rar50Plus { + redirection: Some(link), + .. + } = &mut copy.detail + { + link.target_name = b"other".to_vec(); + } + builder.add_archive_redirection(©).unwrap(); + + builder.rename_by_id(0, b"first".to_vec()).unwrap(); + let output = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for (name, target) in [ + (b"copy".as_slice(), b"first".as_slice()), + (b"later-copy".as_slice(), b"same".as_slice()), + (b"other-copy".as_slice(), b"other".as_slice()), + ] { + let member = output + .members() + .find(|member| member.meta.name == name) + .unwrap(); + assert_eq!(member.supported_redirection().unwrap().target_name, target); + } +} + +#[test] +fn file_copy_validation_ignores_interleaved_unix_symlink() { + let copy = file_copy_member(b"copy", b"other"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"other".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .add_unix_symlink(b"link".to_vec(), b"other".to_vec(), false, None, None) + .unwrap(); + builder.add_archive_redirection(©).unwrap(); + + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"other", None).unwrap().unwrap(), + b"payload" + ); + let members: Vec<_> = archive.members().collect(); + assert_eq!(members[1].unix_symlink().unwrap().target_name, b"other"); + assert_eq!( + members[2].supported_redirection().unwrap().target_name, + b"other" + ); +} + +#[test] +fn removing_duplicate_does_not_treat_unix_symlink_as_file_copy() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .allow_duplicate_names(true); + builder + .add_bytes(b"same".to_vec(), b"first".to_vec(), None, None) + .unwrap(); + builder + .add_unix_symlink(b"link".to_vec(), b"same".to_vec(), false, None, None) + .unwrap(); + builder + .add_bytes(b"same".to_vec(), b"second".to_vec(), None, None) + .unwrap(); + + builder.remove_by_id(0).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive + .members() + .next() + .unwrap() + .unix_symlink() + .unwrap() + .target_name, + b"same" + ); + assert_eq!( + archive.read_member(b"same", None).unwrap().unwrap(), + b"second" + ); +} diff --git a/arca-rar/vendor/rars/tests/builder_metadata.rs b/arca-rar/vendor/rars/tests/builder_metadata.rs new file mode 100644 index 00000000..880520e3 --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_metadata.rs @@ -0,0 +1,263 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::{ArchiveFamily, ArchiveReader, ArchiveVersion, Builder}; + +const MODES: [Option; 4] = [None, Some(0o640), Some(0o100750), Some(0)]; + +#[test] +fn rar5_archive_metadata_settings_refuse_volume_output() { + for (locked, quick_open) in [(true, false), (false, true)] { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .archive_metadata(None, locked, quick_open) + .unwrap() + .volume_size(Some(4096)); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert!(builder.to_bytes().is_err()); + assert!(builder.build_volumes(None).is_err()); + } +} + +#[test] +fn builder_preserves_explicit_dos_attributes_in_all_writer_paths() { + let flags = [0, 1, 2, 4, 0x20, 0x27, 0x80]; + let root = scratch::case("builder-dos-attributes"); + for format in ArchiveVersion::ALL { + for store in [false, true] { + let mut builder = Builder::new(format).store(store); + for (index, attr) in flags.iter().enumerate() { + let name = format!("file{index}").into_bytes(); + builder + .add_source( + name.clone(), + rars::EntrySource::from_bytes(b"DOS payload".to_vec()), + None, + Some(0o640), + ) + .unwrap(); + builder.set_dos_attributes(&name, *attr).unwrap(); + } + let path = root.join(format!("{format}-{store}.rar")); + builder.write_to_path(&path, None).unwrap(); + let mut outputs = vec![builder.to_bytes().unwrap(), std::fs::read(path).unwrap()]; + if format.family() == ArchiveFamily::Rar50Plus { + outputs.extend(builder.volume_size(Some(4096)).build_volumes(None).unwrap()); + } + for bytes in outputs { + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert_eq!(members.len(), flags.len()); + for (member, attr) in members.iter().zip(flags) { + assert_eq!(member.meta.attr_source(), rars::AttrSource::Dos); + assert_eq!(member.meta.file_attr, attr, "{format}, store={store}"); + assert!(!member.meta.is_directory); + assert_eq!( + archive + .read_member(&member.meta.name, None) + .unwrap() + .unwrap(), + b"DOS payload" + ); + } + } + } + } +} + +#[test] +fn invalid_dos_attributes_leave_builder_metadata_unchanged() { + for (format, oversized) in [ + (ArchiveVersion::Rar14, Some(256)), + (ArchiveVersion::Rar29, Some(u64::from(u32::MAX) + 1)), + (ArchiveVersion::Rar50, None), + ] { + let mut builder = builder(format, true); + let before = builder.to_bytes().unwrap(); + assert!(builder.set_dos_attributes(b"missing", 0x20).is_err()); + assert!(builder.set_dos_attributes(b"file1", 0x10).is_err()); + if let Some(oversized) = oversized { + assert!(builder.set_dos_attributes(b"file1", oversized).is_err()); + } + assert_eq!(builder.to_bytes().unwrap(), before); + } +} + +#[test] +fn dos_attribute_field_width_is_not_truncated() { + for (format, attr) in [ + (ArchiveVersion::Rar14, 0xef), + (ArchiveVersion::Rar29, u64::from(u32::MAX) & !0x10), + (ArchiveVersion::Rar50, !0x10u64), + ] { + let mut builder = builder(format, true); + builder.set_dos_attributes(b"file1", attr).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!(archive.members().nth(1).unwrap().meta.file_attr, attr); + } +} + +#[test] +fn member_attribute_source_uses_family_specific_host_rules() { + use rars::AttrSource::{Dos, Unix, Unknown}; + + let archive = + ArchiveReader::read_owned(builder(ArchiveVersion::Rar50, true).to_bytes().unwrap()) + .unwrap(); + let mut meta = archive.members().next().unwrap().meta; + for (family, host, expected) in [ + (ArchiveFamily::Rar13, None, Dos), + (ArchiveFamily::Rar15To40, Some(0), Dos), + (ArchiveFamily::Rar15To40, Some(1), Dos), + (ArchiveFamily::Rar15To40, Some(2), Dos), + (ArchiveFamily::Rar15To40, Some(3), Unix), + (ArchiveFamily::Rar15To40, Some(4), Dos), + (ArchiveFamily::Rar15To40, Some(5), Unix), + (ArchiveFamily::Rar15To40, Some(6), Unknown), + (ArchiveFamily::Rar15To40, Some(259), Unknown), + (ArchiveFamily::Rar15To40, None, Unknown), + (ArchiveFamily::Rar50Plus, Some(0), Dos), + (ArchiveFamily::Rar50Plus, Some(1), Unix), + (ArchiveFamily::Rar50Plus, Some(2), Unknown), + (ArchiveFamily::Rar50Plus, None, Unknown), + ] { + meta.family = family; + meta.host_os = host; + assert_eq!(meta.attr_source(), expected, "{family:?}, {host:?}"); + } +} + +fn builder(format: ArchiveVersion, store: bool) -> Builder { + let mut builder = Builder::new(format).store(store); + for (index, mode) in MODES.into_iter().enumerate() { + builder + .add_bytes( + format!("file{index}").into_bytes(), + b"payload".repeat(64), + None, + mode, + ) + .unwrap(); + } + builder +} + +fn check_metadata(bytes: Vec, format: ArchiveVersion) { + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert_eq!(members.len(), MODES.len()); + for (member, mode) in members.iter().zip(MODES) { + // RAR 1.5 deliberately converts Unix metadata to DOS for compatibility. + let mode = if format == ArchiveVersion::Rar15 { + None + } else { + mode + }; + let (host, attr) = match mode { + None => (0, 0x20), + Some(mode) => ( + if format.family() == ArchiveFamily::Rar50Plus { + 1 + } else { + 3 + }, + u64::from(mode | 0o100000), + ), + }; + assert_eq!(member.meta.host_os, Some(host)); + assert_eq!(member.meta.file_attr, attr); + assert_eq!( + archive + .read_member(&member.meta.name, None) + .unwrap() + .unwrap(), + b"payload".repeat(64) + ); + } +} + +#[test] +fn builder_pairs_host_ids_with_dos_or_unix_attributes() { + for format in ArchiveVersion::ALL { + if format.family() == ArchiveFamily::Rar13 { + continue; // This family has no host field or Unix modes. + } + for store in [true, false] { + check_metadata(builder(format, store).to_bytes().unwrap(), format); + } + } +} + +#[test] +fn builder_volume_metadata_matches_single_archive_metadata() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for store in [true, false] { + let volumes = builder(format, store) + .volume_size(Some(4096)) + .build_volumes(None) + .unwrap(); + assert_eq!(volumes.len(), 1); + check_metadata(volumes.into_iter().next().unwrap(), format); + } + } +} + +#[cfg(unix)] +#[test] +#[ignore = "requires the external unrar executable"] +fn reference_unrar_restores_builder_permissions() { + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::process::Command; + + let root = scratch::case("builder-host-attributes"); + let control = root.join("default-permissions"); + fs::write(&control, b"control").unwrap(); + let default_mode = fs::metadata(control).unwrap().permissions().mode() & 0o777; + for format in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for store in [true, false] { + let archive = root.join(format!("{format}-{store}.rar")); + let output = root.join(format!("{format}-{store}")); + fs::create_dir(&output).unwrap(); + builder(format, store) + .write_to_path(&archive, None) + .unwrap(); + let result = Command::new("unrar") + .args(["x", "-idq", "-o+"]) + .arg(&archive) + .arg(format!("{}/", output.display())) + .output() + .unwrap(); + assert!( + result.status.success(), + "{format}: {} {}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + for (index, expected) in [default_mode, 0o640, 0o750, 0].into_iter().enumerate() { + let expected = if format == ArchiveVersion::Rar15 { + default_mode + } else { + expected + }; + let path = output.join(format!("file{index}")); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + expected, + "{format}, store={store}, file{index}" + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"payload".repeat(64)); + } + } + } +} diff --git a/arca-rar/vendor/rars/tests/builder_output.rs b/arca-rar/vendor/rars/tests/builder_output.rs new file mode 100644 index 00000000..a04498fc --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_output.rs @@ -0,0 +1,511 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::{entry_relative_path, ArchiveReader, ArchiveVersion, Builder, EntrySource}; +use std::fs; + +#[test] +fn empty_rar50_builder_writes_a_readable_archive() { + let archive = + ArchiveReader::read_owned(Builder::new(ArchiveVersion::Rar50).to_bytes().unwrap()).unwrap(); + assert_eq!(archive.members().count(), 0); +} + +#[test] +fn empty_legacy_builder_refuses_single_archive_output() { + assert_eq!( + Builder::new(ArchiveVersion::Rar29).to_bytes().unwrap_err(), + rars::Error::InvalidArgument("archive builder has no entries") + ); +} + +#[test] +fn legacy_volume_builder_refuses_existing_directories_and_symlinks() { + let mut directory = Builder::new(ArchiveVersion::Rar29); + directory + .add_directory(b"dir".to_vec(), None, None) + .unwrap(); + assert_eq!( + directory + .volume_size(Some(64)) + .build_volumes(None) + .unwrap_err(), + rars::Error::InvalidArgument( + "legacy directories and symbolic links are unsupported in volume output" + ) + ); + + let mut symlink = Builder::new(ArchiveVersion::Rar29); + symlink + .add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + assert_eq!( + symlink + .volume_size(Some(64)) + .build_volumes(None) + .unwrap_err(), + rars::Error::InvalidArgument( + "legacy directories and symbolic links are unsupported in volume output" + ) + ); +} + +#[test] +fn legacy_volumes_accept_regular_unix_mode_files() { + for version in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ] { + let mut builder = Builder::new(version).store(true).volume_size(Some(64)); + builder + .add_bytes(b"file".to_vec(), vec![42; 128], None, Some(0o100640)) + .unwrap(); + let volumes = builder.build_volumes(None).unwrap(); + assert!(volumes.len() > 1, "{version:?}"); + let first = ArchiveReader::read_owned(volumes[0].clone()).unwrap(); + let member = first.members().next().unwrap(); + assert_eq!(member.meta.attr_source(), rars::AttrSource::Unix); + assert_eq!(member.meta.file_attr & 0o170777, 0o100640); + } +} + +#[test] +fn single_output_rejects_volume_metadata_before_opening_a_source() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .archive_metadata(None, true, false) + .unwrap() + .volume_size(Some(64)); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(1, || panic!("single-output refusal opened source")), + None, + None, + ) + .unwrap(); + assert_eq!( + builder.to_bytes().unwrap_err(), + rars::Error::InvalidArgument( + "archive metadata settings require the RAR5/7 streaming writer" + ) + ); +} + +#[test] +fn path_output_reports_missing_parent_without_creating_an_archive() { + let root = scratch::case("builder-missing-parent"); + let destination = root.join("missing").join("archive.rar"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert_eq!( + builder + .write_to_path(&destination, None) + .unwrap_err() + .kind(), + rars::ErrorKind::Io + ); + assert!(!destination.exists()); + assert_eq!(fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn volume_builder_refuses_quick_open_before_source_io() { + let source = || EntrySource::from_opener(1, || panic!("volume setting opened source")); + let mut builder = Builder::new(ArchiveVersion::Rar50) + .archive_metadata(None, false, true) + .unwrap() + .volume_size(Some(64)); + builder + .add_source(b"file".to_vec(), source(), None, None) + .unwrap(); + assert_eq!( + builder.build_volumes(None).unwrap_err(), + rars::Error::InvalidArgument( + "archive metadata settings are not supported in volume output" + ) + ); +} + +#[test] +fn legacy_entry_paths_keep_relative_components_and_reject_escape() { + assert_eq!( + entry_relative_path(b"folder\\subdir/file.txt").unwrap(), + std::path::Path::new("folder/subdir/file.txt") + ); + assert_eq!( + entry_relative_path(b"./file.txt").unwrap(), + std::path::Path::new("file.txt") + ); + for name in [ + b"../escape".as_slice(), + b"/absolute", + b"folder/../../escape", + b"", + b".", + ] { + assert!(entry_relative_path(name).is_err(), "{name:?}"); + } +} + +#[cfg(unix)] +#[test] +fn legacy_entry_paths_preserve_non_utf8_bytes_on_unix() { + use std::os::unix::ffi::OsStrExt; + + let path = entry_relative_path(b"folder/\xff.bin").unwrap(); + assert_eq!(path.as_os_str().as_bytes(), b"folder/\xff.bin"); +} + +#[test] +fn resource_aware_byte_output_matches_the_standard_builder_path() { + for version in [ArchiveVersion::Rar29, ArchiveVersion::Rar50] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let expected = builder.to_bytes().unwrap(); + let actual = builder + .to_bytes_with_resources(&rars::WriterResources::default(), None) + .unwrap(); + assert_eq!(actual, expected, "{version:?}"); + let archive = ArchiveReader::read_owned(actual).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } +} + +#[test] +fn header_encryption_uses_a_shared_entry_password_without_a_builder_default() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .header_encryption(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_entry_encryption(b"file", Some(b"secret".to_vec()), None) + .unwrap(); + let archive = ArchiveReader::read_owned_with_options( + builder.to_bytes().unwrap(), + rars::ArchiveReadOptions::with_password(b"secret"), + ) + .unwrap(); + assert_eq!( + archive + .read_member(b"file", Some(b"secret")) + .unwrap() + .unwrap(), + b"payload" + ); +} + +#[test] +fn missing_header_password_fails_before_reading_a_source() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .header_encryption(true); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(7, || panic!("password error must precede source I/O")), + None, + None, + ) + .unwrap(); + assert_eq!(builder.to_bytes().unwrap_err(), rars::Error::NeedPassword); + let mut output = Vec::new(); + assert_eq!( + builder + .write_to(&mut output, &rars::WriterResources::default(), None) + .unwrap_err(), + rars::Error::NeedPassword + ); + assert!(output.is_empty()); +} + +#[test] +fn failed_builder_writes_preserve_the_destination_and_remove_staging_files() { + for existing in [false, true] { + let root = scratch::case("builder-failed-write"); + let destination = root.join("archive.rar"); + if existing { + fs::write(&destination, b"previous archive").unwrap(); + } + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(10, || { + Err(std::io::Error::other("injected source failure").into()) + }), + None, + None, + ) + .unwrap(); + assert!(builder.write_to_path(&destination, None).is_err()); + if existing { + assert_eq!(fs::read(&destination).unwrap(), b"previous archive"); + } else { + assert!(!destination.exists()); + } + assert_eq!(fs::read_dir(&root).unwrap().count(), usize::from(existing)); + } +} + +#[test] +fn builder_publishes_successful_writes_and_cleans_up_failed_renames() { + let root = scratch::case("builder-publish"); + let destination = root.join("archive.rar"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + fs::write(&destination, b"previous archive").unwrap(); + builder.write_to_path(&destination, None).unwrap(); + let archive = ArchiveReader::read_owned(fs::read(&destination).unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + assert_eq!(fs::read_dir(&root).unwrap().count(), 1); + + fs::remove_file(&destination).unwrap(); + fs::create_dir(&destination).unwrap(); + assert!(builder.write_to_path(&destination, None).is_err()); + assert!(destination.is_dir()); + assert_eq!(fs::read_dir(&root).unwrap().count(), 1); +} + +#[test] +fn legacy_recovery_requests_fail_before_reading_sources_or_writing_output() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for percent in [0, 10] { + let mut builder = Builder::new(version).recovery_percent(Some(percent)); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(7, || { + panic!("unsupported recovery request must be rejected before opening input") + }), + None, + None, + ) + .unwrap(); + assert_eq!( + builder.to_bytes().unwrap_err().kind(), + rars::ErrorKind::UnsupportedFeature + ); + let mut output = Vec::new(); + assert_eq!( + builder + .write_to(&mut output, &rars::WriterResources::default(), None) + .unwrap_err() + .kind(), + rars::ErrorKind::UnsupportedFeature + ); + assert!(output.is_empty()); + assert_eq!( + builder + .volume_size(Some(1024)) + .build_volumes(None) + .unwrap_err() + .kind(), + rars::ErrorKind::UnsupportedFeature + ); + } + } +} + +#[test] +fn volume_builder_rejects_unsupported_settings_before_opening_sources() { + let source = || { + EntrySource::from_opener(7, || { + panic!("volume preflight must not open the member source") + }) + }; + + let mut commented = Builder::new(ArchiveVersion::Rar50) + .comment(Some(b"comment".to_vec())) + .volume_size(Some(1024)); + commented + .add_source(b"file".to_vec(), source(), None, None) + .unwrap(); + assert!(matches!( + commented.build_volumes(None), + Err(rars::Error::InvalidArgument( + "RAR 5 volume comments are not supported" + )) + )); + + let mut locked = Builder::new(ArchiveVersion::Rar50) + .archive_metadata(None, true, false) + .unwrap() + .volume_size(Some(1024)); + locked + .add_source(b"file".to_vec(), source(), None, None) + .unwrap(); + assert!(matches!( + locked.build_volumes(None), + Err(rars::Error::InvalidArgument( + "archive metadata settings are not supported in volume output" + )) + )); + + let mut directory = Builder::new(ArchiveVersion::Rar20); + directory + .add_directory(b"directory".to_vec(), None, None) + .unwrap(); + directory + .add_source(b"file".to_vec(), source(), None, None) + .unwrap(); + assert!(matches!( + directory.volume_size(Some(1024)).build_volumes(None), + Err(rars::Error::InvalidArgument( + "legacy directories and symbolic links are unsupported in volume output" + )) + )); + + let empty = Builder::new(ArchiveVersion::Rar20).volume_size(Some(1024)); + assert!(matches!( + empty.build_volumes(None), + Err(rars::Error::InvalidArgument( + "archive builder has no entries" + )) + )); + + let mut multiple = Builder::new(ArchiveVersion::Rar20) + .store(true) + .volume_size(Some(1024)); + for name in [b"first".as_slice(), b"second".as_slice()] { + multiple + .add_bytes(name.to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + } + assert!(matches!( + multiple.build_volumes(None), + Err(rars::Error::InvalidArgument( + "legacy volumes support one input" + )) + )); +} + +#[test] +fn legacy_builder_rejects_modern_streaming_outputs_before_creating_files() { + let mut builder = Builder::new(ArchiveVersion::Rar20).store(true); + builder + .add_source( + b"file".to_vec(), + EntrySource::from_opener(7, || panic!("unsupported output must not open the source")), + None, + None, + ) + .unwrap(); + let resources = rars::WriterResources::default(); + assert!(matches!( + builder.to_volume_output(&resources, None), + Err(rars::Error::UnsupportedFamilyFeature { + feature: "streaming managed volume output", + .. + }) + )); + let mut sink = rars::rar50::CollectedVolumes::new(); + assert!(matches!( + builder.write_volumes_to(&mut sink, &resources, None), + Err(rars::Error::UnsupportedFamilyFeature { + feature: "streaming managed volume output", + .. + }) + )); + assert!(sink.take().is_empty()); + + let root = scratch::case("builder-legacy-memory-quota"); + let destination = root.join("archive.rar"); + let limited = resources.with_max_memory_bytes(1024); + assert!(matches!( + builder.write_to_path_with_resources(&destination, &limited, None), + Err(rars::Error::UnsupportedFamilyFeature { + feature: "aggregate managed memory quota", + .. + }) + )); + assert!(!destination.exists()); +} + +#[test] +fn legacy_builder_materializes_reopenable_sources_once() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + let payload = b"legacy source materialization ".repeat(32); + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ] { + let opens = Arc::new(AtomicUsize::new(0)); + let opened = opens.clone(); + let data = payload.clone(); + let mut builder = Builder::new(version).store(false); + builder + .add_source( + b"source".to_vec(), + EntrySource::from_opener(payload.len() as u64, move || { + opened.fetch_add(1, Ordering::SeqCst); + Ok(Box::new(std::io::Cursor::new(data.clone()))) + }), + None, + None, + ) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"source", None).unwrap().unwrap(), + payload + ); + assert_eq!(opens.load(Ordering::SeqCst), 1, "{version:?}"); + } +} + +#[cfg(unix)] +#[test] +fn add_path_refuses_symlinks_without_following_them() { + use std::os::unix::fs::symlink; + + let root = scratch::case("builder-input-symlink"); + let target = root.join("target"); + let link = root.join("link"); + fs::write(&target, b"private payload").unwrap(); + symlink(&target, &link).unwrap(); + let mut builder = Builder::new(ArchiveVersion::Rar50); + let error = builder.add_path(&link, b"member").unwrap_err(); + assert_eq!(error.entry_context(), Some((&b"member"[..], "adding"))); + assert_eq!(error.kind(), rars::ErrorKind::InvalidArgument); + assert!(builder.is_empty()); +} + +#[cfg(unix)] +#[test] +fn add_path_refuses_special_files_instead_of_silently_omitting_them() { + let mut builder = Builder::new(ArchiveVersion::Rar50); + let error = builder + .add_path(std::path::Path::new("/dev/null"), b"member") + .unwrap_err(); + assert_eq!(error.entry_context(), Some((&b"member"[..], "adding"))); + assert_eq!(error.kind(), rars::ErrorKind::InvalidArgument); + assert!(builder.is_empty()); +} diff --git a/arca-rar/vendor/rars/tests/builder_symlinks.rs b/arca-rar/vendor/rars/tests/builder_symlinks.rs new file mode 100644 index 00000000..2bb078a8 --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_symlinks.rs @@ -0,0 +1,460 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReader, ArchiveVersion, Builder}; + +#[test] +fn unix_symlinks_round_trip_without_resolving_targets() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for stored in [false, true] { + let mut builder = Builder::new(format).store(stored); + builder + .add_unix_symlink( + b"link".to_vec(), + b"../missing".to_vec(), + false, + Some(123), + Some(0o750), + ) + .unwrap(); + builder + .set_file_comment(b"link", Some(b"note".to_vec())) + .unwrap(); + builder + .add_unix_symlink( + b"directory-link".to_vec(), + b"missing-dir".to_vec(), + true, + None, + None, + ) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.rename(b"link", b"renamed".to_vec()).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + let members: Vec<_> = archive.members().collect(); + assert_eq!( + members[0].unix_symlink().unwrap().target_name, + b"../missing" + ); + assert_eq!(members[0].meta.file_attr, 0o120750); + assert_eq!(members[0].meta.file_time, Some(123)); + assert_eq!(members[1].unix_symlink().unwrap().flags, 1); + assert!(!members[1].meta.is_directory); + assert_eq!(archive.read_member_at(0, None).unwrap(), None); + assert_eq!( + archive.read_member_at(2, None).unwrap().unwrap(), + b"payload" + ); + assert_eq!( + archive.member_comment_at(0, None).unwrap(), + Some(b"note".to_vec()) + ); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } + } +} + +#[test] +fn archive_redirection_retains_unix_symlink_attributes() { + let mut seed = Builder::new(ArchiveVersion::Rar50).store(true); + seed.add_unix_symlink( + b"link".to_vec(), + b"missing".to_vec(), + false, + Some(123), + Some(0o750), + ) + .unwrap(); + let source = ArchiveReader::read_owned(seed.to_bytes().unwrap()).unwrap(); + let member = source.members().next().unwrap(); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder.add_archive_redirection(&member).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let link = archive.members().next().unwrap(); + assert_eq!(link.unix_symlink().unwrap().target_name, b"missing"); + assert_eq!(link.meta.host_os, Some(1)); + assert_eq!(link.meta.file_attr, 0o120750); + assert_eq!(link.meta.file_time, Some(123)); +} + +#[test] +fn unsupported_link_options_leave_entries_unchanged() { + let mut builder = Builder::new(ArchiveVersion::Rar50); + for target in [b"".as_slice(), b"bad\0target", b"\xff"] { + assert!(builder + .add_unix_symlink(b"link".to_vec(), target.to_vec(), false, None, None) + .is_err()); + assert!(builder.is_empty()); + } + builder + .add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + assert!(builder.set_dos_attributes(b"link", 0x20).is_err()); + assert!(builder.volume_size(Some(4096)).build_volumes(None).is_err()); + let mut volume = Builder::new(ArchiveVersion::Rar50).volume_size(Some(4096)); + assert!(volume + .add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .is_err()); + assert!(volume.is_empty()); + let mut legacy = Builder::new(ArchiveVersion::Rar15); + assert!(legacy + .add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .is_err()); + assert!(legacy.is_empty()); +} + +#[test] +fn preflight_rejects_unsupported_redirections_and_inconsistent_link_headers() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + let original = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for damage in 0..6 { + let mut archive = original.clone(); + let rars::Archive::Rar50Plus(a) = &mut archive else { + unreachable!() + }; + let file = a + .blocks + .iter_mut() + .find_map(|block| match block { + rars::rar50::Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + match damage { + 0 => file.redirection.as_mut().unwrap().redirection_type = 4, + 1 => file.redirection.as_mut().unwrap().flags = 2, + 2 => file.host_os = 0, + 3 => file.attributes = 0o100644, + 4 => file.unpacked_size = 1, + _ => file.file_flags |= 1, + } + assert!(archive.members().next().unwrap().unix_symlink().is_none()); + assert!(!archive.rewrite_preservation_issues().is_empty()); + } +} + +#[test] +fn zero_size_rar5_unix_link_header_preserves_its_target() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + let mut source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let rars::Archive::Rar50Plus(archive) = &mut source else { + unreachable!() + }; + let file = archive + .blocks + .iter_mut() + .find_map(|block| match block { + rars::rar50::Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + file.unpacked_size = 0; + + let member = source.members().next().unwrap(); + assert_eq!(member.unix_symlink().unwrap().target_name, b"target"); + assert!(source.rewrite_preservation_issues().is_empty()); + let mut rewritten = source.preserving_builder(None).unwrap(); + rewritten.add_archive_redirection(&member).unwrap(); + let output = ArchiveReader::read_owned(rewritten.to_bytes().unwrap()).unwrap(); + assert_eq!( + output + .members() + .next() + .unwrap() + .unix_symlink() + .unwrap() + .target_name, + b"target" + ); +} + +#[test] +fn links_survive_solid_and_encrypted_output_and_unix_byte_mapping() { + for solid in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .solid(solid) + .password(Some(b"secret".to_vec())); + let target = rars::filename::encode_rar50(b"native-\xff").into_owned(); + builder + .add_unix_symlink(b"link".to_vec(), target.clone(), false, None, None) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".repeat(100), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + assert_eq!(member.unix_symlink().unwrap().target_name, target); + assert_eq!(member.meta.unpacked_size, 8); + assert_eq!( + archive.read_member_at(1, Some(b"secret")).unwrap().unwrap(), + b"payload".repeat(100) + ); + } +} + +#[test] +fn retains_windows_links_and_file_copy_records() { + let mut seed = Builder::new(ArchiveVersion::Rar50).store(true); + seed.add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + let seed = ArchiveReader::read_owned(seed.to_bytes().unwrap()) + .unwrap() + .members() + .next() + .unwrap(); + for kind in [2, 3, 4, 5] { + let mut member = seed.clone(); + member.meta.host_os = Some(0); + member.meta.is_directory = kind == 3; + member.meta.file_attr = match kind { + 2 => 0x400, + 3 => 0x410, + _ => 0x20, + }; + member.meta.unpacked_size = if kind >= 4 { 7 } else { 0 }; + if let rars::ArchiveMemberDetail::Rar50Plus { + redirection: Some(link), + .. + } = &mut member.detail + { + link.redirection_type = kind; + link.flags = u64::from(kind == 3); + } + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"target".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.add_archive_redirection(&member).unwrap(); + let output = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!( + output.rewrite_preservation_issues().is_empty(), + "{kind}: {:?}", + output.rewrite_preservation_issues() + ); + let actual = output.members().nth(1).unwrap(); + assert_eq!( + actual.supported_redirection(), + member.supported_redirection() + ); + assert_eq!(actual.meta.unpacked_size, member.meta.unpacked_size); + assert_eq!(actual.meta.file_attr, member.meta.file_attr); + if kind >= 4 { + let mut broken = output.clone(); + if let rars::Archive::Rar50Plus(archive) = &mut broken { + if let rars::rar50::Block::File(file) = &mut archive.blocks[1] { + file.redirection.as_mut().unwrap().target_name = b"missing".to_vec(); + } + } + assert!( + broken + .rewrite_preservation_issues() + .iter() + .any(|issue| issue + .contains("missing, forward or inconsistent redirection target")) + ); + builder.rename(b"target", b"renamed".to_vec()).unwrap(); + let output = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + output + .members() + .nth(1) + .unwrap() + .supported_redirection() + .unwrap() + .target_name, + b"renamed" + ); + builder.remove(b"renamed").unwrap(); + assert!(builder.to_bytes().is_err()); + } + } +} + +#[test] +fn link_setters_refuse_shapes_the_target_format_cannot_write() { + for (volume, directory_target, target) in [ + (true, false, b"target".as_slice()), + (false, true, b"target".as_slice()), + (false, false, b"".as_slice()), + (false, false, b"bad\0target".as_slice()), + ] { + let mut builder = Builder::new(ArchiveVersion::Rar20).volume_size(volume.then_some(1024)); + assert!(builder + .add_unix_symlink( + b"link".to_vec(), + target.to_vec(), + directory_target, + None, + None, + ) + .is_err()); + assert!(builder.is_empty()); + } + + let mut seed = Builder::new(ArchiveVersion::Rar50).store(true); + seed.add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(seed.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + for format in [ArchiveVersion::Rar20, ArchiveVersion::Rar50] { + let mut builder = + Builder::new(format).volume_size((format == ArchiveVersion::Rar50).then_some(1024)); + assert!(builder.add_archive_redirection(&member).is_err()); + assert!(builder.is_empty()); + } +} + +#[test] +fn legacy_link_payloads_are_decoded_with_integrity_and_password_checks() { + for format in [ArchiveVersion::Rar29, ArchiveVersion::Rar40] { + let mut builder = Builder::new(format).password(Some(b"secret".to_vec())); + builder + .add_bytes( + b"link".to_vec(), + b"missing-\xff".to_vec(), + None, + Some(0o120750), + ) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, Some(0o100644)) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.legacy_symlink_targets(None).is_err()); + assert_eq!( + archive.legacy_symlink_targets(Some(b"secret")).unwrap(), + vec![Some(b"missing-\xff".to_vec()), None] + ); + assert!(archive.legacy_symlink_target_at(0, None).is_err()); + assert_eq!( + archive + .legacy_symlink_target_at(0, Some(b"secret")) + .unwrap(), + Some(b"missing-\xff".to_vec()) + ); + assert_eq!( + archive + .legacy_symlink_target_at(1, Some(b"secret")) + .unwrap(), + None + ); + assert!(archive + .legacy_symlink_target_at(2, Some(b"secret")) + .is_err()); + } +} + +#[test] +fn batch_link_targets_keep_member_order_and_raw_bytes() { + for format in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ] { + for solid in [false, true] { + let mut builder = Builder::new(format).solid(solid); + builder + .add_directory(b"directory".to_vec(), None, None) + .unwrap(); + builder + .add_unix_symlink( + b"link1".to_vec(), + b"../raw-\xff".to_vec(), + false, + None, + None, + ) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .add_unix_symlink(b"link2".to_vec(), b"/absolute".to_vec(), false, None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.legacy_symlink_targets(None).unwrap(), + vec![ + None, + Some(b"../raw-\xff".to_vec()), + None, + Some(b"/absolute".to_vec()) + ] + ); + } + } +} + +#[test] +fn batch_link_targets_reject_invalid_targets_and_corrupt_dependencies() { + use rars::ErrorKind; + for target in [b"".as_slice(), b"bad\0target"] { + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes(b"link".to_vec(), target.to_vec(), None, Some(0o120777)) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let error = archive.legacy_symlink_targets(None).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert_eq!(error.entry_context().unwrap().0, b"link"); + } + // These ordinary data entries intentionally carry native Unix link modes, + // exercising compressed solid links from existing archives too. + for solid in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar29).solid(solid); + builder + .add_bytes(b"first".to_vec(), b"prefix".repeat(100), None, None) + .unwrap(); + builder + .add_bytes( + b"link1".to_vec(), + b"target one".to_vec(), + None, + Some(0o120777), + ) + .unwrap(); + builder + .add_bytes( + b"link2".to_vec(), + b"target two".to_vec(), + None, + Some(0o120777), + ) + .unwrap(); + let data = builder.to_bytes().unwrap(); + let mut archive = ArchiveReader::read_owned(data).unwrap(); + // Alter the parsed expected CRC to isolate integrity failure from decoding. + if let rars::Archive::Rar15To40(legacy) = &mut archive { + if let rars::rar15_40::Block::File(file) = &mut legacy.blocks[0] { + file.file_crc ^= 1; + } else { + panic!("expected first file"); + } + } + let result = archive.legacy_symlink_targets(None); + if solid { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ChecksumMismatch); + } else { + assert_eq!( + result.unwrap(), + vec![ + None, + Some(b"target one".to_vec()), + Some(b"target two".to_vec()) + ] + ); + } + } +} diff --git a/arca-rar/vendor/rars/tests/builder_timestamps.rs b/arca-rar/vendor/rars/tests/builder_timestamps.rs new file mode 100644 index 00000000..7be07445 --- /dev/null +++ b/arca-rar/vendor/rars/tests/builder_timestamps.rs @@ -0,0 +1,441 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReader, ArchiveVersion, Builder}; +use std::time::{Duration, UNIX_EPOCH}; + +#[path = "support/scratch.rs"] +mod scratch; + +#[test] +fn source_metadata_preserves_unix_epoch_and_refuses_unrepresentable_seconds() { + let root = scratch::case("source-timestamp-boundaries"); + let file = std::fs::File::create(root.join("source")).unwrap(); + for (instant, expected_unix, expected_dos_year) in [ + (UNIX_EPOCH - Duration::from_secs(1), None, None), + (UNIX_EPOCH, Some(0), None), + ( + UNIX_EPOCH + Duration::from_secs(1_594_771_200), + Some(1_594_771_200), + Some(2020), + ), + ( + UNIX_EPOCH + Duration::from_secs(u64::from(u32::MAX)), + Some(u32::MAX), + Some(2106), + ), + ( + UNIX_EPOCH + Duration::from_secs(u64::from(u32::MAX) + 1), + None, + Some(2106), + ), + ] { + file.set_modified(instant).unwrap(); + let metadata = file.metadata().unwrap(); + assert_eq!(metadata.modified().unwrap(), instant); + assert_eq!(rars::timestamp::source_unix_mtime(&metadata), expected_unix); + let packed = rars::timestamp::source_dos_mtime(&metadata); + assert_eq!( + (packed != 0).then_some(1980 + (packed >> 25)), + expected_dos_year + ); + } + let current = rars::timestamp::current_filetime(); + assert!(current >= 116_444_736_000_000_000); + assert!(!rars::timestamp::format_filetime_utc(current).starts_with("0x")); +} + +#[test] +fn stored_timestamp_view_retains_family_presence_and_refinements() { + use rars::{ArchiveFamily, StoredTimestamp, TimeRefinement}; + for format in ArchiveVersion::ALL { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"time".to_vec(), vec![], Some(0), None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let mut meta = archive.members().next().unwrap().meta; + for raw in [0, 0x5022_1882] { + meta.file_time = Some(raw); + meta.mtime_refinement = Some(TimeRefinement { + add_second: true, + nanoseconds: 123, + }); + let before = meta.clone(); + let expected = match archive.family() { + ArchiveFamily::Rar50Plus => StoredTimestamp::UnixSeconds(raw), + _ => StoredTimestamp::DosLocal(raw), + }; + assert_eq!(meta.stored_modification_time(), Some(expected), "{format}"); + assert_eq!(meta, before); + } + meta.file_time = None; + assert_eq!(meta.stored_modification_time(), None); + } +} + +#[test] +fn fractional_mtime_survives_all_rar5_output_paths() { + let root = scratch::case("fractional-mtime"); + for (format, encrypted) in [ + (ArchiveVersion::Rar50, false), + (ArchiveVersion::Rar70, true), + ] { + let password = encrypted.then_some(b"secret".to_vec()); + let mut builder = Builder::new(format) + .password(password.clone()) + .header_encryption(encrypted); + builder + .add_bytes( + b"file".to_vec(), + b"timestamp payload".repeat(100), + Some(1_700_000_002), + None, + ) + .unwrap(); + builder.set_mtime_nanoseconds(b"file", 704_088_300).unwrap(); + let path = root.join(format!("{format}.rar")); + builder.write_to_path(&path, None).unwrap(); + let mut outputs = vec![builder.to_bytes().unwrap(), std::fs::read(path).unwrap()]; + outputs.extend(builder.volume_size(Some(4096)).build_volumes(None).unwrap()); + for bytes in outputs { + let archive = ArchiveReader::read_with_options( + &bytes, + password.as_deref().map_or_else( + rars::ArchiveReadOptions::default, + rars::ArchiveReadOptions::with_password, + ), + ) + .unwrap(); + let meta = archive.members().next().unwrap().meta; + assert_eq!( + meta.modification_time(), + Some(UNIX_EPOCH + Duration::new(1_700_000_002, 704_088_300)) + ); + archive + .extract_to(password.as_deref(), |meta| { + assert_eq!(meta.mtime_refinement.unwrap().nanoseconds, 704_088_300); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + } + } +} + +#[test] +fn split_volume_extraction_retains_fractional_mtime() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .volume_size(Some(512)); + builder + .add_bytes(b"file".to_vec(), vec![42; 2048], Some(123), None) + .unwrap(); + builder.set_mtime_nanoseconds(b"file", 987_654_321).unwrap(); + let volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + let mut opened = 0; + rars::extract_volumes_to(&volumes, None, |meta| { + opened += 1; + assert_eq!(meta.file_time, Some(123)); + assert_eq!(meta.mtime_refinement.unwrap().nanoseconds, 987_654_321); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + assert_eq!(opened, 1); +} + +#[test] +fn an_epoch_fraction_is_not_treated_as_missing_time() { + let detail = rars::TimeRefinement { + add_second: false, + nanoseconds: 123, + }; + assert_eq!( + rars::timestamp::extracted_system_time( + rars::ArchiveFamily::Rar50Plus, + Some(0), + Some(detail) + ), + Some(UNIX_EPOCH + Duration::from_nanos(123)) + ); + assert_eq!( + rars::timestamp::extracted_system_time(rars::ArchiveFamily::Rar50Plus, Some(0), None), + Some(UNIX_EPOCH) + ); + assert_eq!( + rars::timestamp::extracted_system_time(rars::ArchiveFamily::Rar50Plus, None, Some(detail)), + None + ); +} + +#[test] +fn invalid_fractional_mtime_leaves_queued_metadata_unchanged() { + for (format, seconds) in [ + (ArchiveVersion::Rar29, Some(123)), + (ArchiveVersion::Rar50, None), + (ArchiveVersion::Rar50, Some(123)), + ] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"file".to_vec(), vec![], seconds, None) + .unwrap(); + let before = builder.to_bytes().unwrap(); + assert!(builder + .set_mtime_nanoseconds(b"file", 1_000_000_000) + .is_err()); + assert!(builder.set_mtime_nanoseconds(b"missing", 100).is_err()); + if seconds.is_none() || format == ArchiveVersion::Rar29 { + assert!(builder.set_mtime_nanoseconds(b"file", 100).is_err()); + } + assert_eq!(builder.to_bytes().unwrap(), before); + } +} + +#[test] +fn extraction_retains_missing_and_epoch_times_in_single_and_split_archives() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for time in [None, Some(0)] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"file".to_vec(), vec![42; 2048], time, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let check = |meta: &rars::ExtractedEntryMeta| { + assert_eq!(meta.file_time, time); + assert_eq!(meta.mtime_refinement, None); + assert_eq!( + rars::timestamp::extracted_system_time( + rars::ArchiveFamily::Rar50Plus, + meta.file_time, + meta.mtime_refinement, + ), + time.map(|_| UNIX_EPOCH), + ); + }; + archive + .extract_to(None, |meta| { + check(meta); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + let volumes: Vec<_> = builder + .volume_size(Some(512)) + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + let mut opened = 0; + rars::extract_volumes_to(&volumes, None, |meta| { + opened += 1; + check(meta); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + assert_eq!(opened, 1); + } + } +} + +#[test] +fn complete_file_times_survive_streaming_and_volume_paths() { + use rars::{FileTimes, FileTimestamp}; + for times in [ + FileTimes { + modified: Some(FileTimestamp::WindowsFiletime(0)), + created: Some(FileTimestamp::WindowsFiletime(u64::MAX)), + accessed: Some(FileTimestamp::WindowsFiletime(116_444_736_000_000_001)), + }, + FileTimes { + modified: None, + created: Some(FileTimestamp::Unix { + seconds: 0, + nanoseconds: 123, + }), + accessed: Some(FileTimestamp::Unix { + seconds: u32::MAX, + nanoseconds: 999_999_999, + }), + }, + ] { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), Some(123), None) + .unwrap(); + builder.set_file_times(b"file", Some(times)).unwrap(); + let mut outputs = vec![builder.to_bytes().unwrap()]; + outputs.extend(builder.volume_size(Some(4096)).build_volumes(None).unwrap()); + for bytes in outputs { + let archive = ArchiveReader::read_owned(bytes).unwrap(); + assert_eq!( + archive.members().next().unwrap().file_times().unwrap(), + Some(times) + ); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } + } +} + +#[test] +fn adding_creation_time_preserves_an_existing_fractional_modification_time() { + use rars::{FileTimes, FileTimestamp}; + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), Some(123), None) + .unwrap(); + builder.set_mtime_nanoseconds(b"file", 456_789_123).unwrap(); + let before = builder.to_bytes().unwrap(); + let mixed = FileTimes { + modified: None, + created: Some(FileTimestamp::WindowsFiletime(116_444_736_000_000_001)), + accessed: None, + }; + assert!(builder.set_file_times(b"file", Some(mixed)).is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + let times = FileTimes { + modified: None, + created: Some(FileTimestamp::Unix { + seconds: 456, + nanoseconds: 0, + }), + accessed: None, + }; + builder.set_file_times(b"file", Some(times)).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.members().next().unwrap().file_times().unwrap(), + Some(FileTimes { + modified: Some(FileTimestamp::Unix { + seconds: 123, + nanoseconds: 456_789_123, + }), + ..times + }) + ); +} + +#[test] +fn invalid_complete_times_leave_builder_output_unchanged() { + use rars::{FileTimes, FileTimestamp}; + for format in [ArchiveVersion::Rar29, ArchiveVersion::Rar50] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), Some(123), None) + .unwrap(); + let before = builder.to_bytes().unwrap(); + let invalid = FileTimes { + modified: Some(FileTimestamp::Unix { + seconds: 123, + nanoseconds: 1_000_000_000, + }), + created: None, + accessed: None, + }; + assert!(builder.set_file_times(b"file", Some(invalid)).is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + if format == ArchiveVersion::Rar29 { + assert!(builder + .set_file_times(b"file", Some(FileTimes::default())) + .is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + } + } +} + +#[test] +fn clearing_complete_times_removes_the_rar5_time_record() { + use rars::{FileTimes, FileTimestamp}; + + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), Some(123), None) + .unwrap(); + builder + .set_file_times( + b"file", + Some(FileTimes { + modified: Some(FileTimestamp::Unix { + seconds: 456, + nanoseconds: 789, + }), + created: None, + accessed: None, + }), + ) + .unwrap(); + builder.set_file_times(b"file", None).unwrap(); + + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.members().next().unwrap().file_times().unwrap(), + None + ); +} + +#[test] +fn legacy_extended_times_are_validated_before_the_archive_is_written() { + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes( + b"file".to_vec(), + b"payload".to_vec(), + Some(0x5a21_0000), + None, + ) + .unwrap(); + let before = builder.to_bytes().unwrap(); + // No odd-second or fraction control bits may be present without the + // corresponding timestamp-present bit, in any of the four legacy slots. + for index in 0..4 { + for mode in 1..8u16 { + let flags = mode << (12 - index * 4); + assert_eq!( + builder.set_legacy_extended_times(b"file", Some(flags.to_le_bytes().to_vec())), + Err(rars::Error::InvalidArgument( + "legacy extended timestamps are incomplete or invalid" + )) + ); + assert_eq!(builder.to_bytes().unwrap(), before); + } + } + assert!(builder + .set_legacy_extended_times(b"file", Some(vec![0x00, 0x90])) + .is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + builder + .set_legacy_extended_times(b"file", Some(vec![0x00, 0x90, 0x01])) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive + .members() + .next() + .unwrap() + .file_times() + .unwrap() + .is_some()); + + let mut old = Builder::new(ArchiveVersion::Rar20).store(true); + old.add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert!(old + .set_legacy_extended_times(b"file", Some(vec![0x00, 0x90, 0x01])) + .is_err()); + + let mut volume = Builder::new(ArchiveVersion::Rar29) + .store(true) + .volume_size(Some(512)); + volume + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert!(volume + .set_legacy_extended_times(b"file", Some(vec![0x00, 0x90, 0x01])) + .is_err()); +} diff --git a/arca-rar/vendor/rars/tests/comment_options.rs b/arca-rar/vendor/rars/tests/comment_options.rs new file mode 100644 index 00000000..b112fbdf --- /dev/null +++ b/arca-rar/vendor/rars/tests/comment_options.rs @@ -0,0 +1,403 @@ +#![cfg(feature = "write")] + +use rars::{ + Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, ErrorKind, + ReadCancellation, +}; +use std::io::{self, Cursor, Read, Seek, SeekFrom}; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; + +const FORMATS: [ArchiveVersion; 9] = [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, +]; + +#[test] +fn legacy_encrypted_archive_comment_requires_comment_password_and_rar3_or_rar4() { + for (format, comment, password) in [ + (ArchiveVersion::Rar30, None, b"secret".as_slice()), + ( + ArchiveVersion::Rar30, + Some(b"comment".as_slice()), + b"".as_slice(), + ), + ( + ArchiveVersion::Rar20, + Some(b"comment".as_slice()), + b"secret".as_slice(), + ), + ( + ArchiveVersion::Rar29, + Some(b"comment".as_slice()), + b"secret".as_slice(), + ), + ] { + let mut builder = Builder::new(format) + .store(true) + .comment(comment.map(Vec::from)) + .archive_comment_password(Some(password.to_vec())); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let error = builder.to_bytes().unwrap_err(); + assert_eq!( + error.kind(), + ErrorKind::InvalidArgument, + "{format:?}: {error}" + ); + } +} + +fn bytes(format: ArchiveVersion, comment: Option<&[u8]>, encrypted: bool) -> Vec { + let mut builder = Builder::new(format).comment(comment.map(Vec::from)); + if encrypted { + builder = builder.archive_comment_password(Some(b"secret".to_vec())); + } + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder.to_bytes().unwrap() +} + +fn assert_limits(archive: &Archive, expected: &[u8], password: Option<&[u8]>) { + let defaults = ArchiveReadOptions::with_optional_password(password); + assert_eq!( + archive.comment(password).unwrap().as_deref(), + Some(expected) + ); + for total in [false, true] { + let mut options = defaults; + if total { + options.max_total_output_bytes = Some(expected.len() as u64); + } else { + options.max_member_output_bytes = Some(expected.len() as u64); + } + for _ in 0..2 { + assert_eq!( + archive.comment_with_options(options).unwrap().as_deref(), + Some(expected) + ); + } + if !expected.is_empty() { + if total { + options.max_total_output_bytes = Some(expected.len() as u64 - 1); + } else { + options.max_member_output_bytes = Some(expected.len() as u64 - 1); + } + assert_eq!( + archive.comment_with_options(options).unwrap_err().kind(), + ErrorKind::ResourceLimit + ); + } + } + let token = ReadCancellation::new(); + token.cancel(); + assert_eq!( + archive + .comment_with_options(defaults.with_cancellation(&token)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + assert_eq!( + archive.comment(password).unwrap().as_deref(), + Some(expected) + ); +} + +#[test] +fn comment_limits_cover_all_versions_empty_comments_and_fresh_budgets() { + for format in FORMATS { + for comment in [b"archive comment\n".as_slice(), b""] { + let archive = ArchiveReader::read_owned(bytes(format, Some(comment), false)).unwrap(); + assert_limits(&archive, comment, None); + } + let archive = ArchiveReader::read_owned(bytes(format, None, false)).unwrap(); + let options = ArchiveReadOptions::new().with_max_member_output_bytes(0); + assert_eq!(archive.comment_with_options(options).unwrap(), None); + let token = ReadCancellation::new(); + token.cancel(); + assert_eq!( + archive + .comment_with_options(options.with_cancellation(&token)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + } +} + +#[test] +fn encrypted_comment_admission_precedes_password_work() { + for format in [ + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let archive = + ArchiveReader::read_owned(bytes(format, Some(b"encrypted comment"), true)).unwrap(); + assert_eq!( + archive.comment(None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!( + archive + .comment_with_options(ArchiveReadOptions::new().with_max_member_output_bytes(0)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_limits(&archive, b"encrypted comment", Some(b"secret")); + } +} + +#[test] +fn historical_comment_fixtures_keep_their_contents_under_limits() { + for data in [ + include_bytes!("fixtures/rar13/COMMENT.RAR").as_slice(), + include_bytes!("fixtures/rar15_40/rar202/comment_nopsw.rar"), + include_bytes!("fixtures/rar15_40/rar300/with_comment_rar300.rar"), + include_bytes!("fixtures/rar50/with_comment.rar"), + ] { + let archive = ArchiveReader::read(data).unwrap(); + let expected = archive + .comment(None) + .unwrap() + .expect("fixture has archive comment"); + assert!(!expected.is_empty()); + assert_limits(&archive, &expected, None); + } +} + +#[test] +fn modern_comments_apply_dictionary_and_buffering_policy() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for encrypted in [false, true] { + let comment = b"repeatable text\n".repeat(100); + let stored = + ArchiveReader::read_owned(bytes(format, Some(&comment), encrypted)).unwrap(); + let password = encrypted.then_some(b"secret".as_slice()); + let defaults = ArchiveReadOptions::with_optional_password(password); + assert_eq!( + stored + .comment_with_options(defaults.with_rar50_dictionary_size_limit(0)) + .unwrap(), + Some(comment.clone()) + ); + // The writer emits stored comments. Exercise the shared compressed + // service payload representation using a normally encoded member. + let mut builder = Builder::new(format).password(password.map(Vec::from)); + builder + .add_bytes(b"CMT".to_vec(), comment.clone(), None, None) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + if let Archive::Rar50Plus(raw) = &mut archive { + for block in &mut raw.blocks { + if let rars::rar50::Block::File(file) = block { + assert!(!file.is_stored()); + *block = rars::rar50::Block::Service(file.clone()); + } + } + } + assert_limits(&archive, &comment, password); + assert_eq!( + archive + .comment_with_options(defaults.with_rar50_dictionary_size_limit(0)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!( + archive + .comment_with_options(defaults.with_rar50_dictionary_size_limit(u64::MAX)) + .unwrap(), + Some(comment.clone()) + ); + let options = defaults + .with_rar50_buffered_decode_limit(0) + .with_max_member_output_bytes(comment.len() as u64); + assert_eq!( + archive.comment_with_options(options).unwrap(), + Some(comment) + ); + } + } +} + +#[test] +fn unknown_size_modern_comments_reject_limits_before_decoding() { + let mut archive = ArchiveReader::read_owned(bytes( + ArchiveVersion::Rar50, + Some(b"unknown-size comment"), + false, + )) + .unwrap(); + let Archive::Rar50Plus(raw) = &mut archive else { + unreachable!() + }; + for block in &mut raw.blocks { + if let rars::rar50::Block::Service(file) = block { + if file.name == b"CMT" { + file.file_flags |= 0x8; + assert_eq!(file.known_unpacked_size(), None); + } + } + } + assert_eq!( + archive.comment(None).unwrap().unwrap(), + b"unknown-size comment" + ); + let error = archive + .comment_with_options(ArchiveReadOptions::new().with_max_member_output_bytes(100)) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + rars::Error::UnsupportedFeature { + feature: "output-limited decoding of an unknown-size comment", + .. + } + )); +} + +struct ObservedReader { + data: Cursor>, + armed: Arc, + reads: Arc, + cancellation: Option, +} +impl Read for ObservedReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if self.armed.load(Ordering::Relaxed) { + self.reads.fetch_add(1, Ordering::Relaxed); + if let Some(token) = &self.cancellation { + token.cancel(); + } + } + self.data.read(output) + } +} +impl Seek for ObservedReader { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.data.seek(from) + } +} + +#[test] +fn admission_precedes_source_reads_and_cancellation_survives_decoder_adapters() { + for format in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar30, + ArchiveVersion::Rar50, + ] { + let token = ReadCancellation::new(); + let armed = Arc::new(AtomicBool::new(false)); + let reads = Arc::new(AtomicUsize::new(0)); + let archive = ArchiveReader::read_reader(ObservedReader { + data: Cursor::new(bytes(format, Some(b"comment"), false)), + armed: armed.clone(), + reads: reads.clone(), + cancellation: Some(token.clone()), + }) + .unwrap(); + armed.store(true, Ordering::Relaxed); + assert_eq!( + archive + .comment_with_options(ArchiveReadOptions::new().with_max_total_output_bytes(0)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!(reads.load(Ordering::Relaxed), 0); + assert_eq!( + archive + .comment_with_options(ArchiveReadOptions::new().with_cancellation(&token)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + assert!(reads.load(Ordering::Relaxed) > 0); + } +} + +#[test] +fn unpacked_rar13_comment_uses_the_same_output_policy() { + let comment = b"plain legacy comment"; + let mut archive = + ArchiveReader::read_owned(bytes(ArchiveVersion::Rar13, Some(comment), false)).unwrap(); + if let Archive::Rar13(raw) = &mut archive { + // The writer chooses packed comments; construct the alternate plain + // comment representation in the parsed main header. + raw.main.flags &= !0x10; + raw.main.extra = (comment.len() as u16).to_le_bytes().to_vec(); + raw.main.extra.extend_from_slice(comment); + assert!(!raw.main.has_packed_comment()); + } + assert_limits(&archive, comment, None); +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +#[path = "support/scratch.rs"] +mod scratch; + +#[test] +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +fn filtered_comment_uses_bounded_scratch_and_cleans_up() { + use rars::rar50::{ArchiveEntry, Rar50Writer, WriterOptions}; + use rars::{EntrySource, FeatureSet, FilterKind, FilterPolicy, Rar50Scratch}; + let comment: Vec = [0xe8, 1, 0, 0, 0, 0xe9, 3, 0, 0, 0].repeat(1000); + let data = Rar50Writer::new(WriterOptions::new( + ArchiveVersion::Rar50, + FeatureSet::store_only(), + )) + .entries([ArchiveEntry::new( + b"CMT".to_vec(), + EntrySource::from_bytes(Arc::<[u8]>::from(comment.clone())), + )]) + .filter_policy(FilterPolicy::explicit(FilterKind::E8E9)) + .finish() + .unwrap(); + let mut archive = ArchiveReader::read_owned(data).unwrap(); + if let Archive::Rar50Plus(raw) = &mut archive { + for block in &mut raw.blocks { + if let rars::rar50::Block::File(file) = block { + *block = rars::rar50::Block::Service(file.clone()); + } + } + } + let options = ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(0) + .with_max_total_output_bytes(comment.len() as u64); + assert_eq!( + archive.comment_with_options(options).unwrap_err().kind(), + ErrorKind::ResourceLimit + ); + let dir = scratch::case("comment-scratch"); + let policy = Rar50Scratch::new(&*dir, 100_000).with_filter_memory_limit(64 * 1024); + assert_eq!( + archive + .comment_with_options(options.with_rar50_scratch(&policy)) + .unwrap(), + Some(comment) + ); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + let refused = Rar50Scratch::new(&*dir, 1); + assert_eq!( + archive + .comment_with_options(options.with_rar50_scratch(&refused)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); +} diff --git a/arca-rar/vendor/rars/tests/crypto_api.rs b/arca-rar/vendor/rars/tests/crypto_api.rs new file mode 100644 index 00000000..8bfd847d --- /dev/null +++ b/arca-rar/vendor/rars/tests/crypto_api.rs @@ -0,0 +1,79 @@ +#![cfg(feature = "encryption")] + +use rars::crypto::{rar30, rar50}; + +#[test] +fn key_equality_checks_every_byte_and_debug_keeps_material_private() { + let keys = rar50::Rar50Keys::derive(b"secret", [7; 16], 0).unwrap(); + assert_eq!(keys, keys.clone()); + assert_eq!(format!("{keys:?}"), "Rar50Keys { .. }"); + for index in 0..72 { + let mut changed = keys.clone(); + match index { + 0..32 => changed.key[index] ^= 1, + 32..64 => changed.hash_key[index - 32] ^= 1, + _ => changed.password_check[index - 64] ^= 1, + } + assert_ne!(keys, changed, "changed byte {index}"); + assert_ne!(changed, keys, "changed byte {index}"); + } +} + +#[test] +fn password_check_rejects_each_corrupted_byte_and_retains_old_zero_record_policy() { + let keys = rar50::Rar50Keys::derive(b"secret", [7; 16], 0).unwrap(); + let record = keys.password_check_record(); + assert_eq!(keys.check_password(&record), Ok(())); + assert_ne!(record[..8], [0; 8]); + for index in 0..12 { + let mut changed = record; + changed[index] ^= 1; + let error = keys.check_password(&changed).unwrap_err(); + assert_eq!(error, rar50::Error::BadPassword); + assert_eq!( + error.to_string(), + "wrong password or corrupt encrypted data" + ); + } + let mut absent = [0; 12]; + absent[8..].fill(0xa5); + assert_eq!(keys.check_password(&absent), Ok(())); +} + +#[test] +fn excessive_kdf_exponents_are_refused_before_derivation() { + for exponent in [25, u8::MAX] { + let error = rar50::Rar50Keys::derive(b"secret", [0; 16], exponent) + .err() + .unwrap(); + assert_eq!(error, rar50::Error::KdfCountTooLarge); + assert_eq!(error.to_string(), "RAR 5 KDF count is too large"); + } +} + +#[test] +fn legacy_non_utf8_password_error_has_a_useful_diagnostic() { + let error = rar30::Rar30Cipher::new(b"\xffpassword", None) + .err() + .unwrap(); + assert_eq!(error, rar30::Error::NonUtf8Password); + assert_eq!(error.to_string(), "RAR 3.x password is not UTF-8"); +} + +#[test] +fn partial_aes_blocks_are_refused_without_mutating_the_input() { + let mut data = [0x55; 15]; + let error = rar30::Rar30Cipher::new(b"secret", None) + .unwrap() + .decrypt_in_place(&mut data) + .unwrap_err(); + assert_eq!(error, rar30::Error::UnalignedInput); + assert_eq!(error.to_string(), "RAR 3.x AES input is not block aligned"); + assert_eq!(data, [0x55; 15]); + let error = rar50::Rar50Cipher::new([7; 32], [3; 16]) + .decrypt_in_place(&mut data) + .unwrap_err(); + assert_eq!(error, rar50::Error::UnalignedInput); + assert_eq!(error.to_string(), "RAR 5 AES input is not block aligned"); + assert_eq!(data, [0x55; 15]); +} diff --git a/arca-rar/vendor/rars/tests/extraction_control.rs b/arca-rar/vendor/rars/tests/extraction_control.rs new file mode 100644 index 00000000..37f18c5d --- /dev/null +++ b/arca-rar/vendor/rars/tests/extraction_control.rs @@ -0,0 +1,451 @@ +#![cfg(feature = "write")] + +use rars::{ + Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, Error, + ExtractionDecision as Decision, ExtractionOutcome as Outcome, +}; +use std::{cell::RefCell, io, io::Write, rc::Rc}; + +const VERSIONS: [ArchiveVersion; 8] = [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, +]; + +struct Capture(Rc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn archive(version: ArchiveVersion, solid: bool, encrypted: bool) -> Archive { + let mut builder = Builder::new(version) + .solid(solid) + .password(encrypted.then(|| b"secret".to_vec())); + for (name, len) in [(b"first".as_slice(), 4096), (b"second", 32), (b"third", 64)] { + builder + .add_bytes(name.to_vec(), vec![42; len], None, None) + .unwrap(); + } + ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() +} + +#[test] +fn skip_precedes_limits_and_stop_precedes_later_members() { + for version in VERSIONS { + let archive = archive(version, false, false); + let output = Rc::new(RefCell::new(Vec::new())); + let mut visited = Vec::new(); + let outcome = archive + .extract_with_control( + ArchiveReadOptions::new() + .with_max_member_output_bytes(32) + .with_max_total_output_bytes(32), + |member| { + visited.push(member.meta.name.clone()); + Ok(match member.meta.name.as_slice() { + b"first" => { + assert_eq!(member.meta.unpacked_size, 4096); + Decision::Skip + } + b"second" => Decision::Extract(Box::new(Capture(output.clone()))), + _ => Decision::Stop, + }) + }, + ) + .unwrap(); + assert_eq!(outcome, Outcome::Stopped); + assert_eq!(*output.borrow(), vec![42; 32]); + assert_eq!( + visited, + [b"first".to_vec(), b"second".to_vec(), b"third".to_vec()] + ); + let mut calls = 0; + assert_eq!( + archive + .extract_with_control(ArchiveReadOptions::new(), |_| { + calls += 1; + Ok(Decision::Stop) + }) + .unwrap(), + Outcome::Stopped + ); + assert_eq!(calls, 1); + } +} + +#[test] +fn encrypted_members_can_be_skipped_without_a_password() { + for version in VERSIONS.into_iter().filter(|v| *v != ArchiveVersion::Rar14) { + let archive = archive(version, false, true); + let mut calls = 0; + let outcome = archive + .extract_with_control(ArchiveReadOptions::new(), |member| { + assert!(member.meta.is_encrypted); + calls += 1; + Ok(Decision::Skip) + }) + .unwrap(); + assert_eq!(outcome, Outcome::Complete); + assert_eq!(calls, 3); + } +} + +#[test] +fn solid_skipping_is_refused_but_discarding_preserves_history_and_accounting() { + for version in VERSIONS { + let archive = archive(version, true, false); + let error = archive + .extract_with_control(ArchiveReadOptions::new(), |_| Ok(Decision::Skip)) + .unwrap_err(); + assert!(matches!(error.root_cause(), Error::CannotSkipSolidMember)); + assert_eq!(error.entry_context().unwrap().0, b"first"); + let output = Rc::new(RefCell::new(Vec::new())); + assert_eq!( + archive + .extract_with_control(ArchiveReadOptions::new(), |member| { + Ok(Decision::Extract(if member.meta.name == b"first" { + Box::new(io::sink()) as Box + } else { + Box::new(Capture(output.clone())) + })) + }) + .unwrap(), + Outcome::Complete + ); + assert_eq!(*output.borrow(), vec![42; 96]); + let error = archive + .extract_with_control( + ArchiveReadOptions::new().with_max_total_output_bytes(4096), + |_| Ok(Decision::Extract(Box::new(io::sink()))), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ResourceLimit); + assert_eq!(error.entry_context().unwrap().0, b"second"); + } +} + +#[test] +fn callback_cancellation_and_errors_do_not_become_successful_stops() { + let archive = archive(ArchiveVersion::Rar50, false, false); + let token = rars::ReadCancellation::new(); + let error = archive + .extract_with_control(ArchiveReadOptions::new().with_cancellation(&token), |_| { + token.cancel(); + Ok(Decision::Stop) + }) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Cancelled); + let mut calls = 0; + let error = archive + .extract_with_control(ArchiveReadOptions::new(), |_| { + calls += 1; + Err(io::Error::from(io::ErrorKind::PermissionDenied).into()) + }) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert_eq!(calls, 1); +} + +#[test] +fn skipping_a_bad_checksum_continues_but_extracting_it_stops() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut archive = archive(version, false, false); + match &mut archive { + Archive::Rar13(a) => a.entries[0].header.file_crc ^= 1, + Archive::Rar15To40(a) => { + for block in &mut a.blocks { + if let rars::rar15_40::Block::File(file) = block { + if file.name == b"first" { + file.file_crc ^= 1; + } + } + } + } + Archive::Rar50Plus(a) => { + for block in &mut a.blocks { + if let rars::rar50::Block::File(file) = block { + if file.name == b"first" { + file.hash = None; + file.data_crc32 = Some(file.data_crc32.unwrap() ^ 1); + } + } + } + } + _ => unreachable!(), + } + let token = rars::ReadCancellation::new(); + let mut selected = 0; + let mut failures = 0; + let error = archive + .extract_with_control_and_errors( + ArchiveReadOptions::new().with_cancellation(&token), + |_| { + selected += 1; + Ok(Decision::Extract(Box::new(io::sink()))) + }, + |_, error| { + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + failures += 1; + token.cancel(); + Ok(rars::ExtractionErrorAction::Continue) + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Cancelled); + assert_eq!((selected, failures), (1, 1)); + let mut selected = 0; + let mut failures = 0; + let error = archive + .extract_with_control_and_errors( + ArchiveReadOptions::new(), + |_| { + selected += 1; + Ok(Decision::Extract(Box::new(io::sink()))) + }, + |member, error| { + assert_eq!(member.meta.name, b"first"); + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + failures += 1; + Ok(rars::ExtractionErrorAction::Abort) + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + assert_eq!(error.entry_context().unwrap().0, b"first"); + assert_eq!((selected, failures), (1, 1)); + let output = Rc::new(RefCell::new(Vec::new())); + assert_eq!( + archive + .extract_with_control(ArchiveReadOptions::new(), |member| { + Ok(if member.meta.name == b"first" { + Decision::Skip + } else { + Decision::Extract(Box::new(Capture(output.clone()))) + }) + }) + .unwrap(), + Outcome::Complete + ); + assert_eq!(*output.borrow(), vec![42; 96]); + let mut calls = 0; + let error = archive + .extract_with_control(ArchiveReadOptions::new(), |_| { + calls += 1; + Ok(Decision::Extract(Box::new(io::sink()))) + }) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + assert_eq!(calls, 1); + + for limit in [4096, 4192] { + let output = Rc::new(RefCell::new(Vec::new())); + let mut failures = 0; + let result = archive.extract_with_control_and_errors( + ArchiveReadOptions::new() + .with_max_total_output_bytes(limit) + .with_rar50_buffered_decode_limit(0), + |member| { + Ok(Decision::Extract(if member.meta.name == b"first" { + Box::new(io::sink()) as Box + } else { + Box::new(Capture(output.clone())) + })) + }, + |member, error| { + assert_eq!(member.meta.name, b"first"); + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + failures += 1; + Ok(rars::ExtractionErrorAction::Continue) + }, + ); + assert_eq!(failures, 1); + if limit == 4096 { + let error = result.unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ResourceLimit); + assert_eq!(error.entry_context().unwrap().0, b"second"); + assert!(output.borrow().is_empty()); + } else { + assert_eq!(result.unwrap(), Outcome::Complete); + assert_eq!(*output.borrow(), vec![42; 96]); + } + } + } +} + +#[test] +fn solid_failures_and_callback_errors_cannot_be_continued() { + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::PermissionDenied.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for version in VERSIONS { + let archive = archive(version, true, false); + let error = archive + .extract_with_control_and_errors( + ArchiveReadOptions::new(), + |_| Ok(Decision::Extract(Box::new(Broken))), + |_, _| panic!("solid failure must not offer recovery"), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + } + let archive = archive(ArchiveVersion::Rar50, false, false); + let error = archive + .extract_with_control_and_errors( + ArchiveReadOptions::new(), + |_| Err(io::Error::other("callback failed").into()), + |_, _| panic!("decision callback errors are fatal"), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); +} + +#[test] +fn split_members_never_offer_independent_error_recovery() { + // These flags are exposed in the parsed archive model. A split payload + // requires the multivolume reader, even if an error callback asks to continue. + for flags in [0x0001, 0x0002, 0x0003] { + let mut archive = archive(ArchiveVersion::Rar29, false, false); + let Archive::Rar15To40(legacy) = &mut archive else { + unreachable!() + }; + let file = legacy + .blocks + .iter_mut() + .find_map(|block| match block { + rars::rar15_40::Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + file.block.flags |= flags; + let mut selected = 0; + let error = archive + .extract_with_control_and_errors( + ArchiveReadOptions::new(), + |_| { + selected += 1; + Ok(Decision::Extract(Box::new(io::sink()))) + }, + |_, _| panic!("split payloads cannot be recovered independently"), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::InvalidArchive); + assert_eq!(selected, 1); + } +} + +#[test] +fn selector_errors_preserve_existing_entry_context() { + for version in VERSIONS { + let archive = archive(version, false, false); + let mut calls = 0; + let error = archive + .extract_with_control(ArchiveReadOptions::new(), |_| { + calls += 1; + Err(Error::InvalidHeader("selection failed") + .at_entry(b"external entry".to_vec(), "external operation")) + }) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::InvalidArchive); + assert_eq!( + error.entry_context(), + Some((b"external entry".as_slice(), "external operation")) + ); + assert_eq!(calls, 1); + } +} + +#[test] +fn skip_and_stop_do_not_read_payloads_from_a_caller_source() { + use std::io::{Cursor, Read, Seek, SeekFrom}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + struct Source(Cursor>, Arc); + impl Read for Source { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.1.load(Ordering::Relaxed) { + return Err(io::ErrorKind::PermissionDenied.into()); + } + self.0.read(bytes) + } + } + impl Seek for Source { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.0.seek(from) + } + } + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), vec![42; 1024], None, None) + .unwrap(); + let fail = Arc::new(AtomicBool::new(false)); + let archive = ArchiveReader::read_reader(Source( + Cursor::new(builder.to_bytes().unwrap()), + fail.clone(), + )) + .unwrap(); + fail.store(true, Ordering::Relaxed); + assert_eq!( + archive + .extract_with_control(ArchiveReadOptions::new(), |_| Ok(Decision::Skip)) + .unwrap(), + Outcome::Complete + ); + assert_eq!( + archive + .extract_with_control(ArchiveReadOptions::new(), |_| Ok(Decision::Stop)) + .unwrap(), + Outcome::Stopped + ); + } +} + +#[test] +fn controlled_directory_extraction_selects_a_writer_without_spending_output_quota() { + for version in VERSIONS { + let mut builder = Builder::new(version); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let output = Rc::new(RefCell::new(Vec::new())); + let mut visits = 0; + let outcome = archive + .extract_with_control( + ArchiveReadOptions::new().with_max_total_output_bytes(0), + |member| { + assert!(member.meta.is_directory); + visits += 1; + Ok(Decision::Extract(Box::new(Capture(output.clone())))) + }, + ) + .unwrap(); + assert_eq!(outcome, Outcome::Complete); + assert_eq!(visits, 1); + assert!(output.borrow().is_empty()); + } +} diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_comment.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_comment.rar new file mode 100644 index 00000000..9ad9c750 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_comment.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_comment_metadata.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_comment_metadata.rar new file mode 100644 index 00000000..b6fd420d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_comment_metadata.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_quick_open.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_quick_open.rar new file mode 100644 index 00000000..ec8609f2 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_quick_open.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_rar50.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_rar50.rar new file mode 100644 index 00000000..247429a4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_rar50.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_rar70.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_rar70.rar new file mode 100644 index 00000000..247429a4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_rar70.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_1.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_1.rar new file mode 100644 index 00000000..bc425111 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_10.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_10.rar new file mode 100644 index 00000000..f5666a41 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_10.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_5.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_5.rar new file mode 100644 index 00000000..4b946253 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_5.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_50.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_50.rar new file mode 100644 index 00000000..bafa0801 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_50.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_large.sha256 b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_large.sha256 new file mode 100644 index 00000000..14a3f7af --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/golden/stored_recovery_large.sha256 @@ -0,0 +1 @@ +f385e568c5a02c43590c071d7edc1f9dfc8cb94e2e6caabeffd6b59ee892be0b 371530 bytes diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_0.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_0.rar new file mode 100644 index 00000000..cb588172 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_0.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_1.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_1.rar new file mode 100644 index 00000000..0c96917b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_2.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_2.rar new file mode 100644 index 00000000..b49595ad Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_3.rar b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_3.rar new file mode 100644 index 00000000..8dfd530c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/golden/stored_volume_3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/BIG80K.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/BIG80K.RAR new file mode 100644 index 00000000..d9897ef1 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/BIG80K.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTI.TXT b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTI.TXT new file mode 100644 index 00000000..86a83683 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTI.TXT @@ -0,0 +1 @@ +beta alpha epsilon delta delta gamma beta beta eta alpha alpha beta delta delta alpha delta eta delta theta epsilon alpha gamma eta zeta epsilon gamma delta zeta beta beta eta beta zeta zeta epsilon alpha theta beta eta beta epsilon zeta delta beta alpha delta epsilon beta delta beta eta epsilon theta zeta gamma zeta zeta delta epsilon beta gamma delta gamma theta eta epsilon delta zeta alpha delta alpha zeta eta epsilon beta delta zeta delta theta eta theta gamma epsilon gamma delta epsilon eta eta zeta delta gamma theta beta alpha beta gamma gamma eta beta eta eta theta epsilon alpha beta epsilon zeta beta epsilon eta gamma theta alpha epsilon gamma beta epsilon delta gamma zeta gamma alpha zeta theta alpha beta zeta epsilon delta alpha delta beta beta theta beta gamma gamma theta gamma epsilon eta delta delta epsilon eta zeta theta theta beta delta delta beta zeta alpha delta delta alpha beta alpha delta beta alpha zeta beta delta epsilon theta delta gamma theta delta theta eta delta beta beta eta zeta eta eta theta alpha beta alpha eta zeta beta delta delta delta theta gamma eta gamma epsilon theta delta beta theta beta alpha alpha beta delta gamma eta theta theta delta eta alpha gamma eta alpha eta epsilon theta epsilon eta theta gamma delta epsilon delta alpha alpha zeta alpha alpha theta gamma alpha beta gamma beta beta delta eta beta delta alpha beta eta zeta epsilon delta zeta delta epsilon eta gamma epsilon theta zeta beta alpha theta beta beta delta epsilon gamma zeta beta delta zeta epsilon gamma theta epsilon alpha epsilon beta gamma epsilon beta beta gamma epsilon epsilon delta zeta delta epsilon theta epsilon alpha beta eta epsilon alpha alpha zeta gamma epsilon gamma theta eta alpha beta beta gamma alpha zeta gamma eta gamma alpha epsilon zeta alpha zeta delta delta beta zeta eta gamma delta gamma gamma eta alpha gamma zeta eta delta epsilon gamma beta eta alpha theta delta delta theta zeta epsilon delta delta alpha delta eta zeta epsilon beta epsilon zeta eta zeta alpha beta epsilon gamma epsilon alpha beta eta zeta zeta eta beta eta delta epsilon alpha eta alpha delta zeta eta beta zeta zeta beta epsilon epsilon eta zeta eta epsilon gamma delta eta eta gamma epsilon eta alpha epsilon epsilon delta eta zeta theta theta theta delta theta gamma beta epsilon zeta beta delta epsilon delta delta gamma alpha alpha delta theta beta theta eta delta eta theta eta delta gamma alpha beta eta delta gamma theta alpha delta beta theta gamma theta zeta theta eta theta gamma theta theta epsilon delta epsilon theta delta epsilon theta beta epsilon delta epsilon zeta zeta beta gamma gamma delta eta gamma delta beta eta eta zeta theta eta alpha delta eta eta alpha eta theta alpha zeta epsilon eta eta delta theta delta epsilon eta theta alpha eta zeta eta gamma theta gamma alpha eta alpha beta eta gamma theta gamma alpha epsilon eta zeta delta theta zeta zeta eta epsilon eta epsilon beta theta alpha alpha zeta delta beta alpha alpha delta delta alpha gamma delta gamma theta beta delta theta epsilon zeta gamma beta gamma epsilon beta alpha epsilon eta eta delta beta delta beta epsilon beta alpha zeta eta zeta beta zeta alpha eta theta beta eta zeta theta gamma eta gamma epsilon theta theta eta epsilon zeta delta beta epsilon theta delta theta eta zeta alpha theta zeta gamma theta delta zeta epsilon zeta epsilon epsilon alpha delta beta delta eta theta delta theta theta theta alpha beta epsilon delta eta delta epsilon zeta theta zeta eta zeta zeta theta epsilon epsilon epsilon delta beta delta zeta beta gamma delta delta theta epsilon epsilon beta delta epsilon delta zeta gamma epsilon alpha gamma epsilon alpha alpha epsilon gamma theta beta alpha epsilon theta theta theta zeta gamma alpha epsilon theta beta beta eta theta beta alpha gamma gamma epsilon beta delta beta eta delta eta theta theta epsilon eta epsilon alpha beta delta delta epsilon beta gamma delta gamma beta gamma alpha eta theta theta epsilon alpha delta epsilon epsilon theta beta delta epsilon delta eta beta delta gamma epsilon gamma beta alpha gamma epsilon epsilon theta beta theta epsilon eta epsilon theta theta beta alpha eta zeta epsilon alpha beta delta alpha epsilon alpha gamma theta theta epsilon gamma eta theta beta theta zeta eta zeta zeta beta gamma zeta eta theta epsilon eta alpha theta beta zeta epsilon zeta beta eta alpha theta eta alpha delta zeta theta theta alpha delta epsilon gamma epsilon theta theta beta alpha delta gamma epsilon alpha eta beta delta beta theta beta gamma theta epsilon epsilon eta theta theta delta theta gamma eta delta gamma beta epsilon eta zeta epsilon alpha epsilon epsilon theta gamma theta theta zeta zeta eta theta zeta delta delta eta delta eta alpha zeta theta eta eta gamma theta alpha gamma zeta beta theta beta theta alpha gamma eta gamma beta theta epsilon zeta eta beta zeta eta zeta theta alpha beta delta epsilon delta beta eta beta beta theta gamma epsilon alpha alpha zeta alpha epsilon zeta zeta eta gamma delta eta gamma gamma gamma beta eta delta theta gamma delta theta epsilon theta epsilon alpha theta epsilon gamma beta theta zeta epsilon eta epsilon theta epsilon delta eta theta beta delta eta zeta epsilon epsilon alpha eta epsilon alpha alpha theta epsilon delta zeta delta delta epsilon gamma beta alpha epsilon theta alpha zeta gamma beta epsilon zeta eta gamma delta gamma zeta epsilon gamma epsilon theta epsilon zeta beta theta beta gamma delta eta zeta beta eta alpha epsilon beta theta zeta epsilon eta zeta beta delta theta alpha zeta delta beta theta epsilon eta beta gamma alpha alpha epsilon theta beta beta delta gamma eta theta zeta eta gamma eta beta theta eta epsilon alpha zeta delta theta theta delta zeta beta zeta zeta alpha eta epsilon delta beta theta beta delta alpha alpha zeta delta gamma delta beta delta delta delta zeta gamma alpha epsilon gamma gamma epsilon gamma beta alpha gamma alpha zeta delta zeta alpha gamma epsilon alpha gamma eta beta beta theta theta zeta beta theta delta alpha epsilon theta alpha alpha theta eta eta beta theta theta beta beta zeta gamma beta gamma epsilon zeta eta epsilon theta eta beta beta delta eta theta delta eta zeta theta eta eta beta zeta eta zeta epsilon zeta gamma theta beta beta beta beta eta beta zeta gamma alpha zeta beta eta zeta eta alpha epsilon epsilon zeta beta delta gamma theta delta beta zeta zeta beta epsilon delta eta alpha epsilon alpha gamma epsilon epsilon zeta zeta alpha gamma gamma eta beta gamma alpha beta delta eta eta theta zeta gamma zeta epsilon zeta beta alpha gamma gamma alpha beta epsilon theta eta theta theta eta epsilon delta beta zeta eta beta epsilon theta epsilon alpha delta eta alpha alpha delta epsilon delta gamma epsilon epsilon zeta beta alpha theta eta gamma gamma eta delta zeta beta eta alpha eta alpha theta beta zeta eta eta eta epsilon beta eta alpha zeta gamma theta zeta beta eta beta delta eta eta beta eta epsilon zeta delta zeta gamma beta beta delta zeta zeta gamma delta beta gamma epsilon delta gamma gamma beta gamma theta theta theta zeta zeta gamma theta beta theta theta epsilon epsilon alpha zeta beta epsilon theta theta alpha alpha zeta epsilon beta beta eta theta alpha theta delta zeta theta gamma alpha theta beta zeta beta gamma alpha delta theta theta gamma zeta zeta epsilon eta eta zeta alpha zeta beta zeta beta eta epsilon epsilon gamma zeta beta gamma zeta epsilon eta gamma beta epsilon eta zeta gamma beta eta zeta alpha zeta epsilon gamma delta zeta theta delta delta gamma gamma beta epsilon beta alpha zeta gamma eta gamma gamma gamma gamma theta alpha eta zeta delta theta epsilon theta delta delta epsilon theta delta zeta theta theta epsilon eta eta gamma delta gamma epsilon alpha theta zeta beta beta epsilon beta gamma epsilon theta gamma eta beta delta theta zeta alpha eta alpha eta zeta delta eta beta zeta delta alpha zeta beta zeta gamma gamma alpha epsilon theta gamma theta theta alpha beta alpha epsilon delta gamma eta beta epsilon delta epsilon beta alpha delta eta theta beta beta theta alpha delta gamma epsilon eta alpha zeta delta eta gamma beta zeta beta alpha eta theta alpha eta zeta epsilon alpha zeta beta zeta delta beta zeta gamma alpha zeta zeta gamma theta theta gamma gamma beta theta alpha epsilon delta alpha delta alpha zeta epsilon eta theta epsilon alpha delta epsilon zeta alpha zeta epsilon beta zeta eta eta theta eta zeta gamma theta theta zeta epsilon beta eta beta eta gamma epsilon zeta beta beta zeta epsilon epsilon theta eta gamma theta zeta theta alpha zeta eta epsilon alpha beta eta zeta gamma alpha gamma theta alpha gamma beta delta zeta zeta eta alpha gamma theta zeta zeta theta beta gamma zeta eta zeta epsilon delta beta alpha gamma theta eta beta epsilon epsilon theta delta epsilon theta delta beta gamma beta theta gamma theta beta zeta zeta beta epsilon epsilon gamma gamma zeta delta beta delta gamma delta theta alpha zeta zeta theta gamma beta beta epsilon eta theta eta eta beta gamma zeta beta theta theta zeta gamma gamma gamma eta alpha beta gamma epsilon gamma gamma zeta delta zeta epsilon beta epsilon delta epsilon gamma epsilon beta gamma gamma gamma zeta alpha alpha beta alpha epsilon delta eta alpha theta epsilon epsilon theta delta eta epsilon theta beta alpha gamma theta eta theta theta delta zeta gamma zeta zeta zeta eta gamma zeta beta zeta delta theta beta epsilon theta delta gamma beta alpha epsilon eta eta delta gamma zeta zeta delta gamma theta theta theta epsilon theta alpha beta eta theta delta delta zeta alpha alpha epsilon theta theta epsilon alpha beta eta gamma epsilon zeta eta zeta alpha eta alpha delta zeta epsilon beta eta theta epsilon beta gamma beta eta zeta zeta zeta gamma delta eta alpha alpha alpha gamma zeta theta theta gamma gamma zeta zeta gamma eta epsilon zeta theta epsilon theta alpha zeta zeta beta eta epsilon alpha theta epsilon delta alpha theta gamma eta gamma delta alpha beta delta alpha theta zeta eta gamma eta delta eta theta alpha gamma delta zeta theta eta zeta gamma theta zeta zeta epsilon theta delta delta epsilon epsilon delta epsilon epsilon delta theta zeta theta zeta epsilon epsilon beta eta eta zeta gamma epsilon alpha epsilon beta zeta theta epsilon theta delta delta epsilon epsilon gamma beta delta delta alpha delta delta alpha beta eta zeta theta beta gamma alpha gamma eta theta theta delta epsilon zeta epsilon alpha beta delta alpha gamma eta gamma alpha eta theta gamma epsilon alpha alpha epsilon beta zeta epsilon theta theta gamma theta epsilon delta beta zeta gamma theta epsilon gamma alpha zeta epsilon delta gamma eta eta zeta beta eta beta gamma gamma theta zeta delta alpha epsilon eta delta theta epsilon zeta epsilon alpha epsilon zeta delta alpha beta theta epsilon gamma eta epsilon beta epsilon zeta delta delta gamma theta gamma theta zeta eta theta delta delta beta theta zeta beta beta alpha delta gamma gamma zeta theta beta delta theta beta theta alpha theta gamma eta theta alpha theta epsilon alpha eta epsilon alpha delta beta alpha eta zeta beta alpha beta theta alpha epsilon eta gamma gamma eta zeta eta theta eta eta beta gamma zeta beta gamma eta gamma delta alpha alpha epsilon theta eta eta delta delta theta zeta gamma epsilon delta beta alpha eta alpha delta delta beta beta alpha theta alpha delta alpha eta theta delta delta alpha gamma epsilon delta zeta zeta delta epsilon gamma delta eta epsilon epsilon alpha gamma eta theta alpha zeta eta zeta eta eta gamma epsilon eta gamma theta delta delta epsilon gamma theta alpha eta eta gamma eta delta epsilon delta zeta beta theta zeta beta delta alpha epsilon eta alpha beta delta delta theta delta zeta epsilon alpha delta delta beta theta delta delta eta delta zeta epsilon eta theta zeta epsilon epsilon zeta theta theta beta theta zeta delta zeta zeta eta alpha delta gamma alpha epsilon eta epsilon gamma delta zeta delta eta delta theta zeta epsilon theta theta theta gamma zeta gamma gamma theta gamma alpha alpha beta alpha alpha eta gamma delta beta gamma alpha delta theta zeta alpha theta theta alpha alpha eta alpha alpha epsilon epsilon alpha eta gamma beta beta gamma delta delta epsilon zeta epsilon eta beta zeta eta theta delta delta epsilon beta alpha beta eta eta eta theta alpha alpha gamma beta theta eta zeta beta alpha delta delta theta epsilon alpha beta epsilon alpha gamma zeta alpha delta gamma eta beta epsilon gamma delta eta zeta eta gamma beta zeta alpha beta eta delta beta zeta eta zeta alpha epsilon theta theta delta zeta eta eta gamma eta beta epsilon delta beta beta epsilon gamma eta gamma eta zeta zeta beta beta alpha epsilon theta zeta epsilon beta gamma beta gamma eta theta eta beta alpha beta zeta beta zeta eta alpha epsilon eta eta beta delta gamma eta gamma gamma epsilon epsilon epsilon theta gamma beta gamma eta epsilon eta epsilon theta beta zeta epsilon delta theta delta theta beta gamma epsilon alpha eta zeta eta zeta theta zeta eta gamma epsilon zeta delta theta zeta gamma eta zeta epsilon theta delta zeta eta epsilon eta zeta beta delta gamma alpha theta delta theta epsilon beta eta theta gamma epsilon delta epsilon gamma eta eta beta theta theta eta eta alpha zeta beta beta delta zeta gamma alpha eta zeta eta beta alpha beta epsilon delta delta theta zeta eta theta gamma zeta alpha theta beta epsilon eta beta beta gamma zeta epsilon zeta theta delta theta zeta theta beta theta theta zeta beta epsilon alpha beta alpha zeta beta gamma delta gamma gamma zeta eta theta delta eta gamma gamma eta eta alpha delta theta eta eta alpha delta delta epsilon beta beta gamma zeta zeta delta theta beta epsilon theta zeta eta eta beta zeta zeta theta gamma epsilon beta gamma zeta beta delta epsilon beta gamma zeta gamma eta eta gamma eta eta gamma theta gamma gamma theta epsilon gamma gamma zeta theta alpha zeta alpha theta gamma delta eta theta eta theta eta theta theta gamma beta alpha delta epsilon alpha epsilon delta epsilon gamma theta theta beta beta epsilon zeta alpha theta delta eta beta delta epsilon alpha theta epsilon zeta beta theta beta delta delta zeta eta gamma gamma delta delta alpha zeta epsilon gamma zeta epsilon epsilon epsilon beta gamma eta alpha epsilon gamma gamma delta gamma zeta delta eta theta gamma epsilon eta eta theta beta beta eta epsilon zeta theta theta zeta alpha beta theta zeta beta eta delta eta delta theta epsilon zeta epsilon zeta gamma theta zeta alpha alpha beta theta alpha beta gamma theta theta alpha eta delta gamma epsilon gamma epsilon beta zeta epsilon beta zeta gamma alpha eta epsilon delta eta beta beta alpha delta theta beta gamma delta theta alpha alpha zeta beta eta gamma theta beta delta eta beta beta beta zeta zeta epsilon gamma eta gamma gamma beta alpha gamma theta zeta delta gamma eta theta delta beta beta gamma beta eta zeta eta zeta gamma delta epsilon beta delta epsilon alpha epsilon delta delta eta epsilon alpha delta theta eta beta delta theta beta alpha zeta zeta gamma eta eta zeta gamma theta beta alpha beta alpha epsilon delta alpha alpha eta epsilon eta eta eta beta gamma epsilon beta epsilon beta delta gamma zeta eta beta epsilon eta delta alpha delta beta eta beta theta alpha epsilon gamma beta alpha gamma alpha gamma theta zeta epsilon gamma zeta gamma epsilon beta alpha zeta eta epsilon beta epsilon beta theta theta zeta alpha theta gamma zeta gamma epsilon beta beta delta alpha alpha alpha delta alpha theta zeta eta gamma gamma alpha eta delta zeta delta eta zeta epsilon beta zeta beta alpha gamma delta alpha eta beta theta epsilon epsilon zeta beta theta alpha zeta delta epsilon epsilon delta theta zeta theta delta delta gamma alpha eta alpha delta zeta alpha zeta alpha eta epsilon beta delta delta eta theta alpha gamma epsilon beta alpha delta eta zeta theta beta beta gamma eta beta alpha eta gamma delta epsilon epsilon zeta eta zeta zeta theta epsilon delta beta delta gamma beta gamma beta gamma theta theta zeta eta beta zeta delta theta epsilon theta epsilon beta beta gamma epsilon alpha delta zeta gamma beta delta zeta eta alpha epsilon epsilon gamma alpha eta theta delta beta theta beta gamma beta alpha alpha delta gamma delta eta zeta beta epsilon beta alpha beta delta theta gamma beta zeta beta alpha theta alpha gamma eta eta theta alpha eta eta gamma zeta delta gamma epsilon epsilon theta gamma alpha delta epsilon theta eta theta alpha beta epsilon eta gamma eta delta delta alpha eta zeta theta theta zeta zeta eta epsilon gamma alpha zeta delta alpha epsilon alpha theta zeta delta gamma beta delta delta epsilon alpha delta eta zeta gamma gamma delta zeta zeta alpha zeta gamma delta theta epsilon gamma theta alpha beta alpha delta delta alpha theta alpha zeta delta gamma zeta gamma zeta alpha alpha gamma gamma beta zeta beta zeta eta beta zeta epsilon zeta gamma gamma eta theta zeta gamma zeta delta gamma eta epsilon epsilon gamma gamma alpha eta alpha gamma zeta delta beta theta gamma zeta beta delta zeta zeta gamma beta zeta theta alpha epsilon delta delta beta zeta epsilon beta alpha alpha eta theta eta gamma eta theta eta zeta eta beta theta delta gamma theta beta alpha epsilon alpha zeta epsilon beta beta zeta gamma eta gamma beta beta zeta theta alpha eta gamma eta gamma alpha beta zeta delta delta eta epsilon epsilon epsilon delta beta alpha eta theta gamma alpha zeta alpha eta beta epsilon theta delta beta alpha delta gamma epsilon beta theta beta epsilon eta theta theta epsilon beta eta gamma zeta eta zeta gamma theta alpha epsilon theta theta epsilon delta epsilon alpha gamma beta beta zeta eta delta alpha eta eta theta delta theta epsilon beta eta alpha gamma beta theta gamma gamma delta delta gamma alpha eta beta eta delta gamma epsilon zeta beta beta eta eta zeta epsilon theta alpha eta beta eta gamma gamma beta beta beta epsilon zeta eta epsilon eta theta theta alpha gamma epsilon eta gamma eta alpha eta zeta delta alpha theta epsilon zeta alpha zeta epsilon epsilon epsilon theta beta delta gamma epsilon theta zeta epsilon eta beta delta theta delta eta theta zeta alpha gamma beta epsilon eta gamma alpha gamma delta delta alpha delta alpha theta alpha zeta delta epsilon zeta theta eta beta alpha delta zeta theta theta gamma theta zeta zeta gamma epsilon beta epsilon alpha eta alpha gamma delta delta delta delta epsilon eta alpha alpha theta gamma gamma alpha delta epsilon epsilon epsilon eta eta zeta theta epsilon delta theta epsilon eta beta alpha zeta epsilon epsilon beta theta beta zeta epsilon zeta epsilon epsilon epsilon delta gamma delta alpha eta zeta gamma alpha theta epsilon epsilon eta eta eta alpha delta zeta delta theta zeta epsilon gamma gamma epsilon beta theta gamma epsilon gamma zeta beta delta zeta delta epsilon eta zeta zeta epsilon epsilon theta beta theta alpha beta theta delta beta eta epsilon eta alpha gamma gamma delta zeta eta theta gamma zeta gamma beta theta zeta gamma zeta alpha alpha theta epsilon delta gamma gamma theta beta gamma eta eta eta eta theta eta alpha alpha delta zeta alpha zeta delta alpha alpha delta delta zeta epsilon gamma beta eta epsilon gamma beta alpha epsilon epsilon theta zeta eta gamma zeta theta zeta delta gamma eta alpha delta delta gamma delta alpha gamma beta zeta alpha eta epsilon alpha alpha beta alpha alpha beta eta theta eta beta epsilon theta gamma delta alpha epsilon eta beta epsilon gamma eta beta beta epsilon beta beta theta delta delta epsilon delta zeta eta epsilon gamma alpha theta delta theta zeta delta alpha alpha beta beta theta gamma beta beta beta epsilon delta theta epsilon epsilon theta alpha beta gamma alpha epsilon zeta zeta eta beta beta alpha alpha gamma gamma zeta zeta theta epsilon beta zeta zeta gamma beta eta eta theta epsilon eta theta eta gamma beta gamma alpha gamma beta eta theta theta gamma eta zeta alpha gamma theta theta beta eta theta alpha beta epsilon zeta alpha delta zeta beta eta delta theta zeta eta gamma alpha alpha gamma theta theta gamma beta theta zeta delta zeta epsilon alpha delta eta eta delta beta theta theta theta beta theta theta zeta beta theta alpha beta eta eta alpha alpha beta epsilon delta theta zeta zeta alpha delta theta zeta theta zeta theta beta beta delta alpha zeta zeta epsilon zeta beta alpha gamma theta gamma eta beta epsilon delta delta beta eta delta theta delta zeta beta eta alpha beta theta theta gamma theta alpha alpha eta theta gamma gamma gamma beta eta zeta eta eta delta epsilon eta zeta epsilon theta gamma gamma eta epsilon zeta delta delta delta epsilon zeta gamma gamma zeta beta zeta theta eta epsilon eta alpha beta alpha eta gamma alpha alpha delta epsilon gamma epsilon epsilon gamma alpha epsilon delta alpha zeta theta beta delta eta delta epsilon alpha eta eta eta zeta alpha alpha epsilon delta theta delta zeta delta delta epsilon theta gamma beta gamma gamma beta eta alpha eta epsilon epsilon gamma gamma epsilon alpha zeta theta gamma alpha gamma zeta alpha epsilon theta zeta beta zeta delta gamma beta gamma eta eta beta zeta delta delta zeta zeta zeta epsilon delta theta alpha beta zeta theta delta delta alpha zeta eta beta eta epsilon epsilon alpha zeta theta theta alpha epsilon delta zeta beta beta gamma alpha beta delta delta eta beta delta eta beta gamma alpha theta zeta alpha beta beta alpha gamma epsilon beta delta epsilon eta eta theta eta eta zeta alpha eta beta alpha beta alpha beta zeta beta epsilon epsilon beta epsilon theta eta eta alpha theta gamma delta gamma eta epsilon gamma epsilon zeta alpha gamma beta alpha alpha eta beta epsilon delta zeta delta eta gamma gamma gamma delta epsilon delta beta gamma alpha theta beta epsilon beta epsilon beta delta theta zeta zeta gamma delta beta epsilon beta delta zeta theta theta zeta epsilon gamma zeta zeta eta gamma alpha zeta delta delta eta epsilon zeta gamma zeta theta theta theta zeta epsilon theta beta gamma beta epsilon gamma delta epsilon beta beta alpha alpha eta delta alpha epsilon theta gamma zeta eta delta epsilon gamma theta beta eta zeta alpha delta gamma gamma theta gamma gamma gamma theta zeta alpha delta theta delta beta epsilon zeta delta alpha delta zeta eta theta theta alpha alpha zeta beta epsilon epsilon gamma eta eta zeta beta epsilon eta delta eta zeta zeta theta theta alpha gamma theta gamma delta beta epsilon delta gamma gamma gamma zeta beta delta eta zeta beta theta theta theta delta gamma eta alpha delta alpha gamma gamma gamma alpha gamma alpha gamma epsilon gamma eta alpha epsilon beta delta theta theta theta gamma delta eta gamma gamma epsilon gamma zeta theta beta delta eta eta gamma beta alpha delta eta gamma beta gamma theta alpha beta beta theta gamma alpha eta eta zeta alpha eta delta theta eta zeta beta eta gamma epsilon theta delta beta epsilon eta epsilon delta alpha alpha beta alpha eta delta zeta eta zeta beta gamma eta epsilon epsilon eta zeta epsilon eta delta eta theta gamma zeta theta epsilon epsilon zeta eta gamma beta delta theta gamma beta delta eta zeta alpha gamma eta alpha beta gamma theta eta beta beta epsilon delta zeta beta alpha eta eta epsilon alpha gamma theta theta alpha delta epsilon delta alpha delta gamma alpha zeta delta alpha gamma beta epsilon eta zeta eta eta delta alpha beta beta epsilon zeta zeta epsilon delta delta theta eta alpha epsilon gamma eta beta beta epsilon zeta beta epsilon epsilon theta eta theta delta zeta theta theta alpha theta beta epsilon beta eta delta delta zeta eta zeta alpha epsilon delta epsilon beta eta zeta beta gamma theta zeta zeta delta epsilon gamma delta theta delta delta delta epsilon delta delta delta delta zeta gamma beta beta beta theta alpha alpha theta zeta zeta theta epsilon theta gamma epsilon eta delta gamma alpha eta theta alpha zeta eta gamma theta beta theta zeta alpha alpha delta eta beta zeta zeta epsilon gamma epsilon delta epsilon theta zeta zeta gamma theta eta delta eta zeta zeta eta epsilon theta gamma gamma delta beta epsilon delta eta eta delta gamma gamma theta alpha gamma eta theta gamma zeta theta epsilon epsilon alpha eta beta gamma alpha delta epsilon theta beta theta zeta epsilon zeta zeta epsilon zeta alpha eta beta zeta delta gamma epsilon zeta zeta theta theta beta eta epsilon theta gamma gamma zeta eta eta beta beta gamma zeta delta zeta zeta epsilon epsilon eta epsilon theta zeta theta eta gamma gamma alpha gamma delta epsilon beta delta gamma eta beta beta beta theta alpha alpha eta beta beta epsilon gamma beta eta epsilon delta delta epsilon theta gamma gamma gamma alpha alpha zeta zeta theta theta epsilon theta gamma gamma theta eta epsilon zeta theta delta beta theta epsilon zeta eta epsilon gamma epsilon alpha alpha gamma zeta epsilon theta alpha theta theta epsilon alpha eta epsilon delta alpha zeta delta eta eta eta delta gamma eta eta delta epsilon beta eta delta epsilon epsilon epsilon eta delta beta gamma gamma epsilon beta theta theta epsilon theta delta zeta alpha gamma alpha alpha gamma beta epsilon epsilon gamma theta alpha delta beta gamma alpha theta delta zeta eta epsilon theta zeta theta alpha zeta beta gamma alpha epsilon eta beta theta gamma delta epsilon eta alpha beta delta eta theta delta alpha delta theta beta delta theta eta epsilon zeta gamma alpha beta epsilon theta epsilon delta zeta alpha delta beta delta eta zeta alpha gamma epsilon beta eta delta alpha alpha theta gamma zeta beta delta gamma theta beta zeta alpha alpha zeta gamma beta gamma theta delta zeta zeta theta delta alpha eta zeta gamma epsilon alpha theta theta epsilon theta delta gamma zeta zeta theta epsilon gamma beta gamma theta epsilon epsilon zeta theta alpha eta gamma alpha zeta delta zeta beta eta delta theta gamma epsilon zeta eta zeta beta beta beta alpha gamma gamma alpha delta beta theta delta beta alpha eta theta epsilon alpha zeta delta beta alpha delta theta beta theta alpha zeta eta zeta alpha epsilon alpha beta epsilon beta delta beta beta delta eta delta delta delta gamma eta beta zeta theta eta beta gamma gamma eta zeta gamma theta zeta zeta zeta zeta eta eta zeta delta delta eta epsilon zeta theta theta eta alpha alpha epsilon alpha gamma epsilon alpha epsilon delta delta zeta alpha epsilon gamma beta alpha theta zeta gamma gamma epsilon zeta delta delta epsilon epsilon beta eta delta beta zeta theta gamma theta beta epsilon alpha eta beta alpha zeta zeta eta eta beta delta alpha eta alpha epsilon alpha zeta gamma zeta delta gamma alpha eta theta beta delta eta epsilon eta gamma gamma zeta beta theta alpha eta epsilon beta eta eta gamma gamma theta delta theta eta delta delta delta beta epsilon eta zeta beta beta eta eta zeta zeta delta zeta zeta alpha epsilon gamma zeta theta gamma theta alpha delta delta zeta gamma beta beta alpha delta gamma epsilon alpha zeta beta epsilon theta theta alpha zeta delta delta delta epsilon theta gamma alpha eta alpha epsilon gamma alpha epsilon delta epsilon alpha theta alpha beta eta epsilon zeta beta alpha epsilon eta delta eta beta zeta theta beta gamma delta theta alpha alpha delta epsilon zeta theta eta gamma alpha zeta zeta zeta epsilon delta epsilon alpha epsilon eta alpha epsilon delta gamma zeta eta gamma beta eta delta eta delta gamma eta epsilon zeta theta gamma delta gamma delta gamma epsilon gamma eta gamma gamma zeta beta beta gamma zeta gamma alpha zeta eta eta epsilon theta epsilon gamma delta beta gamma gamma alpha epsilon delta delta delta alpha zeta beta theta gamma theta zeta gamma eta theta theta gamma gamma delta delta alpha eta alpha epsilon eta delta delta delta beta gamma theta theta zeta theta delta eta eta alpha eta zeta eta alpha beta theta theta alpha delta eta alpha eta alpha delta epsilon theta eta epsilon epsilon theta gamma eta epsilon alpha gamma theta theta beta beta gamma gamma eta eta zeta theta alpha gamma theta delta theta beta epsilon zeta gamma beta beta beta theta zeta beta eta gamma eta eta alpha gamma zeta gamma gamma gamma gamma alpha eta epsilon epsilon theta epsilon gamma beta gamma delta alpha epsilon alpha theta theta theta alpha theta delta delta theta theta eta theta beta alpha delta epsilon delta alpha gamma gamma alpha zeta alpha gamma alpha epsilon zeta beta alpha beta eta gamma gamma epsilon gamma theta zeta alpha beta epsilon zeta zeta eta beta eta beta epsilon theta gamma beta epsilon zeta eta delta delta alpha beta beta theta epsilon eta zeta epsilon zeta gamma theta theta theta gamma delta epsilon delta alpha beta alpha theta beta delta eta eta alpha zeta zeta delta alpha delta epsilon epsilon beta epsilon epsilon zeta alpha beta zeta zeta eta alpha gamma delta zeta zeta theta theta alpha gamma theta gamma theta zeta theta eta alpha epsilon zeta alpha delta zeta zeta beta gamma epsilon alpha zeta eta gamma eta gamma beta epsilon zeta alpha epsilon eta beta theta delta zeta theta epsilon delta epsilon gamma beta zeta alpha delta beta theta alpha theta gamma delta zeta theta epsilon zeta gamma beta gamma delta alpha theta eta beta alpha epsilon alpha beta zeta theta alpha beta epsilon alpha epsilon delta zeta delta delta epsilon epsilon zeta theta beta epsilon gamma gamma epsilon beta alpha beta delta zeta zeta epsilon gamma epsilon delta gamma beta epsilon zeta theta alpha gamma alpha theta alpha beta eta alpha beta eta eta gamma theta zeta epsilon zeta delta beta theta gamma beta theta beta theta epsilon theta theta delta delta gamma epsilon zeta eta gamma zeta zeta theta theta alpha beta theta beta zeta theta theta gamma epsilon zeta gamma beta eta beta zeta beta epsilon theta eta eta delta delta gamma theta eta beta zeta delta eta eta zeta delta gamma beta theta beta theta zeta beta theta gamma epsilon eta beta beta eta zeta theta alpha beta delta gamma alpha gamma gamma theta epsilon eta theta delta delta zeta epsilon epsilon eta zeta beta alpha eta gamma epsilon alpha zeta alpha alpha zeta beta epsilon alpha delta eta eta zeta eta eta eta eta delta gamma theta gamma delta eta zeta eta gamma epsilon gamma gamma alpha beta alpha gamma theta eta gamma beta gamma eta theta eta gamma theta eta theta eta zeta delta gamma eta alpha theta delta epsilon eta zeta alpha alpha theta zeta beta theta beta gamma epsilon theta beta beta delta theta beta epsilon alpha zeta alpha theta theta theta zeta eta zeta epsilon epsilon gamma alpha epsilon epsilon alpha theta gamma alpha epsilon alpha epsilon theta beta gamma beta epsilon epsilon gamma epsilon epsilon epsilon alpha beta gamma gamma theta zeta gamma epsilon delta theta gamma alpha zeta beta gamma gamma theta eta eta delta theta gamma delta zeta alpha epsilon beta alpha gamma theta eta theta epsilon theta delta eta eta theta epsilon gamma eta alpha epsilon theta eta zeta alpha beta zeta eta epsilon zeta epsilon epsilon beta epsilon eta theta delta delta zeta gamma zeta alpha alpha beta alpha eta beta beta theta zeta delta epsilon epsilon delta alpha alpha beta alpha delta beta beta beta gamma eta gamma beta gamma epsilon epsilon theta beta theta epsilon eta zeta epsilon gamma beta epsilon alpha eta eta zeta alpha epsilon epsilon gamma theta beta theta delta eta gamma theta beta gamma eta delta beta theta epsilon beta alpha beta beta epsilon theta alpha theta gamma alpha alpha eta epsilon gamma zeta eta alpha theta theta eta eta beta delta alpha eta theta beta zeta zeta epsilon eta alpha beta zeta zeta beta theta epsilon delta gamma alpha eta delta epsilon eta beta zeta theta beta theta delta gamma alpha delta alpha delta theta eta beta epsilon theta theta alpha alpha zeta alpha zeta alpha eta beta delta theta gamma delta beta alpha gamma theta alpha theta alpha delta gamma alpha gamma alpha epsilon beta theta gamma beta beta beta theta delta beta delta eta zeta eta zeta beta gamma zeta delta alpha gamma zeta beta beta epsilon gamma eta gamma zeta gamma theta delta alpha gamma beta eta gamma alpha theta beta alpha alpha gamma beta epsilon eta theta gamma delta gamma epsilon theta eta delta gamma alpha gamma theta beta epsilon zeta eta alpha delta delta beta beta zeta delta theta delta epsilon alpha delta beta gamma theta theta gamma gamma alpha alpha zeta epsilon delta eta epsilon zeta theta alpha beta alpha gamma gamma alpha theta beta beta delta theta zeta eta delta zeta delta beta zeta delta beta delta eta delta eta delta beta theta theta alpha eta gamma delta theta gamma eta zeta gamma theta delta zeta epsilon delta zeta gamma alpha alpha epsilon gamma beta theta theta eta gamma zeta delta epsilon gamma gamma delta beta eta theta delta theta theta epsilon alpha eta delta delta zeta eta delta gamma alpha eta alpha alpha theta alpha eta eta theta beta theta theta delta beta alpha eta delta gamma zeta epsilon delta beta alpha theta epsilon gamma theta alpha theta theta gamma eta gamma zeta theta alpha beta eta zeta alpha epsilon eta epsilon delta beta zeta alpha zeta alpha beta eta zeta theta alpha zeta zeta gamma beta delta gamma zeta delta beta theta delta delta zeta zeta epsilon eta epsilon alpha beta zeta zeta theta theta beta theta epsilon theta zeta beta delta theta beta beta eta gamma epsilon beta beta gamma epsilon beta gamma theta gamma delta alpha epsilon eta gamma theta zeta eta theta alpha epsilon delta beta alpha delta zeta eta zeta zeta theta theta theta epsilon gamma epsilon gamma epsilon delta theta zeta epsilon alpha gamma zeta epsilon theta gamma gamma epsilon gamma gamma gamma gamma delta beta alpha zeta gamma theta epsilon zeta gamma delta alpha zeta zeta gamma zeta theta gamma epsilon beta alpha eta theta gamma theta zeta gamma alpha zeta epsilon theta epsilon gamma theta zeta beta theta alpha gamma beta epsilon alpha eta delta theta zeta beta zeta alpha delta delta epsilon delta alpha beta theta beta eta delta theta zeta alpha zeta eta gamma eta zeta eta zeta alpha epsilon zeta zeta epsilon theta beta delta theta eta theta epsilon gamma eta eta gamma beta gamma delta epsilon theta zeta beta eta beta alpha eta epsilon eta gamma theta gamma delta gamma eta epsilon theta alpha epsilon epsilon theta theta delta alpha alpha gamma epsilon epsilon theta beta epsilon eta gamma theta delta alpha beta epsilon theta beta zeta theta eta epsilon theta beta zeta alpha epsilon gamma alpha gamma zeta theta gamma zeta gamma alpha epsilon zeta eta zeta alpha epsilon epsilon beta delta gamma beta gamma zeta zeta epsilon epsilon zeta beta zeta gamma alpha gamma epsilon zeta theta eta zeta gamma beta delta gamma delta delta beta eta theta beta beta gamma alpha theta zeta epsilon zeta gamma theta zeta gamma epsilon alpha beta delta gamma gamma delta epsilon gamma delta gamma beta eta beta beta alpha theta eta alpha delta beta alpha alpha theta beta epsilon alpha beta epsilon alpha beta beta delta delta gamma zeta alpha beta gamma epsilon zeta zeta gamma delta beta beta alpha delta delta epsilon gamma alpha beta delta theta zeta gamma theta beta gamma zeta eta eta delta delta eta beta zeta beta theta beta gamma theta zeta epsilon alpha beta zeta theta beta eta epsilon theta epsilon theta delta eta epsilon eta delta beta theta zeta gamma alpha theta beta eta theta beta beta gamma eta eta beta gamma beta zeta delta zeta delta eta beta delta eta theta epsilon epsilon eta zeta theta eta theta epsilon gamma theta beta beta epsilon theta theta delta gamma alpha beta alpha beta theta alpha gamma eta zeta gamma epsilon beta beta theta theta delta epsilon theta theta epsilon zeta alpha alpha eta beta eta theta zeta eta theta theta alpha alpha gamma epsilon gamma theta eta gamma epsilon eta theta beta theta theta theta gamma alpha eta epsilon theta eta zeta epsilon zeta alpha alpha alpha gamma zeta zeta alpha zeta eta beta eta zeta zeta eta zeta alpha delta delta delta epsilon beta beta epsilon beta zeta gamma alpha zeta alpha theta eta eta epsilon alpha theta alpha zeta eta eta zeta beta theta beta delta zeta theta zeta gamma epsilon zeta eta delta theta beta delta epsilon epsilon alpha epsilon eta delta beta beta zeta epsilon beta theta eta delta zeta beta epsilon epsilon eta zeta delta eta eta epsilon theta gamma alpha gamma theta delta theta epsilon eta alpha delta zeta theta gamma delta gamma epsilon eta eta zeta beta delta eta epsilon epsilon gamma epsilon delta eta delta beta gamma delta alpha delta epsilon theta theta eta eta alpha delta gamma epsilon gamma alpha epsilon epsilon gamma alpha theta alpha alpha beta zeta epsilon beta eta zeta beta eta gamma delta beta beta epsilon theta beta delta beta zeta zeta eta delta eta delta gamma beta gamma delta epsilon theta eta epsilon alpha theta zeta beta zeta eta theta delta epsilon gamma zeta theta zeta epsilon alpha alpha alpha delta epsilon eta epsilon zeta beta delta theta eta alpha eta epsilon epsilon beta delta epsilon delta gamma theta zeta eta alpha epsilon beta theta theta eta zeta eta beta gamma epsilon beta zeta eta zeta theta alpha alpha gamma theta gamma alpha eta delta epsilon zeta beta beta gamma delta gamma eta zeta theta theta gamma epsilon gamma zeta epsilon delta beta eta zeta gamma eta eta delta alpha alpha delta beta theta delta epsilon gamma alpha eta eta epsilon gamma beta zeta gamma beta alpha eta theta theta beta delta delta delta epsilon epsilon theta eta gamma eta delta delta epsilon delta eta gamma eta gamma alpha epsilon epsilon delta theta delta zeta theta beta alpha theta gamma zeta eta delta eta eta epsilon beta beta alpha alpha zeta delta theta zeta alpha alpha epsilon eta epsilon eta zeta gamma alpha gamma epsilon gamma theta zeta theta gamma beta zeta eta gamma epsilon beta zeta alpha gamma theta epsilon zeta eta epsilon epsilon epsilon zeta eta gamma zeta theta gamma theta alpha delta gamma alpha delta gamma alpha theta epsilon gamma theta eta beta theta alpha theta zeta theta theta theta beta gamma beta zeta beta delta epsilon gamma gamma epsilon delta alpha gamma eta beta delta theta beta zeta alpha epsilon beta eta zeta beta zeta gamma alpha beta theta theta theta epsilon alpha zeta alpha theta zeta gamma delta alpha epsilon zeta alpha beta eta beta eta epsilon gamma delta eta beta alpha delta gamma eta eta epsilon gamma beta epsilon theta eta epsilon gamma zeta beta zeta delta theta theta eta theta epsilon eta beta eta zeta gamma eta eta beta alpha zeta theta delta gamma theta beta epsilon theta eta beta theta epsilon gamma gamma epsilon delta eta epsilon alpha beta alpha beta eta eta beta theta epsilon zeta epsilon beta alpha beta theta gamma gamma eta alpha beta eta epsilon zeta theta theta epsilon zeta gamma delta gamma beta epsilon beta eta alpha alpha eta beta alpha gamma theta theta eta theta epsilon zeta eta epsilon beta gamma gamma alpha delta beta alpha delta alpha delta theta gamma delta eta delta beta epsilon theta gamma theta eta alpha beta gamma gamma theta theta theta alpha theta delta delta epsilon eta alpha zeta alpha beta eta eta theta epsilon alpha theta eta theta theta delta alpha alpha eta zeta beta epsilon eta theta beta theta zeta theta eta alpha gamma epsilon delta theta zeta theta delta delta zeta theta gamma epsilon eta eta zeta theta gamma delta epsilon beta theta gamma gamma delta gamma theta beta eta theta epsilon delta delta alpha alpha zeta epsilon zeta gamma gamma alpha beta zeta eta beta delta beta delta theta epsilon zeta zeta alpha epsilon gamma beta alpha theta theta beta beta beta alpha beta gamma beta delta zeta gamma epsilon theta theta beta gamma zeta zeta gamma beta delta theta eta alpha delta theta zeta epsilon gamma beta epsilon zeta eta zeta delta gamma eta delta gamma eta eta eta theta zeta alpha theta gamma alpha beta beta zeta zeta eta gamma eta alpha eta alpha gamma alpha epsilon eta eta epsilon zeta zeta alpha epsilon gamma eta theta delta theta zeta eta delta epsilon beta alpha zeta eta gamma epsilon delta beta epsilon zeta gamma alpha alpha epsilon delta epsilon eta beta alpha zeta epsilon zeta beta alpha beta theta epsilon alpha theta zeta delta delta zeta epsilon epsilon gamma eta delta beta zeta delta zeta theta beta gamma eta eta beta beta alpha eta gamma theta delta theta beta zeta eta eta epsilon theta eta theta theta eta gamma gamma gamma theta theta gamma beta delta theta theta alpha gamma zeta beta eta beta beta zeta epsilon alpha gamma beta zeta epsilon zeta epsilon epsilon theta alpha gamma gamma eta alpha epsilon epsilon delta gamma theta alpha beta zeta beta alpha eta theta epsilon theta zeta eta theta gamma gamma theta theta theta gamma zeta beta eta eta delta eta zeta alpha theta epsilon beta gamma alpha epsilon beta beta delta alpha delta theta alpha beta gamma alpha epsilon delta delta theta beta zeta eta alpha theta alpha zeta eta gamma eta alpha zeta eta epsilon beta beta beta gamma epsilon gamma gamma epsilon beta gamma epsilon eta gamma alpha zeta gamma alpha alpha epsilon eta beta zeta gamma eta zeta epsilon eta beta alpha delta delta delta theta eta beta epsilon alpha zeta theta eta beta theta alpha eta delta alpha epsilon beta delta gamma epsilon eta theta beta delta gamma zeta gamma delta eta theta alpha alpha theta epsilon gamma alpha alpha theta epsilon epsilon eta zeta delta epsilon alpha gamma beta epsilon beta eta beta gamma epsilon eta delta zeta zeta epsilon theta epsilon delta zeta zeta delta theta zeta alpha gamma alpha theta alpha alpha gamma eta theta eta beta beta alpha theta beta zeta eta zeta eta gamma alpha beta theta zeta delta theta alpha delta alpha epsilon zeta delta beta epsilon delta delta theta zeta alpha epsilon theta delta alpha delta eta beta epsilon theta gamma delta gamma beta gamma beta gamma epsilon delta epsilon delta alpha epsilon theta alpha eta alpha zeta eta eta epsilon beta theta theta zeta alpha delta alpha epsilon eta theta zeta alpha eta delta zeta beta alpha delta eta beta delta delta gamma zeta epsilon beta beta alpha theta eta alpha zeta epsilon delta beta theta delta zeta alpha zeta delta beta theta beta theta beta beta alpha alpha eta zeta beta gamma alpha theta alpha epsilon eta beta beta alpha alpha epsilon eta eta beta eta delta alpha theta eta theta eta beta zeta alpha theta zeta delta beta beta alpha beta delta gamma epsilon zeta eta theta theta gamma delta gamma alpha alpha gamma epsilon eta zeta epsilon gamma gamma theta epsilon zeta epsilon zeta alpha gamma alpha delta alpha eta zeta delta delta delta beta zeta delta theta theta zeta zeta epsilon eta zeta gamma beta theta alpha theta delta alpha delta delta theta delta delta theta zeta eta delta eta gamma alpha zeta alpha beta zeta alpha beta gamma epsilon theta delta zeta alpha alpha delta delta epsilon gamma delta epsilon gamma zeta delta gamma delta alpha gamma epsilon beta alpha gamma eta beta zeta delta epsilon theta delta zeta zeta gamma eta beta beta gamma gamma delta delta eta zeta zeta eta delta alpha delta epsilon gamma zeta delta alpha epsilon gamma theta delta beta theta beta gamma eta zeta theta theta gamma zeta zeta theta theta theta epsilon eta alpha delta delta delta theta delta eta delta epsilon eta eta gamma zeta epsilon beta gamma beta epsilon theta zeta delta alpha theta zeta eta theta theta eta eta zeta theta zeta epsilon eta alpha gamma theta delta epsilon beta zeta beta epsilon beta epsilon delta beta beta beta zeta alpha delta zeta delta zeta epsilon epsilon epsilon delta gamma epsilon alpha epsilon epsilon zeta eta eta alpha theta gamma eta delta delta delta theta delta theta theta zeta beta zeta delta eta eta delta theta theta alpha zeta gamma gamma epsilon alpha theta zeta zeta delta eta eta eta epsilon zeta alpha alpha gamma gamma zeta eta eta delta eta beta eta delta theta beta zeta theta epsilon beta gamma beta gamma alpha theta alpha gamma delta beta eta epsilon delta delta epsilon beta theta alpha zeta alpha gamma zeta eta eta gamma beta gamma eta theta beta eta delta gamma delta epsilon eta theta gamma epsilon delta theta epsilon delta alpha gamma theta beta eta delta zeta theta theta theta alpha beta epsilon theta zeta epsilon eta gamma eta delta gamma eta theta eta gamma beta eta alpha delta gamma epsilon epsilon beta delta theta eta beta zeta beta beta zeta eta theta gamma epsilon delta alpha theta alpha beta theta eta zeta eta delta eta alpha alpha epsilon beta gamma zeta theta theta theta beta beta delta theta gamma theta epsilon gamma gamma alpha beta gamma epsilon beta eta gamma beta alpha epsilon gamma theta zeta gamma delta alpha delta theta alpha beta eta epsilon theta alpha alpha alpha eta delta delta zeta delta delta beta alpha eta alpha alpha zeta delta beta delta beta gamma gamma alpha eta beta zeta zeta delta delta eta eta beta delta delta beta zeta zeta gamma alpha eta zeta alpha eta delta theta delta beta epsilon zeta alpha beta delta eta epsilon theta beta gamma epsilon beta zeta eta gamma zeta gamma zeta theta beta zeta alpha theta eta delta alpha delta alpha beta epsilon delta eta delta beta alpha gamma zeta theta epsilon delta gamma eta delta alpha delta theta zeta gamma epsilon zeta alpha eta epsilon alpha delta zeta zeta delta delta alpha delta delta eta zeta gamma theta alpha beta theta beta delta alpha gamma delta gamma beta gamma delta delta beta eta theta epsilon gamma delta delta eta gamma delta epsilon epsilon theta delta epsilon theta beta zeta eta eta gamma theta theta eta eta gamma theta alpha eta eta beta epsilon gamma gamma epsilon eta theta epsilon theta epsilon eta theta beta zeta gamma theta delta gamma delta alpha alpha zeta theta eta eta eta zeta eta theta epsilon beta eta alpha beta eta gamma delta eta epsilon zeta theta zeta alpha eta eta delta delta theta alpha zeta alpha gamma epsilon theta gamma epsilon delta gamma alpha beta beta eta eta beta delta eta gamma eta epsilon epsilon eta epsilon eta beta beta zeta gamma epsilon delta eta epsilon alpha gamma theta alpha beta beta eta eta epsilon eta gamma gamma zeta gamma gamma delta theta gamma beta theta delta gamma delta alpha gamma delta epsilon eta theta beta gamma beta epsilon beta eta beta epsilon beta zeta alpha delta theta theta beta alpha alpha epsilon alpha alpha beta delta alpha eta epsilon theta theta zeta eta zeta zeta zeta zeta delta zeta eta eta alpha zeta zeta beta alpha theta eta gamma alpha beta epsilon delta zeta theta theta eta delta alpha zeta alpha zeta alpha zeta theta delta theta eta beta zeta eta theta beta eta epsilon delta gamma alpha gamma theta beta epsilon theta gamma theta beta theta delta delta delta gamma beta beta eta alpha gamma zeta eta epsilon delta epsilon theta beta gamma theta beta zeta delta beta zeta eta eta eta delta delta epsilon beta epsilon beta beta beta zeta delta beta zeta gamma zeta eta alpha epsilon alpha delta eta eta zeta zeta gamma delta beta eta alpha theta zeta gamma epsilon delta theta theta gamma epsilon gamma alpha zeta epsilon beta epsilon theta delta theta epsilon eta alpha zeta gamma delta delta alpha gamma theta eta epsilon gamma alpha eta zeta eta theta zeta eta eta beta epsilon epsilon delta eta beta gamma zeta alpha epsilon theta gamma gamma theta theta epsilon gamma delta delta delta alpha zeta beta gamma eta eta gamma alpha beta zeta theta gamma alpha beta zeta eta gamma eta delta alpha beta delta eta theta epsilon gamma eta beta theta eta epsilon theta zeta alpha beta beta beta beta theta delta alpha theta epsilon beta gamma eta theta eta epsilon gamma beta zeta beta gamma eta alpha eta theta beta beta delta beta gamma gamma alpha theta zeta beta alpha gamma delta gamma beta epsilon alpha beta beta epsilon zeta gamma eta alpha delta epsilon alpha epsilon theta gamma theta zeta zeta delta beta alpha delta alpha delta gamma epsilon delta gamma delta zeta epsilon gamma eta beta epsilon epsilon eta gamma eta theta beta epsilon eta gamma delta beta epsilon delta zeta gamma eta gamma zeta eta eta alpha delta beta epsilon beta beta eta beta zeta beta zeta zeta epsilon gamma delta epsilon alpha beta eta delta eta beta beta gamma epsilon theta beta zeta epsilon epsilon beta alpha gamma zeta eta gamma zeta eta theta gamma epsilon alpha gamma gamma gamma eta epsilon alpha zeta delta eta theta gamma theta zeta theta epsilon epsilon eta delta epsilon beta epsilon eta zeta beta beta beta delta beta gamma alpha beta epsilon theta alpha eta gamma theta theta beta eta alpha delta alpha eta zeta delta alpha theta gamma delta gamma beta zeta gamma zeta alpha eta theta eta gamma zeta beta zeta delta beta epsilon epsilon theta epsilon epsilon delta alpha eta delta epsilon alpha theta theta alpha zeta alpha theta zeta epsilon zeta beta alpha zeta gamma eta zeta alpha beta eta theta delta beta gamma eta beta theta alpha gamma epsilon delta theta eta zeta epsilon zeta eta gamma epsilon beta theta theta gamma gamma zeta eta alpha zeta delta beta delta epsilon alpha gamma eta eta zeta alpha gamma delta theta epsilon beta eta theta theta gamma zeta eta eta gamma beta beta alpha gamma gamma zeta alpha delta alpha zeta eta alpha eta theta epsilon epsilon theta beta eta delta gamma gamma delta epsilon beta epsilon delta eta delta theta beta gamma delta gamma delta beta gamma eta alpha epsilon delta gamma eta alpha delta beta alpha eta eta epsilon delta delta zeta eta theta alpha eta alpha epsilon beta epsilon zeta epsilon zeta eta delta zeta alpha gamma alpha alpha alpha zeta delta epsilon theta epsilon beta eta gamma epsilon theta zeta zeta epsilon beta beta alpha gamma zeta zeta zeta zeta delta epsilon delta eta beta beta gamma beta alpha delta delta theta beta beta zeta epsilon theta delta gamma alpha gamma delta eta epsilon zeta zeta theta beta eta beta theta eta alpha gamma epsilon gamma epsilon alpha alpha delta eta theta beta gamma theta eta eta eta epsilon delta gamma alpha gamma delta alpha theta epsilon epsilon delta epsilon epsilon beta gamma delta alpha delta epsilon alpha gamma epsilon gamma epsilon alpha delta delta beta alpha zeta zeta zeta beta epsilon gamma eta epsilon zeta alpha alpha zeta delta epsilon gamma epsilon alpha beta eta epsilon delta theta delta zeta epsilon delta eta gamma theta delta eta zeta gamma theta beta theta gamma theta gamma gamma delta theta gamma epsilon alpha epsilon gamma gamma eta delta beta gamma beta eta delta theta eta beta gamma alpha beta eta eta alpha beta alpha alpha gamma delta beta epsilon gamma alpha alpha beta zeta beta delta gamma theta theta gamma epsilon zeta theta gamma delta alpha epsilon beta alpha alpha beta epsilon beta eta gamma alpha delta alpha delta eta eta gamma zeta epsilon delta eta beta theta gamma beta delta beta epsilon zeta beta zeta beta zeta beta alpha gamma gamma eta zeta gamma eta gamma epsilon epsilon zeta beta epsilon theta zeta epsilon beta alpha gamma gamma zeta gamma eta theta theta beta beta eta epsilon gamma beta zeta gamma gamma eta zeta zeta theta eta theta eta zeta gamma delta eta eta theta theta delta eta beta eta delta beta delta zeta alpha delta alpha zeta theta gamma beta alpha zeta zeta gamma epsilon theta gamma eta delta delta delta alpha gamma beta gamma eta alpha eta zeta alpha beta beta theta gamma alpha gamma zeta gamma beta delta beta alpha eta gamma gamma theta gamma epsilon theta eta zeta theta alpha epsilon beta gamma alpha alpha gamma gamma alpha eta eta theta epsilon epsilon delta eta gamma epsilon beta eta gamma delta eta alpha delta eta gamma beta theta theta zeta eta eta gamma alpha zeta eta delta epsilon beta beta beta zeta zeta alpha delta epsilon gamma beta beta delta alpha epsilon delta delta alpha delta epsilon beta alpha theta alpha alpha zeta beta epsilon alpha zeta epsilon eta delta zeta eta zeta gamma beta zeta eta beta gamma alpha eta theta alpha epsilon gamma gamma theta beta beta delta delta gamma gamma zeta theta gamma zeta epsilon zeta eta alpha eta delta delta beta gamma theta beta alpha gamma epsilon beta theta beta theta zeta gamma epsilon gamma beta epsilon zeta zeta beta beta gamma theta epsilon theta epsilon theta gamma beta theta alpha zeta epsilon eta eta gamma theta zeta zeta zeta beta eta theta epsilon alpha alpha delta alpha gamma beta epsilon eta gamma eta beta delta eta theta beta zeta theta theta theta beta beta gamma beta eta eta delta eta zeta beta zeta zeta theta delta theta beta epsilon theta alpha gamma theta gamma delta zeta alpha eta alpha zeta eta theta delta alpha delta alpha eta epsilon epsilon gamma beta gamma beta epsilon eta beta epsilon theta eta delta epsilon gamma alpha beta eta zeta epsilon gamma delta zeta eta theta theta gamma eta zeta eta alpha delta beta gamma zeta zeta alpha gamma theta gamma theta zeta beta eta epsilon alpha theta gamma theta beta alpha delta delta eta gamma eta delta zeta delta delta alpha zeta zeta alpha epsilon epsilon beta theta theta eta eta zeta zeta eta zeta beta gamma theta eta zeta gamma gamma eta eta epsilon eta epsilon beta beta beta beta alpha eta beta theta beta theta alpha epsilon eta delta delta theta eta beta delta theta beta zeta theta eta eta alpha zeta delta epsilon epsilon epsilon epsilon eta delta alpha eta alpha alpha alpha gamma zeta zeta zeta gamma eta delta eta delta eta alpha eta gamma eta gamma eta zeta gamma eta delta beta gamma beta eta epsilon gamma zeta delta beta beta alpha eta alpha gamma alpha beta eta alpha gamma theta zeta gamma delta delta epsilon delta delta eta epsilon alpha epsilon beta gamma alpha beta beta theta beta beta gamma epsilon epsilon gamma beta beta theta zeta delta eta beta theta alpha eta beta theta beta delta eta eta zeta theta gamma gamma epsilon alpha gamma alpha beta zeta zeta theta alpha gamma epsilon beta delta alpha alpha gamma delta zeta beta delta alpha zeta gamma delta beta gamma epsilon epsilon theta delta beta zeta epsilon theta theta eta eta delta epsilon gamma delta epsilon eta beta epsilon gamma delta theta eta theta eta epsilon zeta beta eta eta delta zeta zeta beta theta gamma alpha epsilon eta epsilon zeta eta alpha beta gamma zeta eta gamma zeta delta beta delta zeta zeta gamma zeta beta alpha zeta alpha zeta delta theta zeta delta beta eta epsilon delta delta beta epsilon beta eta eta zeta zeta delta delta alpha theta theta alpha epsilon eta beta beta theta gamma zeta eta delta alpha epsilon alpha alpha theta epsilon gamma eta alpha alpha zeta alpha beta theta beta eta epsilon epsilon zeta zeta eta beta zeta theta alpha eta epsilon theta theta beta zeta beta alpha delta alpha alpha gamma beta zeta eta delta gamma beta delta zeta alpha alpha epsilon alpha beta alpha epsilon beta epsilon zeta eta epsilon gamma gamma theta beta gamma zeta gamma eta theta beta delta theta theta epsilon zeta theta delta beta epsilon epsilon beta zeta delta delta beta beta theta alpha delta alpha delta theta eta delta theta beta zeta delta zeta epsilon theta gamma beta gamma delta zeta eta theta delta eta delta beta epsilon eta gamma epsilon beta gamma theta beta beta delta alpha theta gamma epsilon alpha theta theta zeta epsilon beta beta zeta eta delta gamma eta eta zeta alpha theta epsilon zeta beta alpha eta eta delta epsilon delta eta gamma epsilon theta eta beta alpha delta beta delta beta alpha alpha epsilon delta beta beta epsilon delta eta eta eta beta delta epsilon theta delta gamma delta zeta epsilon epsilon alpha beta gamma theta epsilon delta gamma zeta beta gamma gamma zeta theta gamma alpha theta eta zeta alpha gamma epsilon beta gamma alpha zeta theta alpha theta delta theta beta alpha alpha epsilon delta gamma gamma beta beta epsilon zeta theta theta zeta epsilon theta epsilon zeta theta alpha zeta epsilon zeta theta beta epsilon alpha gamma eta delta gamma zeta theta alpha delta eta zeta delta beta delta eta beta epsilon theta gamma epsilon zeta zeta eta eta epsilon epsilon delta alpha zeta beta eta theta zeta epsilon beta alpha theta gamma theta gamma epsilon gamma epsilon theta epsilon beta epsilon alpha theta alpha delta epsilon beta alpha theta epsilon alpha eta zeta alpha alpha epsilon gamma zeta epsilon gamma beta epsilon gamma delta eta beta epsilon delta alpha theta zeta delta eta eta eta beta epsilon epsilon epsilon gamma eta zeta gamma delta epsilon gamma eta delta epsilon epsilon beta epsilon theta gamma eta eta beta gamma theta zeta zeta theta zeta eta theta eta eta theta alpha alpha eta beta gamma epsilon delta zeta gamma epsilon alpha eta gamma eta alpha theta theta alpha beta alpha epsilon beta zeta beta epsilon theta zeta alpha beta alpha delta gamma theta gamma delta gamma epsilon epsilon alpha theta delta beta beta alpha gamma beta beta delta delta beta theta alpha theta eta alpha beta epsilon eta theta gamma delta epsilon theta beta eta gamma alpha gamma gamma delta gamma delta theta theta epsilon epsilon beta theta zeta epsilon eta eta beta theta eta eta gamma alpha epsilon theta beta eta alpha epsilon alpha theta alpha zeta gamma theta theta epsilon alpha gamma zeta delta zeta eta beta gamma gamma zeta zeta theta gamma epsilon epsilon eta beta theta eta theta gamma epsilon gamma zeta zeta eta beta alpha alpha gamma delta beta beta theta theta zeta delta delta beta gamma eta eta beta alpha epsilon beta alpha beta theta zeta alpha gamma zeta beta theta eta gamma delta delta epsilon theta theta epsilon delta delta gamma eta delta beta alpha beta delta alpha gamma alpha zeta eta beta zeta gamma theta delta delta theta zeta zeta gamma beta theta eta eta beta beta gamma epsilon beta beta theta delta alpha zeta alpha delta epsilon epsilon eta zeta beta gamma epsilon beta delta epsilon eta alpha beta theta epsilon alpha alpha zeta eta epsilon delta zeta alpha gamma delta zeta alpha delta theta beta alpha zeta beta eta beta theta gamma epsilon alpha beta alpha theta delta zeta eta theta epsilon delta alpha alpha alpha delta eta alpha theta alpha eta eta gamma eta eta gamma beta theta beta zeta zeta delta zeta eta alpha delta delta beta alpha delta eta alpha alpha delta alpha beta alpha gamma theta gamma epsilon alpha zeta zeta delta epsilon gamma eta theta beta beta beta epsilon delta theta zeta delta theta epsilon zeta epsilon delta epsilon zeta beta alpha zeta eta theta epsilon beta delta gamma delta epsilon delta beta zeta gamma alpha delta theta theta beta alpha theta alpha epsilon epsilon eta epsilon beta alpha epsilon eta theta alpha eta eta theta alpha delta delta eta gamma eta alpha epsilon alpha zeta alpha zeta epsilon delta alpha zeta eta eta alpha beta eta epsilon theta eta epsilon gamma eta theta delta gamma alpha epsilon epsilon beta delta epsilon zeta eta alpha epsilon zeta epsilon epsilon alpha beta epsilon zeta epsilon gamma theta theta alpha theta alpha theta alpha zeta theta delta eta alpha beta epsilon zeta beta theta gamma epsilon epsilon epsilon theta gamma delta gamma gamma alpha delta beta alpha gamma theta zeta alpha delta alpha epsilon epsilon alpha zeta alpha delta zeta beta beta theta beta beta delta delta zeta zeta eta beta eta epsilon gamma alpha zeta zeta zeta eta zeta theta epsilon alpha gamma zeta alpha epsilon gamma alpha beta delta zeta zeta delta eta alpha gamma eta beta beta delta eta beta alpha theta theta zeta epsilon delta epsilon delta zeta delta zeta delta theta zeta beta delta theta eta zeta beta zeta beta zeta theta delta theta eta delta gamma delta eta gamma epsilon epsilon zeta zeta beta beta eta eta gamma theta delta gamma theta alpha gamma beta eta epsilon theta beta gamma alpha eta alpha gamma beta gamma eta eta zeta gamma eta delta epsilon beta zeta zeta beta alpha beta zeta theta eta theta eta epsilon theta gamma gamma beta gamma alpha zeta theta eta zeta theta gamma epsilon beta delta epsilon theta zeta alpha eta epsilon zeta eta zeta gamma gamma alpha delta eta epsilon zeta beta zeta epsilon gamma beta epsilon epsilon theta theta delta gamma epsilon epsilon beta delta epsilon gamma beta delta gamma alpha alpha delta alpha theta alpha zeta delta epsilon eta theta gamma theta epsilon beta beta theta eta theta epsilon beta gamma epsilon beta zeta beta delta beta zeta gamma zeta epsilon theta beta zeta alpha eta beta zeta delta alpha alpha delta gamma eta delta delta epsilon theta gamma theta gamma theta epsilon gamma delta beta delta alpha eta beta theta delta gamma alpha eta epsilon eta delta zeta theta gamma epsilon delta epsilon theta theta eta zeta theta epsilon beta delta beta epsilon eta theta zeta delta eta beta gamma theta zeta beta zeta eta gamma theta theta eta delta alpha epsilon epsilon delta eta alpha zeta epsilon zeta eta eta theta eta eta epsilon alpha epsilon theta delta zeta alpha epsilon theta eta delta gamma delta beta zeta gamma alpha alpha eta beta zeta zeta beta alpha delta zeta delta gamma eta delta epsilon gamma zeta zeta eta delta delta alpha alpha alpha beta eta theta alpha gamma epsilon epsilon alpha delta delta alpha epsilon beta eta eta gamma epsilon beta beta beta eta alpha alpha gamma beta eta delta epsilon zeta eta alpha epsilon epsilon theta delta beta epsilon theta eta epsilon eta beta delta eta epsilon theta epsilon delta eta alpha theta eta zeta beta theta delta alpha eta delta theta zeta theta beta epsilon zeta gamma beta beta gamma delta theta delta beta gamma eta theta delta alpha theta epsilon eta epsilon zeta gamma alpha zeta eta alpha epsilon delta theta theta zeta delta zeta theta theta beta beta eta beta eta gamma alpha gamma beta gamma delta theta beta alpha eta theta gamma theta gamma eta eta alpha epsilon beta eta epsilon epsilon zeta zeta gamma gamma epsilon delta zeta epsilon theta theta delta zeta gamma gamma theta alpha zeta eta gamma gamma theta beta alpha epsilon delta delta beta theta zeta beta alpha theta gamma beta zeta delta zeta alpha delta epsilon epsilon eta epsilon zeta eta epsilon delta beta gamma beta delta epsilon gamma epsilon beta gamma eta zeta epsilon theta delta gamma alpha delta delta beta gamma gamma gamma theta alpha zeta beta alpha gamma theta eta theta zeta epsilon beta delta eta eta eta theta zeta epsilon zeta beta delta gamma epsilon gamma eta theta theta epsilon beta eta zeta gamma alpha zeta theta zeta eta delta delta eta zeta gamma alpha delta theta zeta delta alpha epsilon zeta beta zeta epsilon alpha epsilon alpha epsilon theta eta alpha epsilon epsilon alpha alpha theta theta theta gamma theta beta alpha zeta alpha gamma gamma gamma beta eta beta alpha theta alpha zeta beta epsilon delta alpha zeta zeta gamma zeta alpha eta eta alpha beta delta gamma epsilon beta gamma theta alpha epsilon delta eta gamma alpha epsilon epsilon zeta beta zeta theta beta theta beta beta beta gamma beta gamma zeta alpha gamma alpha delta zeta theta beta delta epsilon delta gamma epsilon alpha eta epsilon gamma epsilon gamma zeta eta delta delta epsilon delta gamma epsilon epsilon theta epsilon beta theta zeta zeta zeta theta gamma alpha eta gamma delta eta zeta epsilon delta zeta epsilon beta delta epsilon delta theta epsilon alpha beta gamma theta gamma theta zeta zeta eta alpha zeta beta zeta alpha gamma eta theta zeta zeta beta gamma zeta epsilon gamma theta gamma alpha theta theta eta eta theta delta epsilon zeta zeta theta eta epsilon alpha delta beta eta eta epsilon eta alpha gamma gamma epsilon delta gamma gamma zeta alpha beta epsilon gamma zeta theta eta gamma theta alpha gamma delta gamma epsilon epsilon beta gamma delta delta eta beta delta gamma zeta alpha theta gamma zeta beta theta eta delta theta eta theta delta beta eta gamma zeta gamma eta zeta epsilon eta eta beta zeta theta alpha delta gamma delta zeta zeta zeta alpha eta beta delta gamma theta delta alpha delta epsilon eta beta zeta eta gamma eta gamma epsilon epsilon eta theta alpha epsilon eta gamma gamma eta beta theta beta zeta beta eta alpha gamma zeta gamma eta theta alpha theta eta eta alpha epsilon eta epsilon beta theta zeta theta zeta zeta beta gamma eta eta beta gamma alpha zeta epsilon delta theta eta alpha theta epsilon alpha beta theta theta zeta alpha zeta epsilon eta zeta epsilon alpha epsilon gamma epsilon zeta eta alpha theta zeta epsilon alpha beta zeta alpha eta delta gamma beta beta gamma gamma theta eta epsilon zeta theta delta theta alpha beta zeta zeta alpha gamma beta eta beta zeta epsilon eta eta alpha gamma epsilon beta eta beta delta epsilon epsilon delta alpha zeta theta delta eta epsilon gamma epsilon alpha eta gamma epsilon epsilon alpha eta delta alpha eta gamma epsilon epsilon theta zeta beta delta delta alpha zeta delta beta epsilon theta eta zeta eta theta alpha zeta zeta zeta zeta theta gamma theta theta delta gamma eta epsilon eta alpha beta theta zeta beta delta zeta epsilon epsilon zeta theta theta alpha gamma gamma zeta eta eta alpha epsilon theta delta eta gamma eta beta beta eta delta delta beta epsilon gamma gamma eta eta zeta delta eta epsilon zeta zeta beta zeta eta delta eta theta theta epsilon theta gamma gamma eta epsilon alpha zeta theta gamma beta eta eta eta eta delta alpha delta gamma beta eta alpha zeta delta gamma theta beta zeta alpha eta delta gamma theta zeta delta gamma theta delta zeta gamma epsilon beta alpha delta theta theta delta theta delta gamma theta beta zeta gamma delta theta beta alpha alpha eta theta eta beta eta epsilon beta alpha theta theta alpha epsilon eta alpha zeta eta theta eta beta beta gamma epsilon beta gamma zeta eta zeta gamma delta theta delta epsilon beta zeta beta theta theta delta epsilon zeta epsilon theta zeta theta epsilon epsilon alpha gamma gamma epsilon alpha delta zeta zeta epsilon epsilon gamma epsilon theta theta epsilon gamma gamma gamma epsilon eta epsilon delta alpha delta epsilon gamma alpha epsilon zeta eta theta gamma zeta gamma zeta zeta eta theta gamma theta gamma theta eta alpha beta zeta delta epsilon delta delta gamma epsilon beta delta zeta theta theta alpha delta delta zeta alpha eta beta zeta beta epsilon beta alpha theta delta delta epsilon gamma delta delta theta gamma alpha epsilon epsilon beta delta beta gamma epsilon alpha delta delta delta theta theta beta alpha zeta beta delta theta beta theta beta theta delta beta delta theta epsilon alpha eta beta theta delta theta alpha epsilon eta epsilon theta gamma zeta theta epsilon epsilon epsilon delta zeta epsilon epsilon theta beta gamma theta alpha zeta theta theta gamma gamma beta eta epsilon beta theta zeta gamma delta beta delta delta epsilon epsilon zeta theta beta epsilon delta gamma zeta gamma delta eta gamma gamma gamma zeta theta eta epsilon delta epsilon theta epsilon zeta gamma epsilon zeta zeta zeta theta epsilon eta eta eta alpha eta alpha eta zeta gamma theta epsilon alpha delta delta epsilon epsilon zeta alpha zeta eta eta zeta delta delta theta epsilon beta beta delta gamma zeta theta theta alpha eta theta delta epsilon eta alpha beta alpha zeta epsilon zeta theta epsilon beta theta gamma beta zeta eta gamma gamma epsilon delta beta theta delta zeta epsilon epsilon alpha gamma beta zeta zeta theta delta gamma zeta theta delta theta zeta zeta delta epsilon epsilon alpha gamma beta eta eta eta theta alpha alpha zeta alpha eta alpha zeta theta epsilon zeta beta beta beta beta gamma beta theta delta beta eta epsilon theta theta gamma zeta beta gamma zeta theta eta alpha eta epsilon alpha eta alpha alpha delta zeta eta gamma epsilon delta zeta theta delta zeta gamma delta gamma delta epsilon gamma beta alpha alpha gamma zeta beta zeta zeta delta delta delta epsilon gamma eta epsilon delta delta zeta alpha gamma theta beta eta alpha theta delta eta alpha theta theta eta delta delta eta beta beta gamma beta gamma delta beta theta zeta zeta eta delta delta beta zeta theta epsilon beta theta alpha alpha eta eta eta beta zeta zeta epsilon beta eta epsilon delta beta eta theta alpha epsilon zeta alpha gamma delta gamma alpha gamma gamma gamma theta alpha alpha delta epsilon zeta eta gamma eta gamma gamma gamma gamma zeta beta gamma alpha theta delta epsilon eta zeta epsilon zeta alpha alpha zeta alpha theta alpha gamma eta alpha eta gamma gamma beta alpha zeta delta beta eta beta beta theta theta eta theta gamma alpha epsilon theta theta epsilon gamma zeta zeta beta theta theta delta gamma delta gamma gamma delta beta beta eta epsilon beta delta delta epsilon gamma epsilon zeta theta epsilon gamma zeta eta zeta gamma alpha eta alpha delta gamma alpha gamma alpha delta gamma eta zeta alpha theta theta zeta delta eta theta delta theta epsilon theta beta epsilon zeta zeta gamma epsilon zeta theta theta beta zeta epsilon beta zeta eta alpha gamma beta delta theta beta beta epsilon gamma zeta epsilon theta eta beta gamma eta delta gamma delta alpha delta zeta gamma alpha zeta delta gamma zeta gamma beta gamma delta gamma eta delta epsilon alpha eta epsilon eta eta theta theta zeta gamma epsilon alpha beta epsilon epsilon delta beta alpha eta gamma gamma theta gamma delta eta gamma eta eta delta delta zeta beta delta epsilon gamma eta beta gamma delta theta gamma epsilon zeta alpha beta epsilon beta theta delta theta epsilon beta theta gamma beta zeta epsilon alpha eta epsilon beta theta eta gamma theta epsilon delta beta beta zeta delta theta eta eta alpha beta epsilon epsilon epsilon zeta eta epsilon zeta beta alpha beta theta delta theta delta eta epsilon beta theta eta eta alpha beta beta beta beta delta zeta eta eta alpha zeta theta theta alpha eta eta alpha delta gamma alpha zeta beta epsilon eta alpha theta epsilon zeta zeta zeta delta epsilon theta alpha zeta theta epsilon gamma delta zeta eta zeta gamma theta beta zeta zeta delta zeta epsilon beta beta eta zeta theta gamma eta delta epsilon theta zeta zeta theta zeta gamma gamma zeta alpha beta delta epsilon epsilon gamma eta beta delta beta beta eta epsilon gamma zeta zeta eta gamma eta beta epsilon beta zeta delta theta alpha delta gamma epsilon alpha theta delta delta delta delta theta delta gamma alpha epsilon delta alpha theta theta eta delta eta gamma eta eta epsilon zeta epsilon eta zeta zeta delta zeta zeta beta gamma zeta delta gamma beta theta zeta beta delta zeta delta theta epsilon alpha theta theta theta eta beta gamma alpha theta alpha zeta theta gamma beta zeta beta epsilon epsilon alpha theta eta gamma eta delta theta epsilon epsilon zeta delta eta gamma zeta alpha alpha zeta beta beta beta eta gamma delta delta gamma gamma alpha theta beta zeta beta beta eta eta eta eta theta gamma zeta gamma beta epsilon zeta gamma eta epsilon theta beta gamma epsilon eta delta delta zeta beta zeta theta beta epsilon alpha alpha alpha zeta beta gamma alpha alpha alpha zeta eta epsilon alpha zeta beta gamma delta theta gamma delta zeta beta gamma zeta gamma theta gamma gamma theta alpha alpha beta epsilon epsilon beta zeta gamma gamma beta theta delta gamma zeta theta eta theta beta beta zeta delta zeta gamma delta theta delta eta alpha zeta alpha theta beta beta zeta eta delta epsilon zeta gamma theta delta epsilon eta eta eta eta beta theta alpha theta beta zeta theta eta epsilon theta epsilon alpha alpha delta beta epsilon gamma zeta alpha alpha alpha epsilon eta delta zeta alpha eta gamma zeta alpha delta beta theta gamma delta zeta beta delta theta delta alpha alpha eta delta beta epsilon alpha delta zeta gamma eta delta zeta eta alpha gamma gamma delta epsilon eta epsilon delta epsilon eta alpha eta eta eta epsilon delta theta epsilon alpha eta alpha theta gamma delta beta beta beta gamma theta beta gamma zeta gamma epsilon eta epsilon epsilon epsilon epsilon beta beta eta delta beta theta delta gamma alpha theta theta beta beta delta alpha eta gamma beta beta delta eta zeta theta gamma theta alpha delta delta zeta alpha alpha gamma zeta epsilon gamma zeta theta delta alpha eta eta gamma epsilon beta delta gamma delta zeta theta alpha theta beta delta epsilon gamma gamma zeta alpha beta beta eta alpha gamma zeta zeta alpha gamma beta delta alpha eta alpha alpha eta beta delta alpha beta alpha theta eta epsilon eta theta delta theta epsilon zeta theta theta gamma theta zeta delta gamma epsilon gamma eta epsilon gamma zeta alpha eta epsilon gamma alpha theta eta delta eta gamma zeta delta eta eta eta theta theta theta eta theta zeta zeta zeta eta gamma delta delta delta delta delta zeta theta eta zeta epsilon delta eta gamma theta beta theta epsilon eta alpha delta delta zeta gamma eta eta gamma theta alpha delta eta gamma epsilon gamma eta delta gamma eta zeta theta delta gamma theta zeta alpha beta beta beta alpha alpha alpha zeta beta zeta zeta delta gamma delta epsilon beta zeta zeta theta theta zeta gamma epsilon epsilon beta gamma gamma gamma eta beta eta gamma theta epsilon delta beta gamma theta beta alpha gamma eta theta eta theta gamma alpha zeta theta gamma theta alpha theta zeta zeta beta zeta beta gamma gamma epsilon eta gamma beta delta zeta epsilon beta zeta theta alpha eta gamma zeta epsilon delta epsilon alpha theta zeta gamma alpha delta epsilon epsilon theta eta zeta alpha epsilon theta beta zeta alpha gamma alpha alpha epsilon eta delta theta eta eta alpha epsilon eta zeta zeta eta eta zeta gamma zeta epsilon delta theta epsilon epsilon beta theta beta zeta beta alpha gamma beta delta beta gamma zeta beta gamma delta beta beta alpha alpha zeta beta gamma theta zeta beta beta alpha gamma delta beta gamma beta beta alpha eta delta delta alpha beta eta zeta delta theta delta theta gamma theta epsilon alpha theta gamma alpha beta gamma delta epsilon gamma delta beta beta delta gamma zeta delta gamma eta epsilon eta epsilon delta eta epsilon zeta delta alpha epsilon epsilon gamma beta alpha beta theta zeta alpha eta beta theta alpha beta zeta zeta beta epsilon zeta theta zeta theta zeta theta epsilon theta zeta delta delta gamma alpha zeta eta theta theta zeta eta theta theta theta gamma gamma eta eta alpha gamma zeta eta gamma eta zeta alpha eta gamma gamma gamma delta beta alpha eta theta gamma epsilon zeta delta alpha theta eta gamma epsilon epsilon delta delta theta theta eta eta gamma theta alpha zeta beta beta zeta theta epsilon gamma delta theta delta eta theta alpha epsilon gamma zeta epsilon eta beta gamma epsilon zeta zeta gamma zeta theta beta alpha theta zeta eta delta delta zeta delta zeta theta epsilon zeta zeta delta gamma zeta delta delta delta theta theta alpha zeta beta eta gamma zeta beta theta theta epsilon beta theta eta delta epsilon gamma zeta alpha theta delta delta alpha zeta zeta theta gamma epsilon zeta theta zeta delta delta beta epsilon alpha delta epsilon delta eta eta alpha epsilon epsilon delta gamma epsilon alpha alpha eta beta zeta theta gamma theta eta gamma theta zeta beta epsilon theta epsilon delta theta epsilon gamma gamma zeta alpha beta alpha zeta theta zeta gamma zeta eta alpha eta zeta eta gamma zeta alpha epsilon zeta alpha alpha gamma theta beta beta gamma gamma beta gamma delta beta zeta zeta zeta beta eta zeta beta theta eta alpha eta eta gamma beta delta theta gamma beta epsilon delta delta alpha alpha alpha delta theta alpha alpha delta delta gamma gamma zeta zeta eta eta alpha epsilon theta alpha eta beta alpha delta alpha delta delta theta alpha zeta beta zeta gamma eta delta gamma alpha epsilon epsilon zeta eta gamma delta epsilon zeta delta zeta epsilon eta beta epsilon theta alpha theta eta delta gamma beta alpha eta beta delta delta epsilon delta epsilon zeta theta alpha delta epsilon delta alpha epsilon alpha delta gamma zeta delta theta eta zeta theta eta gamma theta theta epsilon delta beta beta alpha alpha alpha alpha epsilon alpha epsilon theta eta gamma beta alpha zeta delta delta eta zeta beta epsilon eta alpha epsilon alpha beta epsilon gamma epsilon theta delta beta alpha theta gamma alpha beta beta delta theta epsilon epsilon alpha theta beta gamma delta beta eta eta alpha alpha theta eta theta eta gamma alpha alpha beta epsilon epsilon theta delta epsilon alpha epsilon delta delta theta alpha alpha eta theta delta alpha epsilon zeta beta zeta theta theta gamma zeta zeta gamma zeta zeta eta gamma eta zeta theta zeta epsilon gamma theta eta zeta eta beta epsilon gamma zeta zeta theta theta gamma alpha zeta gamma zeta gamma gamma delta eta beta eta beta alpha beta alpha epsilon gamma delta zeta theta beta zeta zeta eta eta delta zeta eta beta delta theta gamma delta alpha beta theta gamma beta epsilon delta eta epsilon delta beta delta alpha alpha alpha epsilon beta delta gamma beta beta epsilon theta beta eta zeta eta alpha epsilon zeta eta zeta delta epsilon eta theta alpha theta alpha theta zeta zeta alpha epsilon theta eta beta eta zeta theta alpha eta alpha theta beta theta epsilon beta beta delta beta theta eta zeta delta theta alpha zeta epsilon theta delta eta beta alpha gamma alpha theta beta gamma theta epsilon eta alpha epsilon beta theta gamma theta epsilon gamma alpha epsilon zeta eta beta gamma beta delta theta theta delta alpha eta gamma eta alpha epsilon gamma delta zeta theta delta theta beta beta gamma gamma gamma alpha theta theta zeta eta alpha zeta beta eta theta alpha epsilon eta eta beta beta beta epsilon zeta theta eta delta gamma beta alpha gamma beta gamma delta theta zeta epsilon theta beta gamma eta beta alpha delta delta gamma epsilon gamma eta alpha zeta alpha alpha beta epsilon alpha epsilon gamma zeta delta theta delta delta theta theta theta delta beta gamma eta beta gamma gamma gamma beta theta beta gamma delta eta zeta delta epsilon eta alpha eta alpha zeta eta alpha delta zeta eta theta delta gamma theta eta gamma delta epsilon theta beta eta alpha eta beta beta gamma alpha zeta epsilon eta epsilon theta zeta epsilon zeta epsilon theta theta eta gamma theta zeta beta theta zeta beta epsilon epsilon epsilon epsilon delta zeta zeta delta delta zeta epsilon delta zeta eta epsilon epsilon eta delta gamma epsilon delta zeta zeta delta theta gamma beta epsilon delta delta zeta gamma theta zeta delta eta theta epsilon theta zeta gamma epsilon beta epsilon beta eta eta alpha beta delta epsilon theta theta epsilon eta delta epsilon alpha eta theta beta theta eta eta zeta beta gamma gamma beta zeta alpha alpha beta epsilon beta epsilon gamma beta eta delta alpha alpha alpha beta zeta epsilon epsilon eta delta delta delta epsilon zeta delta gamma alpha zeta delta gamma eta delta alpha gamma beta epsilon theta beta beta delta eta delta delta theta gamma gamma delta epsilon beta alpha delta epsilon eta eta theta delta delta beta beta eta theta theta gamma eta eta alpha epsilon delta theta eta zeta zeta alpha theta eta theta delta zeta gamma beta eta gamma eta delta delta zeta zeta eta epsilon delta epsilon delta eta theta alpha delta beta theta alpha eta zeta zeta eta alpha theta eta theta eta beta eta epsilon zeta delta epsilon zeta epsilon theta gamma gamma delta delta alpha eta zeta alpha delta beta gamma beta alpha epsilon beta epsilon theta gamma epsilon gamma eta epsilon gamma delta eta epsilon gamma beta delta beta alpha gamma epsilon theta eta gamma delta alpha delta beta beta beta epsilon gamma beta theta zeta gamma gamma gamma delta alpha zeta gamma epsilon alpha alpha eta gamma gamma alpha theta gamma gamma delta zeta epsilon theta delta theta gamma beta delta gamma alpha beta eta gamma alpha epsilon theta theta eta eta theta zeta eta epsilon eta gamma theta gamma alpha delta beta zeta epsilon beta eta eta eta epsilon zeta alpha zeta theta delta alpha delta eta gamma theta gamma epsilon eta delta gamma alpha delta theta epsilon theta eta theta theta zeta delta theta gamma eta alpha delta gamma beta epsilon zeta alpha zeta theta eta eta beta epsilon zeta theta alpha theta beta zeta zeta delta epsilon gamma alpha beta gamma epsilon alpha alpha delta zeta alpha theta eta zeta theta zeta zeta epsilon zeta alpha beta theta theta eta alpha beta gamma eta zeta theta epsilon gamma gamma delta beta eta epsilon eta delta alpha alpha epsilon alpha gamma delta eta delta eta gamma eta epsilon delta alpha beta gamma alpha epsilon theta beta delta theta gamma theta beta beta theta zeta zeta theta epsilon zeta zeta gamma epsilon delta delta alpha eta theta epsilon zeta epsilon theta eta alpha epsilon epsilon beta gamma delta delta delta theta gamma beta zeta eta eta eta zeta epsilon beta delta gamma theta beta zeta alpha eta eta eta gamma eta theta alpha zeta zeta beta epsilon delta theta theta eta delta eta eta delta epsilon delta gamma zeta gamma zeta eta beta beta beta beta alpha gamma eta theta beta gamma alpha alpha theta gamma alpha gamma epsilon zeta zeta beta alpha gamma epsilon zeta beta alpha beta alpha alpha delta gamma eta theta beta epsilon epsilon beta epsilon gamma beta theta epsilon gamma beta theta delta epsilon alpha eta zeta gamma theta theta zeta epsilon epsilon eta eta delta epsilon zeta delta delta theta gamma gamma alpha beta alpha beta gamma beta theta gamma gamma beta epsilon zeta zeta delta theta eta eta beta epsilon delta theta alpha zeta gamma eta delta zeta beta alpha alpha epsilon eta gamma theta theta eta zeta theta gamma zeta eta zeta gamma alpha gamma epsilon eta beta theta zeta gamma alpha gamma delta theta epsilon delta gamma eta alpha alpha zeta zeta zeta zeta alpha beta theta delta delta gamma zeta epsilon eta delta eta alpha theta epsilon theta alpha epsilon zeta alpha eta beta theta eta theta delta zeta delta theta epsilon epsilon alpha delta epsilon eta epsilon alpha delta alpha delta theta delta eta alpha eta alpha beta epsilon eta alpha epsilon alpha theta delta epsilon beta delta eta zeta gamma delta alpha eta beta zeta delta theta alpha beta gamma epsilon gamma epsilon epsilon eta gamma eta delta gamma zeta beta epsilon epsilon beta beta gamma beta alpha delta beta epsilon gamma beta theta zeta beta theta beta theta epsilon beta zeta beta zeta gamma eta delta epsilon alpha beta beta gamma gamma zeta gamma delta theta zeta alpha zeta eta eta beta eta delta gamma delta theta beta delta theta theta eta zeta zeta delta alpha alpha theta delta alpha zeta alpha zeta zeta eta alpha zeta theta beta gamma beta beta gamma theta theta zeta beta alpha zeta delta epsilon eta epsilon alpha beta alpha epsilon gamma epsilon eta gamma epsilon zeta eta beta zeta beta theta epsilon epsilon eta alpha alpha epsilon zeta gamma beta gamma beta zeta beta beta epsilon zeta zeta beta gamma delta beta delta alpha epsilon epsilon beta epsilon epsilon beta eta alpha epsilon eta beta zeta epsilon alpha gamma beta beta alpha beta theta alpha theta zeta eta beta epsilon gamma delta beta zeta gamma gamma beta alpha theta delta gamma gamma delta eta alpha epsilon delta theta delta alpha eta delta eta delta epsilon zeta gamma eta eta epsilon epsilon delta beta eta epsilon theta alpha beta eta delta epsilon gamma gamma beta delta gamma alpha delta delta alpha beta eta gamma delta epsilon beta alpha delta beta alpha eta beta zeta delta theta gamma gamma beta zeta zeta eta eta epsilon theta delta gamma delta eta zeta delta delta theta theta epsilon eta alpha epsilon alpha eta theta gamma alpha alpha theta epsilon delta eta beta epsilon gamma epsilon epsilon gamma eta alpha theta beta theta zeta epsilon delta theta delta epsilon theta eta epsilon gamma theta delta theta gamma alpha delta theta eta delta gamma beta epsilon zeta epsilon beta epsilon epsilon eta alpha beta beta delta epsilon epsilon delta gamma theta alpha gamma alpha zeta eta zeta alpha theta delta gamma beta epsilon eta epsilon epsilon delta eta zeta beta eta theta delta delta zeta alpha zeta zeta theta delta zeta theta delta eta alpha beta beta gamma delta eta alpha delta theta eta beta beta theta gamma delta delta delta zeta beta alpha epsilon alpha eta beta epsilon theta delta beta zeta delta beta beta epsilon delta alpha theta epsilon gamma alpha theta delta eta zeta beta beta delta epsilon alpha eta zeta zeta theta gamma alpha beta alpha epsilon delta gamma gamma delta alpha gamma delta delta delta gamma epsilon theta alpha theta delta gamma alpha beta delta zeta epsilon eta gamma beta gamma theta eta alpha gamma zeta alpha gamma eta beta beta gamma delta beta gamma eta beta eta delta gamma epsilon zeta theta gamma delta theta eta gamma theta epsilon zeta epsilon delta alpha delta eta epsilon eta theta beta beta gamma alpha delta beta theta eta gamma alpha gamma delta gamma delta alpha beta delta beta zeta beta epsilon epsilon zeta gamma gamma alpha eta theta delta gamma delta beta gamma theta beta eta beta zeta theta gamma epsilon zeta alpha alpha alpha alpha alpha alpha beta zeta gamma beta beta delta theta beta theta zeta alpha alpha gamma zeta zeta eta beta epsilon epsilon epsilon gamma beta gamma theta epsilon theta beta gamma gamma epsilon alpha zeta eta theta alpha eta theta alpha alpha alpha alpha epsilon delta eta zeta zeta beta zeta epsilon gamma eta eta beta delta eta gamma epsilon delta zeta zeta alpha zeta epsilon gamma eta alpha alpha theta alpha eta delta delta eta theta zeta alpha gamma gamma beta alpha epsilon alpha beta eta epsilon beta alpha epsilon epsilon gamma theta theta delta delta eta delta beta eta epsilon delta gamma beta gamma eta epsilon gamma epsilon eta epsilon gamma alpha delta eta gamma zeta eta delta beta epsilon epsilon beta theta alpha eta epsilon eta gamma zeta zeta epsilon alpha gamma eta delta zeta zeta gamma epsilon delta alpha beta theta eta epsilon eta epsilon epsilon alpha delta beta zeta delta gamma epsilon zeta gamma theta alpha zeta alpha epsilon alpha epsilon delta delta delta delta zeta delta epsilon zeta beta eta delta gamma beta beta beta theta theta beta alpha delta delta beta delta theta delta gamma eta zeta delta alpha eta alpha beta beta theta zeta eta beta theta eta alpha alpha gamma delta alpha alpha beta zeta eta beta gamma alpha delta eta zeta theta theta epsilon epsilon beta eta alpha delta eta zeta zeta gamma alpha beta epsilon beta beta epsilon beta epsilon delta delta eta theta zeta alpha gamma eta delta gamma gamma delta gamma beta delta theta epsilon zeta zeta eta beta beta epsilon alpha epsilon gamma eta alpha zeta alpha zeta delta theta eta eta eta alpha beta beta gamma eta alpha alpha delta alpha delta zeta eta eta beta alpha eta gamma theta beta zeta delta beta gamma delta epsilon zeta zeta alpha eta gamma eta beta theta theta eta gamma delta epsilon epsilon alpha beta delta epsilon alpha theta delta zeta epsilon delta theta gamma eta alpha delta delta zeta beta alpha beta zeta theta alpha alpha zeta beta epsilon epsilon gamma alpha theta epsilon gamma eta eta eta epsilon theta beta epsilon gamma zeta eta delta gamma zeta beta epsilon alpha epsilon delta gamma delta beta gamma delta epsilon gamma eta eta alpha eta alpha theta gamma eta beta theta eta alpha theta gamma delta epsilon delta delta zeta zeta delta gamma alpha epsilon alpha zeta eta beta gamma eta zeta alpha gamma eta zeta gamma eta epsilon beta eta eta zeta eta beta delta alpha theta eta beta gamma delta zeta zeta beta epsilon delta epsilon eta gamma delta epsilon epsilon alpha alpha gamma theta delta alpha alpha eta eta eta beta alpha beta eta delta beta beta delta zeta alpha alpha beta eta alpha epsilon beta epsilon beta zeta alpha theta eta gamma delta theta eta gamma alpha zeta delta epsilon epsilon alpha theta theta theta epsilon eta gamma alpha theta gamma eta eta gamma gamma zeta alpha beta theta theta eta alpha gamma beta epsilon alpha eta epsilon beta delta delta delta theta delta beta gamma zeta zeta gamma delta zeta eta beta theta beta theta alpha delta gamma beta beta gamma alpha eta beta delta eta delta zeta gamma theta theta zeta eta beta delta delta theta beta epsilon epsilon epsilon alpha gamma zeta alpha gamma eta epsilon alpha beta eta epsilon epsilon alpha delta theta delta epsilon beta eta theta beta gamma theta theta gamma gamma beta alpha gamma beta theta gamma beta delta epsilon eta gamma zeta theta theta delta eta delta delta beta gamma zeta eta alpha zeta delta theta alpha theta eta eta beta theta theta alpha eta theta delta eta gamma alpha zeta alpha eta gamma beta gamma gamma delta zeta zeta gamma beta eta zeta alpha zeta theta beta epsilon eta gamma beta gamma beta gamma zeta eta theta alpha epsilon delta eta epsilon alpha gamma gamma alpha zeta alpha delta eta epsilon epsilon gamma epsilon zeta beta beta epsilon zeta theta theta alpha theta beta zeta gamma delta eta theta eta theta delta gamma alpha theta epsilon eta beta beta theta theta theta eta gamma eta theta gamma gamma zeta delta theta alpha zeta alpha eta gamma delta zeta alpha epsilon alpha zeta eta eta zeta theta alpha alpha beta gamma epsilon gamma zeta zeta epsilon epsilon gamma eta theta eta delta gamma zeta epsilon gamma delta gamma epsilon eta delta delta beta zeta eta gamma epsilon beta zeta gamma zeta theta gamma zeta eta gamma gamma eta delta delta zeta zeta beta delta beta eta zeta zeta alpha theta epsilon eta zeta alpha zeta theta epsilon epsilon delta alpha delta epsilon beta beta beta theta theta gamma alpha delta zeta delta epsilon alpha beta zeta delta eta epsilon beta zeta epsilon theta beta zeta zeta delta beta delta gamma zeta delta eta theta beta delta eta theta zeta alpha alpha epsilon gamma beta epsilon beta alpha zeta gamma theta eta beta theta alpha gamma theta alpha epsilon zeta gamma delta zeta eta gamma gamma zeta eta eta alpha zeta eta theta theta epsilon epsilon gamma gamma theta zeta zeta epsilon zeta eta zeta zeta epsilon gamma gamma epsilon beta beta eta zeta alpha gamma alpha alpha alpha gamma epsilon zeta delta delta beta delta gamma beta zeta zeta beta delta alpha zeta zeta theta alpha delta eta gamma beta epsilon epsilon epsilon eta eta theta gamma theta delta eta theta theta epsilon eta gamma alpha gamma zeta gamma zeta epsilon delta beta delta gamma beta gamma delta beta gamma delta gamma alpha epsilon beta alpha eta alpha beta delta eta epsilon theta gamma delta theta zeta alpha gamma theta eta beta theta gamma gamma zeta theta theta zeta epsilon zeta epsilon theta gamma eta gamma alpha zeta gamma beta theta eta delta beta eta alpha gamma theta beta theta gamma beta theta delta delta theta delta gamma alpha zeta gamma gamma gamma delta eta gamma beta eta zeta beta epsilon delta gamma delta gamma zeta zeta delta delta zeta epsilon epsilon eta beta theta theta theta delta gamma eta epsilon delta alpha zeta epsilon epsilon theta beta gamma theta theta alpha gamma gamma theta beta gamma alpha zeta delta theta theta beta delta theta zeta gamma delta gamma gamma alpha beta gamma zeta eta alpha delta gamma alpha alpha epsilon theta theta theta zeta delta zeta zeta gamma delta eta alpha alpha delta alpha epsilon epsilon delta eta beta zeta beta theta beta zeta gamma delta beta alpha gamma gamma zeta alpha beta epsilon delta theta epsilon delta alpha delta alpha eta theta beta delta theta beta beta gamma eta alpha gamma gamma beta delta gamma beta theta eta theta gamma alpha beta beta beta beta eta eta beta gamma gamma alpha beta alpha theta gamma zeta theta delta delta delta alpha epsilon delta eta eta zeta delta alpha gamma theta epsilon beta gamma epsilon gamma beta beta epsilon alpha eta gamma zeta alpha zeta gamma gamma epsilon eta gamma alpha delta zeta zeta eta beta alpha delta zeta gamma theta alpha beta delta theta beta gamma epsilon delta eta zeta zeta zeta eta epsilon gamma delta alpha epsilon beta gamma alpha gamma theta alpha eta beta zeta gamma theta delta theta epsilon alpha gamma alpha theta beta eta eta alpha eta delta gamma gamma epsilon beta zeta zeta epsilon gamma alpha delta epsilon alpha delta alpha beta alpha zeta eta theta theta zeta eta epsilon beta zeta theta zeta zeta epsilon eta gamma delta epsilon theta beta gamma epsilon theta eta alpha beta alpha alpha beta beta alpha delta alpha eta delta beta theta eta epsilon beta theta gamma eta alpha alpha beta zeta theta beta gamma alpha beta zeta alpha theta beta delta alpha eta theta beta delta alpha theta delta zeta beta alpha epsilon zeta epsilon theta epsilon delta eta gamma beta delta alpha alpha eta gamma zeta zeta theta eta eta epsilon theta alpha delta beta alpha zeta delta delta eta gamma theta epsilon eta eta eta gamma beta zeta beta delta gamma theta alpha alpha alpha epsilon eta eta eta beta epsilon zeta alpha beta beta alpha theta theta theta beta epsilon alpha alpha epsilon delta beta epsilon alpha zeta delta beta zeta gamma alpha epsilon alpha epsilon theta eta theta delta beta delta theta theta beta eta eta epsilon beta eta eta beta gamma delta theta epsilon alpha theta theta epsilon delta delta gamma zeta eta gamma delta epsilon beta theta alpha beta epsilon gamma eta gamma alpha delta theta alpha theta gamma beta beta eta alpha gamma alpha epsilon beta beta delta zeta theta gamma beta beta theta alpha epsilon alpha beta eta epsilon epsilon gamma gamma beta theta zeta delta theta zeta theta alpha gamma alpha alpha epsilon alpha eta theta zeta theta beta gamma delta eta beta epsilon eta beta beta gamma theta epsilon gamma eta gamma theta eta theta alpha epsilon alpha zeta delta beta epsilon beta eta beta zeta eta alpha theta delta gamma theta epsilon delta zeta eta zeta beta theta eta gamma epsilon alpha eta epsilon eta eta delta epsilon eta alpha epsilon theta epsilon gamma beta theta alpha gamma zeta beta eta gamma gamma zeta delta eta epsilon gamma alpha delta epsilon eta delta alpha eta gamma epsilon eta theta gamma theta beta eta alpha beta beta epsilon beta delta beta delta zeta gamma delta eta beta delta delta theta eta alpha zeta gamma delta alpha epsilon zeta beta eta zeta alpha gamma zeta epsilon epsilon theta gamma epsilon beta beta eta zeta alpha delta epsilon epsilon alpha eta eta beta theta theta beta gamma epsilon alpha alpha eta theta gamma beta epsilon beta theta epsilon eta beta delta zeta gamma eta eta epsilon eta delta eta theta epsilon eta epsilon beta zeta zeta epsilon zeta alpha beta beta gamma gamma alpha theta zeta delta eta alpha zeta beta beta theta beta eta alpha delta eta delta eta zeta delta eta gamma beta epsilon zeta zeta alpha eta theta alpha alpha alpha epsilon gamma epsilon epsilon gamma alpha epsilon eta eta epsilon eta eta theta eta alpha zeta eta epsilon gamma alpha epsilon zeta eta zeta theta theta zeta alpha theta epsilon zeta delta delta alpha eta gamma delta zeta epsilon eta eta gamma gamma eta zeta epsilon alpha beta eta zeta zeta delta alpha alpha eta gamma gamma beta gamma gamma epsilon gamma theta eta theta theta epsilon eta gamma zeta beta eta alpha beta beta gamma epsilon beta beta zeta epsilon eta alpha alpha alpha beta theta eta theta theta gamma delta beta gamma zeta theta alpha epsilon theta gamma eta epsilon gamma eta beta gamma beta gamma alpha delta epsilon delta theta zeta delta eta zeta eta alpha theta zeta gamma zeta epsilon gamma epsilon gamma beta delta zeta epsilon zeta epsilon alpha alpha eta epsilon epsilon zeta epsilon theta zeta gamma delta beta beta gamma delta gamma theta epsilon theta beta theta theta eta delta delta epsilon gamma theta gamma alpha delta eta gamma beta theta zeta alpha alpha beta eta delta alpha epsilon zeta delta alpha eta delta zeta gamma epsilon delta zeta delta epsilon alpha alpha beta beta eta theta theta epsilon beta eta zeta gamma epsilon epsilon epsilon zeta epsilon epsilon eta alpha delta zeta beta theta delta theta beta alpha zeta delta beta gamma theta epsilon eta eta gamma alpha epsilon zeta theta gamma theta beta epsilon theta gamma epsilon eta delta alpha beta eta epsilon gamma epsilon gamma delta delta theta gamma delta delta beta gamma beta theta alpha eta gamma delta gamma alpha delta gamma gamma beta alpha beta alpha gamma delta gamma alpha alpha beta eta theta eta eta beta alpha alpha theta eta beta alpha beta epsilon delta zeta eta beta epsilon alpha delta epsilon zeta delta beta alpha theta epsilon eta gamma delta zeta epsilon eta zeta beta alpha alpha delta alpha theta eta gamma alpha zeta epsilon zeta eta theta beta zeta alpha alpha eta eta eta eta epsilon beta beta alpha eta zeta epsilon beta beta delta epsilon alpha theta zeta beta eta epsilon theta alpha alpha zeta delta eta eta delta epsilon gamma eta eta gamma eta theta alpha delta eta beta theta theta beta zeta alpha zeta eta gamma zeta theta beta theta eta alpha delta gamma alpha beta gamma eta zeta theta gamma eta alpha epsilon beta alpha zeta gamma beta theta zeta zeta zeta beta theta theta theta epsilon beta gamma theta epsilon eta gamma beta beta gamma epsilon beta eta gamma eta epsilon gamma gamma gamma theta zeta beta theta alpha epsilon zeta epsilon gamma beta zeta epsilon zeta theta alpha zeta eta gamma alpha gamma gamma zeta theta alpha alpha alpha beta eta delta zeta eta epsilon alpha theta gamma delta gamma epsilon theta gamma gamma zeta epsilon alpha alpha beta theta gamma gamma epsilon theta delta beta gamma eta beta eta zeta alpha theta beta alpha zeta alpha beta beta zeta alpha gamma theta beta beta zeta gamma eta beta alpha zeta delta theta alpha gamma delta alpha alpha theta beta eta zeta alpha alpha theta epsilon beta epsilon zeta epsilon delta gamma theta eta delta eta zeta gamma epsilon epsilon theta beta eta beta gamma theta delta epsilon beta delta beta alpha epsilon zeta alpha gamma zeta eta delta gamma beta theta epsilon delta zeta theta epsilon delta zeta gamma eta alpha eta epsilon eta alpha eta theta delta delta epsilon epsilon delta epsilon gamma zeta delta eta epsilon eta gamma alpha zeta zeta alpha theta theta epsilon alpha eta zeta theta zeta zeta beta zeta eta gamma delta gamma alpha alpha zeta zeta beta alpha theta epsilon alpha beta delta beta beta beta theta eta beta alpha gamma zeta epsilon theta eta alpha gamma gamma alpha eta zeta delta zeta epsilon beta gamma alpha beta gamma gamma theta zeta zeta alpha alpha gamma eta alpha beta zeta beta zeta delta gamma zeta epsilon theta zeta alpha theta epsilon alpha theta beta theta delta gamma theta alpha zeta zeta alpha delta beta alpha alpha eta epsilon delta epsilon alpha delta beta alpha epsilon zeta gamma alpha delta eta alpha epsilon beta beta epsilon beta delta zeta gamma zeta zeta delta zeta zeta gamma alpha zeta eta beta alpha alpha eta theta alpha alpha eta eta alpha beta eta theta theta epsilon alpha delta theta epsilon gamma delta delta epsilon epsilon zeta delta alpha alpha eta theta theta eta gamma zeta zeta alpha delta gamma zeta alpha beta theta delta eta zeta epsilon gamma eta gamma epsilon beta theta epsilon gamma alpha gamma gamma delta delta epsilon zeta beta beta alpha alpha alpha eta alpha zeta delta alpha eta epsilon delta alpha gamma beta beta eta eta theta delta gamma alpha epsilon zeta theta zeta theta zeta gamma beta gamma eta epsilon gamma eta alpha delta delta eta gamma alpha gamma theta zeta zeta gamma beta epsilon theta thet \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R00 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R00 new file mode 100644 index 00000000..4898a376 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R01 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R01 new file mode 100644 index 00000000..bcf79e74 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R02 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R02 new file mode 100644 index 00000000..1f275211 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R02 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R03 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R03 new file mode 100644 index 00000000..67ef1fb7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R03 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R04 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R04 new file mode 100644 index 00000000..249b2e61 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R04 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R05 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R05 new file mode 100644 index 00000000..76c079ee Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R05 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R06 b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R06 new file mode 100644 index 00000000..c7f06e7e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.R06 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.RAR new file mode 100644 index 00000000..b6706cf7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/CMULTIV.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/COMMENT.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/COMMENT.RAR new file mode 100644 index 00000000..59647216 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/COMMENT.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/EMPTY.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/EMPTY.RAR new file mode 100644 index 00000000..057b8cd7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/EMPTY.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/FCOMM.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/FCOMM.RAR new file mode 100644 index 00000000..1f608f3d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/FCOMM.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIFIL.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIFIL.RAR new file mode 100644 index 00000000..6b97f66b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIFIL.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R00 b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R00 new file mode 100644 index 00000000..f7354245 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R01 b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R01 new file mode 100644 index 00000000..796942dd Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R02 b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R02 new file mode 100644 index 00000000..46fbdf6c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.R02 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.RAR new file mode 100644 index 00000000..5a292674 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/MULTIVOL.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/README b/arca-rar/vendor/rars/tests/fixtures/rar13/README new file mode 100644 index 00000000..d978a3a0 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar13/README @@ -0,0 +1,49 @@ + �â®â  à娢 ᮤ¥à¦¨â  à娢 â®à RAR ¢¥àᨨ 1.40 (beta) + +******************************************************************** + �¨ª ª¨å £ à ­â¨© ª ¤ ­­®¬ã ¯à®£à ¬¬­®¬ã ®¡¥á¯¥ç¥­¨î ­¥ ¯à¨« £ ¥âáï. + €¢â®à ­¥ ­¥á¥â ­¨ª ª®© ®â¢¥âá⢥­­®á⨠§  «î¡®© ¢à¥¤ ¨«¨ ã¯ã饭­ãî + ¢ë£®¤ã ¯à¨ ¨á¯®«ì§®¢ ­¨¨ ¨«¨ ­¥¨á¯®«ì§®¢ ­¨¨ ¤ ­­®£® ¯à®£à ¬¬­®£® + ®¡¥á¯¥ç¥­¨ï. +******************************************************************** + + Žá­®¢­ë¥ ®á®¡¥­­®á⨠RAR + +1. �®«¥¥ ¢ë᮪ ï 祬 ã PKZIP 2.04 ¨ ARJ 2.41 ­  ¡®«ì設á⢥ + ä ©«®¢ (.EXE,.LIB,¡®«ì訥 TXT,...) á⥯¥­ì ᦠâ¨ï. + +2. � ¡®â  á ¯®«­®íªà ­­ë¬ ¨­â¥à䥩ᮬ (mouse, á®åà ­¥­¨¥ ª®­ä¨£ãà æ¨¨, + ¢áâ஥­­ë© viewer,...), «¨¡® á ª®¬ ­¤­®© áâப®©. + +3. ‘®§¤ ­¨¥ ­¥¯à¥à뢭ëå (solid)  à娢®¢, ¯®§¢®«ïîé¨å ¯®¢ëá¨âì + á⥯¥­ì ᦠâ¨ï ¯à¨ 㯠ª®¢ª¥ §­ ç¨â¥«ì­®£® ª®«¨ç¥á⢠ ­¥¡®«ìè¨å + ®¤­®â¨¯­ëå ä ©«®¢ ­  10 - 40 % (  ¨­®£¤  ¨ ¡®«ìè¥) ¯® áà ¢­¥­¨î + á ®¡ëç­ë¬¨  à娢 ¬¨. + +4. � ¡®â  ¢ ¯®«­®íªà ­­®¬ ०¨¬¥ á  à娢 ¬¨ .ZIP,.ARJ,.LZH(.ICE) + ⨯  ARCVIEW, ­® á ¡®«¥¥ ¬®é­ë¬¨ ¢®§¬®¦­®áâﬨ (¯à¥®¡à §®¢ ­¨¥ +  à娢  ¢ SFX, ¤®¡ ¢«¥­¨¥ ¨ ¬®¤¨ä¨ª æ¨ï ª®¬¬¥­â à¨¥¢, ãáâ ­®¢ª  + ¢à¥¬¥­­®£® ¯®¤ª â «®£ ,...). + +5. ‘®§¤ ­¨¥ ¨ ¬®¤¨ä¨ª æ¨ï SFX  à娢®¢. + +6. ‘®§¤ ­¨¥ ¯®á«¥¤®¢ â¥«ì­®á⨠ à娢­ëå ⮬®¢ 㪠§ ­­®£® à §¬¥à , + á®§¤ ­¨¥ SFX  à娢­ëå ⮬®¢. + +7. “áâ ­®¢ª  ¯ à®«¥©, ª®¬¬¥­â à¨¨, § é¨â   à娢®¢ ®â ¬®¤¨ä¨ª æ¨¨, + ¤®¡ ¢«¥­¨¥ ª  à娢㠨­ä®à¬ æ¨¨ ®¡  ¢â®à¥ ¨ ¢à¥¬¥­¨ ¯®á«¥¤­¥© + ¬®¤¨ä¨ª æ¨¨, ®¡ ¨¬¥­¨  à娢 , ¢®ááâ ­®¢«¥­¨¥  à娢®¢ á ¯®¢à¥¦¤¥­­®© + áâàãªâãன. + +8. ‚áâ஥­­ ï ¯®¤¤¥à¦ª  ANSI escape ¯®á«¥¤®¢ â¥«ì­®á⥩ ¢ ª®¬¬¥­â à¨ïå. + + ‚ ¤ ­­®¬  à娢¥ ᮤ¥à¦ âáï á«¥¤ãî騥 ä ©«ë: + + README + RAR EXE -  à娢 â®à RAR + RAR DOC - ¤®ªã¬¥­â æ¨ï + OPTIONS DOC - ᯨ᮪ ª®¬ ­¤ ¨ ª«î祩 RAR + TECHNOTE DOC - ä®à¬ â  à娢  RAR ¨ § ¬¥ç ­¨ï ¯® ¬¥â®¤ã ᦠâ¨ï + RAR_BBS DOC - BBS ­  ª®â®àëå ¥áâì ¯®á«¥¤­ïï ¢¥àá¨ï RAR + WHATSNEW DOC - ®â«¨ç¨ï í⮩ ¢¥àᨨ RAR ®â ¯à¥¤ë¤ãé¨å + LICENSE DOC - ¨­ä®à¬ æ¨ï ® «¨æ¥­§¨¨ ¯® ¨á¯®«ì§®¢ ­¨î RAR diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/README-solid-flag.md b/arca-rar/vendor/rars/tests/fixtures/rar13/README-solid-flag.md new file mode 100644 index 00000000..4df35096 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar13/README-solid-flag.md @@ -0,0 +1,12 @@ +# solid_flag_cleared.rar + +A solid two-member RAR 1.4 archive written by rars, with `LHD_SOLID` (file +entry flag `0x10`, at offset +17) cleared on the second member. RAR 1.4 headers +carry no checksum, so the byte is simply flipped. + +Both members still extract under RAR 7.12, UnRAR 7.20 and rars. Below `UnpVer` +20 the per-file flag is written but never read back: solid continuation follows +the archive-level `MHD_SOLID` flag and position alone. + +The second member is 44 packed bytes standing for 2700 unpacked, so decoding it +correctly is only possible with the window carried over from the first. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/README.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/README.RAR new file mode 100644 index 00000000..14d01c9a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/README.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/README_password=password.rar b/arca-rar/vendor/rars/tests/fixtures/rar13/README_password=password.rar new file mode 100644 index 00000000..813b70bb Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/README_password=password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/README_store.rar b/arca-rar/vendor/rars/tests/fixtures/rar13/README_store.rar new file mode 100644 index 00000000..413f8411 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/README_store.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/REPEATB.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/REPEATB.RAR new file mode 100644 index 00000000..86851afa Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/REPEATB.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/SFXSRC.EXE b/arca-rar/vendor/rars/tests/fixtures/rar13/SFXSRC.EXE new file mode 100644 index 00000000..71e6533d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/SFXSRC.EXE differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/SOLID.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/SOLID.RAR new file mode 100644 index 00000000..61ee5d2d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/SOLID.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/STOREPWD.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/STOREPWD.RAR new file mode 100644 index 00000000..20caf716 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/STOREPWD.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/WITHDIR.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/WITHDIR.RAR new file mode 100644 index 00000000..a4e1e172 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/WITHDIR.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.R00 b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.R00 new file mode 100644 index 00000000..39c5dc23 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.R00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.R01 b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.R01 new file mode 100644 index 00000000..65f5a839 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.R01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.RAR b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.RAR new file mode 100644 index 00000000..93b2c84e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/ESPLIT.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/README.md b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/README.md new file mode 100644 index 00000000..b304284f --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar13/encrypted_split/README.md @@ -0,0 +1,9 @@ +RAR 1.402 encrypted compressed split member, generated by the original DOS RAR.EXE under DOSBox-X: + + rar a -v2K -m3 -ppassword ESPLIT.RAR SOURCE.TXT + +SOURCE.TXT is the first 32768 bytes of ../CMULTI.TXT. The password is `password`. +UnRAR 7.20 tests all three volumes successfully. The corresponding unencrypted +RAR 1.402 archive has byte-identical packed data after decrypting the encrypted +volumes with one continuous cipher state; restarting the cipher at each volume +boundary diverges at the first byte of ESPLIT.R00. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/rar140_av/rar140_av_patched.rar b/arca-rar/vendor/rars/tests/fixtures/rar13/rar140_av/rar140_av_patched.rar new file mode 100644 index 00000000..1501325e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/rar140_av/rar140_av_patched.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/rar140_av/rar140_noav_baseline.rar b/arca-rar/vendor/rars/tests/fixtures/rar13/rar140_av/rar140_noav_baseline.rar new file mode 100644 index 00000000..78e83224 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/rar140_av/rar140_noav_baseline.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar13/solid_flag_cleared.rar b/arca-rar/vendor/rars/tests/fixtures/rar13/solid_flag_cleared.rar new file mode 100644 index 00000000..dc3dd61e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar13/solid_flag_cleared.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/README-crafted-legacy.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/README-crafted-legacy.md new file mode 100644 index 00000000..82f5b394 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/README-crafted-legacy.md @@ -0,0 +1,37 @@ +# Crafted RAR 1.5-4.x headers + +Hand-built edge cases that ordinary encoder operation does not emit. +Compatibility observations are recorded with each fixture. + +## empty_compressed_payload_rar30.rar + +A 61-byte RAR 3.0 archive containing `empty.bin`. Its file header declares +RAR29 compression method `0x33`, zero packed bytes, zero unpacked bytes, and +the CRC-32 of an empty file. It was made from a stored empty archive emitted by +rars by changing method `0x30` to `0x33` and recomputing the file-header CRC. + +On 2026-09-12, rars, UnRAR 7.20, and bsdtar/libarchive 3.8.5 all accepted it +and extracted a zero-byte file. 7-Zip 26.00 recognized the archive but reported +the member as an unsupported method. No historical WinRAR or RAR.EXE result is +claimed: none was installed in the test environment. + +## zero_head_size.rar + +A RAR 1.5 file header with `HEAD_SIZE = 0` and the long-block flag cleared, so +the block extent is exactly zero and the next position equals the current one. +A reader that seeks and repeats spins on that offset forever. + +RAR 7.12, UnRAR 7.20 and rars all reject it, and none hang. rars gets there by +refusing any header below the 7-byte minimum rather than by comparing +positions; either route works as long as the walk cannot stand still. + +## solid_flag_cleared_rar15.rar + +A solid two-member RAR 1.5 archive with `FHD_SOLID` cleared on the second +member and the header CRC recomputed. Both members still extract, because +below `UnpVer` 20 the per-file flag is not consulted: continuation follows the +archive-level `MHD_SOLID` and position. + +The second member is 46 packed bytes standing for 2700 unpacked, so extraction +can only succeed by carrying the window across. The same edit to a RAR 2.0 +archive breaks it, which is where the flag starts being read. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/empty_compressed_payload_rar30.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/empty_compressed_payload_rar30.rar new file mode 100644 index 00000000..f090e66a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/empty_compressed_payload_rar30.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/README.md new file mode 100644 index 00000000..d1738fd8 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/README.md @@ -0,0 +1,16 @@ +# RAR 1.5-4.x Encryption Fixtures + +Focused encryption fixtures for `rar15_40_fixtures.rs`. + +| Fixture | Password | Source | Purpose | +|---|---|---|---| +| `header_enc_1234.rar` | `1234` | `node-unrar-js/HeaderEnc1234.rar` | Minimal third-party `MHD_PASSWORD` fixture. Used only to verify parsing without a password returns `NeedPassword`. | +| `header_rar300_password.rar` | `password` | spec repo `fixtures/1.5-4.x/rar300/header_encrypted_rar300.rar` | RAR 3.00 `-hp` header encryption. Decrypted `hello.txt` payload CRC32 is `0xa538535e`. | +| `header_rar420_password.rar` | `password` | spec repo `fixtures/1.5-4.x/rar420/header_encrypted_rar420.rar` | RAR 4.20 `-hp` cross-version header encryption. Same plaintext oracle as the RAR 3.00 fixture. | +| `per_file_rar300_password.rar` | `password` | spec repo `fixtures/1.5-4.x/rar300/encrypted_per_file_rar300.rar` | RAR 3.00 per-file AES-128 encryption. Decrypted `hello.txt` payload CRC32 is `0xa538535e`. | +| `per_file_rar4_libarchive_mixed.rar` | `password` | spec repo `fixtures/1.5-4.x/third_party/libarchive_rar4_mixed_encrypted.rar` | Focused RAR4 encrypted-compressed member oracle. Use member `b.txt` only; historical RAR 3.93 validates it as `This is from b.txt` with CRC32 `0xa9fa1485`, but rejects later member `d.txt` with a CRC/password error. | +| `rar4_junrar_password.rar` | `junrar` | spec repo `fixtures/1.5-4.x/third_party/junrar/rar4-password-junrar.rar` | Tiny RAR4 encrypted compressed member. Payload is `file1\n`, CRC32 `0xe229f704`; RAR 3.93 tests it OK. | +| `rar4_junrar_header_encrypted.rar` | `junrar` | spec repo `fixtures/1.5-4.x/third_party/junrar/rar4-encrypted-junrar.rar` | Tiny RAR4 header-encrypted archive with the same `file1.txt` payload. | +| `rar4_junrar_file_content_encrypted_unicode.rar` | `test` | spec repo `fixtures/1.5-4.x/third_party/junrar/rar4-only-file-content-encrypted.rar` | RAR4 per-file encrypted member with compact Unicode filename `新建文本文档.txt`; payload is `aaaaaaaaaa`, CRC32 `0x4c11cdf0`. | +| `rar4_sharpcompress_files_only.rar` | `test` | spec repo `fixtures/1.5-4.x/third_party/sharpcompress_rar4_encrypted_files_only.rar` | Whole-archive RAR4 per-file encryption fixture with three encrypted compressed files, three directories, and compact Unicode filename `теÑ�Ñ‚.txt`; RAR 3.93 tests it OK. | +| `rar4_mixed_visible_names_password.rar` | `known-pass` | generated with WinRAR 4.20 under Wine | Mixed RAR4 fixture with visible names, one unencrypted stored member (`1File.txt`), one encrypted compressed compact-Unicode member (`2中文.txt`), and one encrypted compressed ASCII member (`3Sec.txt`). Replaces the old `node-unrar-js/FileEncByName.rar` partial oracle whose member passwords were unknown. | diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_enc_1234.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_enc_1234.rar new file mode 100644 index 00000000..628e2f24 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_enc_1234.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_rar300_password.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_rar300_password.rar new file mode 100644 index 00000000..a99ac912 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_rar300_password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_rar420_password.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_rar420_password.rar new file mode 100644 index 00000000..6dd7d8a7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/header_rar420_password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/per_file_rar300_password.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/per_file_rar300_password.rar new file mode 100644 index 00000000..f88d7051 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/per_file_rar300_password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/per_file_rar4_libarchive_mixed.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/per_file_rar4_libarchive_mixed.rar new file mode 100644 index 00000000..f100de51 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/per_file_rar4_libarchive_mixed.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_file_content_encrypted_unicode.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_file_content_encrypted_unicode.rar new file mode 100644 index 00000000..ecb2873f Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_file_content_encrypted_unicode.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_header_encrypted.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_header_encrypted.rar new file mode 100644 index 00000000..88d0b440 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_header_encrypted.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_password.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_password.rar new file mode 100644 index 00000000..0110001b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_junrar_password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_mixed_visible_names_password.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_mixed_visible_names_password.rar new file mode 100644 index 00000000..1ce8603b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_mixed_visible_names_password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_sharpcompress_files_only.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_sharpcompress_files_only.rar new file mode 100644 index 00000000..d4ff767b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/encrypted/rar4_sharpcompress_files_only.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/node_unrar_js/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/node_unrar_js/README.md new file mode 100644 index 00000000..7c7bba72 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/node_unrar_js/README.md @@ -0,0 +1,12 @@ +# node-unrar-js Fixture + +Small third-party archive-comment fixture copied from the external +`node-unrar-js` corpus. + +| Fixture | Purpose | +|---|---| +| `with_comment.rar` | RAR 3.x `CMT` NEWSUB archive comment with UTF-16LE text. It exercises a larger Unpack29 comment payload than the local RAR 3.00 ASCII comment fixture. | + +The original corpus also contains password-protected examples; those are either +represented in the `encrypted/` directory or kept out of positive tests when no +clean reference oracle exists. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/node_unrar_js/with_comment.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/node_unrar_js/with_comment.rar new file mode 100644 index 00000000..b87b90c8 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/node_unrar_js/with_comment.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/README.md new file mode 100644 index 00000000..afee10ea --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/README.md @@ -0,0 +1,21 @@ +# RAR 2.9/3.x PPMd Fixtures + +PPMd-mode fixtures for the RAR 1.5-4.x container. These are copied from the +spec repository's `fixtures/ppmd/` set plus small regression fixtures. + +| Fixture | Purpose | +|---|---| +| `ppmd_lorem_rar300.rar` | Normal RAR 3.00 PPMd text member. | +| `ppmd_escape_rar300.rar` | RAR 3.00 `-m5 -mct` input containing repeated `0x02` bytes. RAR selected LZ, so this records historical mode selection rather than PPMd literal-escape handling. | +| `ppmd_mixed_rar300.rar` | Mixed text/binary PPMd archive. | +| `ppmd_solid_rar300.rar` | Solid PPMd model reuse across members. | +| `ppmd_lz_repeat_rar3.cbr` | PPMd stream with embedded one-byte LZ repeats. | +| `ppmd_lz_match_rar300.rar` | PPMd stream with embedded LZ distance matches. | +| `farmanager170.rar` | Wild solid PPMd archive whose model reaches large-context allocator territory before `Addons\Shell\FARHere.inf`. | + +The `.txt` and `.bin` files in this directory are expected plaintext payloads +used by tests. + +PPMd literal-escape handling is forced and covered at codec level. It cannot be +claimed from `ppmd_escape_rar300.rar`, because the historical encoder chose a +different mode for that input. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/binary_64k.bin b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/binary_64k.bin new file mode 100644 index 00000000..d17cbdee Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/binary_64k.bin differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/escape_64k.bin b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/escape_64k.bin new file mode 100644 index 00000000..aa8c5a6c --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/escape_64k.bin @@ -0,0 +1,1338 @@ +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal path alphabeta gammadelta +escape-char literal pat \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/farmanager170.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/farmanager170.rar new file mode 100644 index 00000000..c9f54454 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/farmanager170.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/lorem_127k.txt b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/lorem_127k.txt new file mode 100644 index 00000000..2c8ee71b --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/lorem_127k.txt @@ -0,0 +1 @@ +incididunt ut ipsum sed dolore et incididunt do et tempor aliqua adipiscing dolore amet do amet sit enim sed magna veniam enim amet do sit quis dolor minim eiusmod et magna sit tempor ut eiusmod enim ad adipiscing magna et labore dolore sed ipsum magna lorem dolor quis incididunt veniam minim ad lorem enim et eiusmod elit quis eiusmod veniam dolor adipiscing aliqua elit elit amet magna labore dolor dolor eiusmod dolore et sit do magna do veniam sit magna eiusmod magna adipiscing enim magna aliqua do labore dolor enim incididunt eiusmod aliqua elit do consectetur adipiscing consectetur ipsum enim minim sed et dolor dolor minim amet amet ipsum dolor veniam magna minim incididunt veniam dolore sed dolore elit adipiscing minim aliqua ut aliqua sed labore et minim ad veniam tempor dolor eiusmod enim sit et aliqua ad eiusmod adipiscing elit lorem quis sed sit veniam elit tempor consectetur eiusmod ut ipsum sit amet veniam elit ipsum aliqua ad magna enim minim dolor lorem sit ad adipiscing enim aliqua sit incididunt dolor tempor sit ipsum enim lorem adipiscing consectetur veniam sit et adipiscing quis ipsum minim lorem magna ut enim sit sed dolor elit dolor ad do tempor ut consectetur ipsum dolore labore ipsum enim sit veniam incididunt adipiscing sed tempor quis et aliqua consectetur veniam minim adipiscing ipsum minim consectetur consectetur eiusmod dolore sed sit enim labore minim consectetur lorem et minim ut aliqua dolore do ad tempor incididunt minim sed amet magna veniam lorem labore quis dolor eiusmod quis ipsum magna sed amet elit et tempor enim do minim tempor aliqua ad enim amet veniam do incididunt quis ut ad dolor lorem enim adipiscing veniam eiusmod consectetur elit elit ad labore incididunt veniam minim aliqua ut ipsum incididunt veniam aliqua ut minim veniam ipsum consectetur labore dolor sed veniam consectetur labore dolore et magna enim lorem ipsum et eiusmod do labore ipsum ut adipiscing magna ad dolor quis amet lorem incididunt minim ut eiusmod lorem adipiscing lorem veniam lorem minim dolore enim sit adipiscing sit enim ad adipiscing do sed veniam consectetur sit et incididunt ad dolor lorem sed labore sit sed amet ad dolore ad ad tempor sit amet sed lorem ipsum ipsum adipiscing minim sed magna eiusmod tempor aliqua ipsum quis veniam enim ad et veniam ad labore ad ut tempor magna consectetur adipiscing incididunt aliqua do lorem amet amet sed eiusmod eiusmod tempor veniam dolor eiusmod enim ipsum ipsum sed consectetur amet aliqua do tempor incididunt magna amet do sit et quis elit ipsum do consectetur dolore quis dolor do incididunt eiusmod do ut sit sit magna et et eiusmod eiusmod sit et sit veniam et ut ipsum do eiusmod quis minim amet consectetur ad aliqua incididunt ad dolor dolor dolor adipiscing quis elit ipsum incididunt lorem sit incididunt magna dolore do labore et aliqua veniam minim adipiscing ut dolor tempor elit sed aliqua consectetur ut adipiscing tempor sit dolor veniam lorem dolore labore minim adipiscing sit et incididunt sed adipiscing ad ipsum adipiscing enim amet sit adipiscing labore incididunt tempor magna amet sit enim et amet aliqua incididunt ad minim ut dolore et minim eiusmod et et ad minim adipiscing magna enim elit lorem eiusmod veniam quis eiusmod eiusmod ipsum dolore amet sed enim amet incididunt aliqua do veniam veniam et dolor dolor dolore ipsum dolor elit amet ipsum do lorem labore eiusmod consectetur amet ad labore tempor dolore incididunt dolore dolore ipsum aliqua dolor minim dolore enim dolor quis ut adipiscing do magna enim ut et incididunt enim aliqua elit lorem minim lorem quis consectetur do dolore aliqua sed eiusmod dolor et sed do ut incididunt incididunt ipsum consectetur ad amet elit do quis eiusmod ipsum ipsum et ut amet et enim veniam dolor minim veniam amet tempor ut ipsum enim labore incididunt labore ipsum sit et amet lorem ipsum enim enim amet ad eiusmod sit veniam magna ad tempor adipiscing incididunt et sit ipsum enim veniam labore enim ad eiusmod ad sit minim veniam enim do amet incididunt do quis minim sit dolore adipiscing ipsum incididunt labore tempor adipiscing labore tempor ad dolor ipsum ipsum et sed lorem dolore minim aliqua aliqua adipiscing elit dolor ad dolore veniam dolore ut dolore do sit amet ut aliqua ut dolor sit ut dolor sit ut amet quis lorem labore ut minim ut lorem et eiusmod quis sed dolor tempor dolor sit tempor veniam lorem tempor tempor consectetur lorem elit tempor dolor enim amet adipiscing lorem adipiscing minim minim quis sit quis lorem do tempor veniam lorem enim elit amet consectetur labore sit et tempor veniam sed amet lorem adipiscing tempor eiusmod et do do magna ad eiusmod consectetur aliqua dolor sit magna aliqua do consectetur incididunt amet amet elit eiusmod dolore elit elit consectetur do tempor ut minim ipsum amet enim lorem incididunt dolor veniam dolor amet ut do magna ut quis amet aliqua ut do ad tempor dolor elit labore ad tempor ad dolore ipsum incididunt ut lorem ut quis eiusmod labore adipiscing tempor do et dolor consectetur sit sed sit magna enim veniam amet veniam labore incididunt consectetur ut ut consectetur elit labore eiusmod dolore amet tempor labore ad ad dolor et adipiscing do lorem veniam labore enim labore lorem adipiscing do sit ad do magna enim amet ut veniam et dolor minim et elit magna incididunt sed ad lorem sit sed minim ipsum lorem sed incididunt dolore aliqua veniam incididunt labore sit quis sed tempor do minim adipiscing enim dolor ipsum dolor sed do magna eiusmod sit dolore elit consectetur dolor ut do do dolore amet aliqua dolore ad adipiscing magna sit ut ad magna incididunt quis sed do labore tempor aliqua ad amet consectetur sit veniam sit incididunt incididunt aliqua labore amet magna minim do tempor ad et quis ut adipiscing et et veniam dolore eiusmod et ad ipsum labore do amet quis et ipsum enim adipiscing lorem tempor et incididunt lorem dolore dolor minim dolor minim quis minim incididunt lorem tempor ipsum sit enim lorem sed ad veniam do quis elit amet aliqua do adipiscing sit ut labore veniam eiusmod incididunt consectetur eiusmod ut ad minim ut amet labore veniam amet dolore eiusmod amet adipiscing consectetur labore tempor incididunt ut et incididunt quis elit adipiscing labore adipiscing aliqua veniam ipsum incididunt ipsum elit ad dolor consectetur tempor ipsum quis ad minim consectetur elit enim do enim dolor veniam dolore do tempor ut labore ipsum ad veniam dolore minim ad magna quis ut aliqua labore et sed veniam et adipiscing eiusmod sed ipsum ipsum ipsum consectetur tempor lorem do ad lorem amet dolor ut minim elit enim incididunt magna elit labore adipiscing eiusmod enim sit enim dolor eiusmod eiusmod magna labore eiusmod sed lorem dolore ipsum adipiscing tempor dolor adipiscing dolore tempor adipiscing adipiscing sed minim quis quis do do dolore incididunt sed et tempor veniam elit ipsum do magna dolor lorem labore et quis labore ipsum ut et labore labore sit dolor dolor elit sit amet ut adipiscing labore enim dolor ut magna incididunt ipsum consectetur elit et elit amet sed tempor eiusmod ut sit magna do enim magna adipiscing veniam do labore dolore enim labore magna ad sed sed elit lorem sit enim veniam sit consectetur quis ut elit adipiscing do quis minim lorem quis magna dolore ut ipsum sit incididunt ad sed sit quis aliqua tempor elit minim veniam veniam magna minim do elit quis elit dolor dolore do minim eiusmod elit tempor ad et do aliqua consectetur amet lorem magna dolore eiusmod tempor aliqua ad lorem amet incididunt amet consectetur dolore dolor amet adipiscing et aliqua veniam adipiscing elit quis amet elit incididunt tempor enim aliqua amet ad et sit enim lorem dolore enim tempor et labore do lorem elit magna ad consectetur minim et quis et magna eiusmod veniam dolor sed amet enim incididunt veniam adipiscing eiusmod do incididunt ipsum adipiscing ipsum eiusmod quis quis elit eiusmod labore minim quis minim minim elit sed tempor minim consectetur do lorem tempor aliqua magna ipsum quis ad amet tempor lorem et ad ipsum lorem elit ipsum lorem elit ad eiusmod dolor ipsum tempor minim ut amet adipiscing labore ut amet tempor do consectetur ad eiusmod quis quis ut incididunt lorem ut sed magna magna quis minim veniam labore ipsum aliqua sit ut incididunt consectetur lorem dolore amet enim minim dolore quis veniam amet dolor eiusmod elit consectetur elit lorem consectetur quis minim magna consectetur veniam dolor ut enim sit enim ad labore veniam amet enim enim ipsum sed eiusmod quis quis incididunt lorem ad ipsum et dolor tempor do minim amet labore elit dolore tempor consectetur quis incididunt eiusmod sed et incididunt lorem do dolore do magna et ipsum magna aliqua magna sed minim ipsum labore incididunt quis sit incididunt tempor et ipsum lorem sed quis ipsum sed minim minim aliqua veniam do minim adipiscing dolore dolore eiusmod incididunt sed adipiscing sit aliqua eiusmod elit aliqua minim quis magna minim tempor consectetur amet eiusmod quis lorem aliqua ipsum aliqua amet tempor tempor do ad do eiusmod et incididunt enim ut consectetur lorem amet aliqua ipsum labore amet eiusmod lorem quis et minim minim sed quis enim adipiscing dolor magna ut sed consectetur dolore consectetur dolor minim ad consectetur aliqua sit dolore ad magna enim incididunt ut sed do do lorem ut veniam sed sed magna dolore magna eiusmod eiusmod adipiscing veniam ut amet lorem dolore amet minim veniam aliqua incididunt tempor labore ipsum magna ut ad enim elit lorem tempor dolore consectetur minim adipiscing ad tempor ad veniam et lorem quis quis elit aliqua elit sed consectetur ut dolor aliqua labore elit quis veniam labore dolore veniam sit adipiscing consectetur labore dolor ut ad incididunt sed sed ut tempor enim eiusmod dolor do lorem et lorem sed adipiscing incididunt incididunt ut ad ad minim incididunt veniam ipsum aliqua labore tempor aliqua amet aliqua veniam sed eiusmod lorem incididunt et dolore amet ipsum dolor aliqua tempor tempor lorem dolor adipiscing veniam sit minim magna et ipsum eiusmod lorem eiusmod incididunt amet ad sed ut minim amet enim amet incididunt do dolore ipsum consectetur amet amet et veniam ad veniam quis ipsum quis dolore ipsum magna veniam quis veniam ad incididunt consectetur tempor aliqua dolor dolor magna consectetur sed adipiscing sed eiusmod veniam veniam sed sed dolore labore amet labore magna amet ipsum ad aliqua consectetur ad dolore ipsum eiusmod dolor adipiscing ad labore enim elit labore dolore consectetur veniam eiusmod ad amet et magna ipsum magna dolor dolore eiusmod lorem dolor sit ut enim tempor aliqua labore eiusmod incididunt dolore tempor ad sit amet eiusmod lorem consectetur quis amet lorem eiusmod enim adipiscing ipsum ut ad ipsum veniam do incididunt ipsum enim veniam consectetur tempor dolor ut ipsum labore tempor enim enim sed minim do aliqua labore ut consectetur lorem labore sed adipiscing incididunt dolor tempor sit sit lorem tempor lorem consectetur incididunt enim veniam ad lorem eiusmod labore magna quis veniam et et dolor ipsum magna incididunt sed lorem ad dolore sit dolor eiusmod tempor sit et ipsum amet dolore ad do ipsum lorem incididunt eiusmod consectetur magna veniam amet consectetur consectetur consectetur ad minim elit enim eiusmod lorem et ad minim incididunt ipsum elit elit ad do eiusmod consectetur elit tempor elit consectetur ut labore tempor aliqua amet incididunt aliqua lorem consectetur aliqua lorem minim incididunt veniam consectetur amet lorem lorem eiusmod dolore lorem ipsum ipsum sit aliqua enim amet amet minim incididunt lorem ut ut aliqua minim eiusmod veniam elit amet tempor dolore adipiscing magna incididunt dolor amet ut aliqua minim tempor sit ut ut elit et incididunt elit incididunt elit ad et incididunt aliqua dolor sed sed dolore tempor magna lorem enim enim et elit sed ipsum enim eiusmod incididunt ad sit magna ipsum amet veniam incididunt lorem ut quis incididunt ut sit veniam labore enim labore consectetur consectetur eiusmod et ut consectetur aliqua do dolore sit tempor tempor amet ad tempor et ad dolore ipsum adipiscing sed consectetur quis aliqua eiusmod do incididunt ad ipsum do magna ut ipsum minim ut sed incididunt quis adipiscing tempor amet amet sit enim tempor consectetur lorem ut aliqua incididunt labore dolor ad veniam veniam minim dolor ut magna quis magna amet consectetur amet adipiscing consectetur elit lorem dolore amet et tempor enim quis do veniam eiusmod minim sit ut sed consectetur eiusmod ad dolore eiusmod magna veniam amet incididunt quis quis magna do elit incididunt tempor incididunt et dolore do ut ut sit minim amet amet lorem aliqua enim ad veniam dolore sit veniam ad veniam adipiscing enim ad enim dolore sit sed veniam quis enim consectetur incididunt dolor ipsum lorem sit tempor quis et eiusmod sit minim labore tempor aliqua veniam sed minim et elit consectetur magna magna dolor dolore consectetur lorem ad consectetur dolore ut enim minim adipiscing labore veniam minim incididunt sed lorem aliqua amet incididunt consectetur labore aliqua ipsum incididunt dolor ad aliqua incididunt eiusmod elit dolore labore ipsum et enim sit sed dolore et magna ad incididunt et sed consectetur elit magna tempor consectetur do enim amet labore dolor dolor et incididunt aliqua ut magna dolor sed et elit sit do amet tempor sit amet minim ipsum minim amet aliqua magna adipiscing lorem ipsum incididunt magna veniam enim et veniam veniam sit et magna tempor eiusmod sit minim lorem elit elit et do sed elit lorem et tempor dolore eiusmod dolor dolor do aliqua ut elit quis tempor incididunt amet elit do adipiscing quis et minim tempor do incididunt enim amet sit incididunt tempor dolore et elit ad veniam tempor ad tempor ut sed tempor incididunt veniam do sit et do sit labore amet tempor elit quis consectetur eiusmod et elit sit minim incididunt incididunt labore dolore labore aliqua enim elit minim incididunt dolore do et elit eiusmod dolore minim lorem dolor et eiusmod incididunt elit ut ipsum aliqua ipsum ut dolor sed adipiscing quis eiusmod consectetur sit consectetur veniam tempor lorem elit ipsum lorem incididunt magna lorem amet sit enim enim adipiscing dolor quis labore adipiscing lorem dolore enim ut dolor magna consectetur elit elit ut incididunt et lorem ut adipiscing incididunt ipsum enim sed lorem aliqua tempor veniam tempor eiusmod minim labore ad amet enim dolore dolor sed sit veniam sit sed lorem veniam amet enim minim amet incididunt adipiscing aliqua minim eiusmod adipiscing ut dolore dolore sit magna sit veniam et sit dolore labore et consectetur labore magna eiusmod amet ut sed incididunt dolor aliqua dolore eiusmod elit labore elit tempor et ut lorem labore quis lorem magna incididunt labore elit ut elit sed et et amet elit labore do tempor ad et enim amet dolore minim dolor adipiscing do dolore ad minim sit ipsum amet eiusmod ipsum eiusmod enim consectetur labore incididunt minim adipiscing ut et adipiscing consectetur incididunt ipsum eiusmod minim dolore adipiscing quis eiusmod consectetur dolore magna veniam enim aliqua ut amet ad veniam enim et minim aliqua adipiscing enim labore ipsum elit et enim eiusmod magna quis adipiscing lorem ipsum ipsum amet elit labore minim enim elit minim minim sit dolore ut ad ad do minim eiusmod et minim adipiscing consectetur tempor magna tempor veniam labore incididunt labore minim eiusmod labore dolor amet elit sit amet incididunt labore veniam magna adipiscing tempor veniam ipsum lorem incididunt adipiscing dolor ut aliqua aliqua ut magna adipiscing ad lorem minim amet dolore ad et do incididunt magna magna dolor dolor enim quis et lorem adipiscing ut enim tempor do dolor quis lorem elit enim lorem aliqua dolore tempor amet labore sed sit aliqua labore sed elit incididunt incididunt quis dolore eiusmod elit aliqua sed dolor ipsum amet eiusmod ad aliqua aliqua et dolor dolore et ut eiusmod incididunt quis lorem sit ut enim minim amet amet lorem sit veniam ut elit ipsum ad sed sed incididunt sit tempor do tempor enim ad adipiscing quis magna magna ipsum eiusmod dolore magna lorem consectetur et lorem ad sed eiusmod amet tempor sit enim magna ut dolor sed aliqua sit dolore dolor aliqua amet et ipsum dolore elit tempor sit magna veniam adipiscing consectetur veniam minim elit incididunt adipiscing enim veniam quis labore tempor consectetur sed quis tempor magna adipiscing adipiscing labore eiusmod amet sed magna minim lorem dolor minim eiusmod do veniam eiusmod ipsum dolor enim lorem do sit consectetur veniam sed adipiscing dolore magna tempor labore dolor ut dolore magna ut sit dolor aliqua labore eiusmod consectetur do elit ad ut minim veniam do ipsum lorem lorem ad sed dolor minim eiusmod eiusmod magna ad do ipsum veniam elit veniam dolore amet sed enim ipsum incididunt ut sed aliqua elit sed eiusmod magna et labore veniam sit ut tempor consectetur dolore ipsum elit veniam et incididunt lorem elit dolor do sit consectetur amet labore lorem tempor dolore ut sit dolore elit et do sed incididunt ut et veniam ipsum consectetur veniam dolore tempor adipiscing elit amet elit do amet consectetur magna sed magna lorem consectetur elit quis ut veniam consectetur elit magna amet aliqua minim consectetur quis elit dolore do veniam quis lorem quis labore adipiscing ipsum ut consectetur incididunt dolore incididunt adipiscing incididunt magna dolore et sit amet dolor magna lorem dolor lorem quis elit quis incididunt labore minim sed ut tempor minim incididunt sed quis dolor consectetur aliqua elit quis adipiscing ipsum ipsum dolore dolor veniam labore elit labore veniam minim amet enim ad dolore do lorem enim amet tempor minim tempor tempor do dolore ad veniam sed amet lorem et quis adipiscing enim incididunt eiusmod et do labore amet et do veniam eiusmod tempor ad enim incididunt incididunt aliqua incididunt veniam dolore do minim aliqua elit adipiscing ut adipiscing sed dolore consectetur dolore amet enim dolore magna dolore ad et eiusmod sed ut do labore ad dolore ipsum quis adipiscing sed labore veniam dolor lorem adipiscing enim sit sed lorem adipiscing consectetur enim consectetur minim tempor enim sed lorem minim minim lorem lorem enim consectetur do elit incididunt et quis tempor tempor dolor aliqua ut sit veniam lorem consectetur lorem enim magna sed tempor quis adipiscing incididunt adipiscing eiusmod consectetur consectetur et minim amet enim aliqua dolore ut aliqua ad elit lorem ipsum ut enim ipsum ad ad ad amet dolore dolor ut do eiusmod ad ad enim adipiscing veniam ipsum eiusmod amet eiusmod minim quis sed et et enim eiusmod sed ad dolor labore consectetur sit dolore sed magna veniam incididunt enim labore ut minim consectetur enim ut amet minim minim sit aliqua sit consectetur quis et do minim quis sit minim ut sed sed sit ad tempor dolore magna dolor lorem ipsum enim lorem amet sit ut incididunt et lorem eiusmod ad amet dolor veniam incididunt ad aliqua quis elit ipsum adipiscing ut lorem ipsum enim incididunt amet magna quis veniam do ut incididunt magna ipsum quis labore magna amet magna amet ad adipiscing aliqua consectetur dolor aliqua et do sed sed adipiscing incididunt minim lorem ad ipsum dolore dolor lorem ad do amet amet dolore amet sit eiusmod lorem quis eiusmod amet eiusmod aliqua dolor dolor tempor adipiscing sed labore eiusmod minim et et magna dolore dolor dolor eiusmod labore incididunt incididunt magna et magna ut dolore dolor elit amet ad incididunt dolore eiusmod amet dolor amet amet ipsum incididunt ad aliqua sit minim dolor dolore sed aliqua do tempor minim quis et incididunt aliqua eiusmod quis minim enim labore dolore tempor elit dolore ipsum quis incididunt tempor eiusmod dolore lorem elit aliqua dolor et sit tempor ut enim elit magna lorem dolore dolore sit ipsum aliqua tempor ipsum tempor consectetur eiusmod sed et et ut lorem ad elit ad ad et quis quis tempor minim quis consectetur ipsum quis et dolore minim consectetur incididunt veniam do minim aliqua et adipiscing lorem veniam labore enim enim minim consectetur enim labore eiusmod aliqua eiusmod ut aliqua ad amet dolore ut incididunt elit tempor incididunt do elit adipiscing sed elit magna aliqua tempor labore elit enim eiusmod enim ad amet eiusmod elit amet sed magna consectetur magna veniam veniam enim sit labore et adipiscing veniam consectetur enim tempor enim elit veniam dolor magna et adipiscing do incididunt sed dolore adipiscing tempor ut eiusmod quis lorem ipsum adipiscing adipiscing enim consectetur magna labore dolor sed dolore do veniam et ut eiusmod minim quis veniam veniam incididunt ipsum sit ipsum eiusmod consectetur sed enim quis magna labore do magna ut et tempor elit minim dolor quis dolor ad dolor dolore lorem minim aliqua dolore eiusmod quis enim sit aliqua elit lorem ipsum sit quis do dolor aliqua incididunt veniam amet aliqua enim dolore adipiscing sed do consectetur consectetur aliqua elit amet amet elit sed et do amet quis enim sed adipiscing dolor amet minim ut consectetur adipiscing aliqua lorem tempor magna minim et minim amet quis sed dolor quis veniam enim veniam sed enim quis elit lorem amet adipiscing quis aliqua minim lorem ut amet amet et magna ipsum dolore eiusmod elit minim consectetur labore eiusmod minim labore elit ad ad quis sed minim veniam adipiscing sit tempor et sed tempor magna aliqua do adipiscing do magna incididunt dolor consectetur incididunt sit aliqua ad sit dolore amet veniam labore amet ut sed quis magna adipiscing consectetur quis dolor sed labore eiusmod incididunt dolore ipsum tempor incididunt quis quis incididunt magna amet veniam veniam magna aliqua amet ad ipsum do enim labore dolore aliqua ut adipiscing labore adipiscing incididunt dolor sit magna incididunt sit dolore minim labore quis ad consectetur ipsum adipiscing elit elit sit enim incididunt ut dolor magna adipiscing dolore adipiscing adipiscing et dolor adipiscing sit dolore amet enim adipiscing magna aliqua dolore ad enim ut enim incididunt ad sed minim et ad ad enim lorem tempor tempor adipiscing veniam ad ipsum adipiscing labore adipiscing dolore minim adipiscing lorem amet elit labore et dolor enim sed ad do incididunt eiusmod quis quis minim veniam do minim dolor enim sit et tempor enim enim ad dolor elit amet ut elit lorem et do amet dolore ad tempor quis minim amet sit dolore eiusmod eiusmod minim quis dolor tempor adipiscing adipiscing do dolore veniam ut ad quis quis incididunt incididunt ipsum amet labore dolor do ad minim amet dolore adipiscing enim et tempor amet aliqua amet aliqua ut sed quis adipiscing sit consectetur amet labore magna ut consectetur ipsum do et aliqua dolor minim tempor ad sit sit et elit dolore ipsum lorem veniam minim aliqua dolor labore sed quis aliqua sed veniam magna incididunt consectetur elit ad sit minim aliqua quis dolore consectetur et ad ad elit lorem elit amet veniam enim minim amet ad amet aliqua enim consectetur dolor sit sed sit sed enim labore aliqua sed aliqua ut ipsum veniam dolor amet labore ipsum tempor eiusmod sed ad ut incididunt lorem do ad elit adipiscing dolor aliqua quis amet elit dolore adipiscing labore eiusmod lorem sed ut aliqua dolore magna amet veniam eiusmod veniam aliqua ad magna incididunt adipiscing do sit labore ad ad sed elit ad sed do dolor tempor amet labore lorem incididunt ad labore ipsum ad do dolore enim ut labore enim aliqua quis dolore adipiscing consectetur lorem sed sit incididunt veniam sed adipiscing et et sed dolore incididunt amet enim et sed consectetur veniam lorem adipiscing eiusmod sed veniam veniam sit minim lorem ut do eiusmod enim quis adipiscing eiusmod ut amet ad sit elit tempor quis et labore consectetur minim tempor ut ut ut tempor aliqua sit dolor magna labore amet sit lorem dolore veniam incididunt ad et elit incididunt ad quis sit ut ipsum minim do enim consectetur do sed dolore sit magna minim minim eiusmod lorem amet lorem dolor eiusmod eiusmod labore adipiscing elit tempor minim aliqua labore sed magna consectetur labore incididunt lorem sit lorem sit eiusmod quis adipiscing do magna sit dolore do dolor sit magna enim do quis ipsum elit sit minim consectetur labore consectetur consectetur labore sed veniam consectetur labore veniam aliqua lorem magna incididunt lorem sit sit do dolor ad incididunt adipiscing sed quis incididunt elit labore ad enim dolore magna tempor consectetur ut ad lorem aliqua ut enim elit do tempor dolore eiusmod et ad quis labore quis veniam sed tempor sed consectetur consectetur elit labore eiusmod ad lorem tempor et enim veniam do minim elit lorem sit labore quis dolore consectetur adipiscing incididunt veniam enim eiusmod magna amet ipsum et ut labore labore tempor lorem adipiscing aliqua adipiscing dolor et amet minim dolor quis dolore veniam minim magna incididunt enim sed adipiscing dolor ut lorem magna et veniam quis eiusmod quis lorem adipiscing enim do ut labore veniam dolore veniam do dolor ipsum dolor tempor incididunt ad ut incididunt ad veniam ad elit do et incididunt magna lorem ad ut ad do minim et amet incididunt minim ut incididunt veniam aliqua quis enim do et magna tempor do ut aliqua ad consectetur tempor ipsum ut labore labore do dolor minim elit eiusmod elit incididunt lorem enim eiusmod amet consectetur et dolor sit consectetur enim ipsum elit ad consectetur lorem ad dolore minim ut minim ut consectetur do incididunt labore do magna ipsum consectetur et consectetur sit minim sit et quis do quis do do incididunt ipsum aliqua sed eiusmod labore dolor ipsum ad veniam do elit labore ad do magna adipiscing incididunt dolor ut ipsum ipsum dolor tempor lorem do do ipsum lorem ipsum enim ipsum amet enim consectetur et ad dolor aliqua amet lorem quis magna amet sit dolor veniam dolor enim minim sit adipiscing sit et ut dolor sit et veniam labore dolore quis consectetur magna sit amet tempor quis ut dolore amet dolor ad dolore ipsum quis magna elit sit quis incididunt tempor dolore eiusmod magna veniam aliqua magna incididunt do elit aliqua veniam dolor amet enim incididunt tempor quis do tempor dolore tempor lorem lorem lorem amet sed quis minim veniam adipiscing ad amet consectetur sit et quis et minim amet sed sed amet ipsum consectetur minim tempor quis eiusmod adipiscing ad sit dolore et amet ad sed eiusmod adipiscing ad sit enim minim veniam elit tempor incididunt sit sed incididunt magna tempor ipsum ad eiusmod elit incididunt ipsum ut magna sed consectetur sed ipsum quis sed quis dolor eiusmod ad ut lorem minim sed elit ad veniam consectetur sed tempor dolore ad sit consectetur et adipiscing aliqua enim aliqua dolor ut do veniam minim ipsum sed lorem lorem enim consectetur eiusmod consectetur incididunt elit aliqua magna adipiscing ipsum ad magna adipiscing labore magna minim labore aliqua enim eiusmod tempor tempor sit lorem ut tempor consectetur et elit quis minim quis ad quis quis ad incididunt enim lorem minim adipiscing enim quis et ut incididunt consectetur amet minim ut incididunt labore quis quis ad quis adipiscing sed dolor veniam consectetur aliqua lorem do quis consectetur sit amet sed consectetur incididunt sed dolore ad ut eiusmod magna quis ut et labore sit sed enim eiusmod sed sed quis do incididunt enim dolore lorem et ad dolore dolore adipiscing ut sed quis ad ut quis ipsum lorem ut magna dolor sed veniam lorem lorem et minim incididunt magna tempor magna et dolore labore magna quis quis veniam et aliqua consectetur aliqua lorem enim tempor adipiscing do ut dolor elit sit consectetur enim labore ad amet et eiusmod labore veniam amet et sed et eiusmod veniam dolore quis elit ipsum tempor dolore sed incididunt magna ad enim sit do amet dolore amet amet dolor eiusmod adipiscing ipsum lorem et et dolor aliqua dolore ipsum amet ad labore aliqua amet aliqua sed dolor do tempor amet labore tempor ipsum eiusmod sit sit enim minim sit do minim do aliqua tempor consectetur lorem minim et incididunt incididunt ad adipiscing dolor ut sed et ut minim et incididunt amet ut adipiscing ut ad adipiscing sed dolor amet labore eiusmod tempor do labore eiusmod elit consectetur labore amet veniam eiusmod elit et amet sit magna aliqua enim labore quis eiusmod consectetur dolor minim eiusmod consectetur sed tempor minim veniam sit sit incididunt eiusmod eiusmod do eiusmod lorem adipiscing minim ad do ad consectetur consectetur magna consectetur et elit dolor veniam dolore sed veniam sit ipsum incididunt quis dolor incididunt labore et minim amet veniam amet ipsum sit amet ad sit incididunt enim sit do dolor aliqua aliqua sit adipiscing quis do incididunt consectetur dolore labore veniam sed eiusmod ad consectetur ad incididunt sed consectetur consectetur quis dolor adipiscing ad dolore tempor minim quis labore minim dolor eiusmod dolore ut do labore ipsum amet tempor consectetur et elit elit dolor amet labore ut sit tempor elit sit eiusmod amet adipiscing elit enim magna eiusmod minim et eiusmod aliqua ad et dolore minim sed do lorem enim minim enim consectetur sed adipiscing dolore magna veniam sit do labore consectetur sit ipsum labore dolore dolore magna enim aliqua magna quis minim incididunt ut eiusmod enim eiusmod elit eiusmod et amet dolore quis quis lorem quis et amet lorem dolor magna lorem amet sit ipsum eiusmod do minim adipiscing dolor minim ipsum elit quis consectetur tempor incididunt ad dolore enim minim tempor ut adipiscing incididunt quis ipsum enim quis elit aliqua lorem labore quis lorem ut amet adipiscing dolor aliqua ad amet eiusmod sed do labore eiusmod dolore ut et eiusmod sed dolore do labore magna eiusmod incididunt minim eiusmod incididunt sed veniam lorem adipiscing veniam labore do incididunt ut quis quis minim amet consectetur et adipiscing consectetur enim adipiscing dolor sed dolore dolore consectetur aliqua consectetur minim et sit et elit ut consectetur sit ipsum dolore ipsum veniam quis dolor quis quis sit aliqua amet amet aliqua adipiscing ad tempor tempor sit sit incididunt et amet eiusmod aliqua dolore adipiscing enim elit enim aliqua labore enim do sit amet ut sit quis adipiscing veniam amet consectetur veniam sed sed minim eiusmod consectetur labore veniam eiusmod magna sit dolor incididunt lorem ipsum adipiscing eiusmod ut elit eiusmod ut aliqua sit eiusmod dolor et enim et do elit incididunt veniam ipsum tempor eiusmod consectetur dolor magna sed labore lorem enim ad consectetur amet quis magna aliqua eiusmod minim quis labore veniam dolor incididunt sed labore do lorem incididunt lorem labore sit amet do eiusmod do enim amet et labore lorem enim eiusmod amet ut lorem do consectetur aliqua consectetur dolor ut dolore labore magna ad amet ut eiusmod amet veniam magna tempor aliqua consectetur dolor dolor elit sit adipiscing et veniam ad adipiscing lorem incididunt eiusmod sed tempor aliqua sed labore dolore quis ipsum do labore ut elit quis quis et lorem dolore enim quis incididunt veniam eiusmod adipiscing aliqua tempor do consectetur consectetur quis veniam sed adipiscing sit tempor quis incididunt amet do sed veniam enim consectetur incididunt lorem minim dolor consectetur incididunt ut dolor dolor minim aliqua elit magna dolor adipiscing eiusmod quis labore magna veniam enim sit consectetur consectetur enim aliqua enim ipsum labore sed adipiscing magna ipsum et dolor lorem ad ipsum enim adipiscing consectetur et minim adipiscing dolore do aliqua quis aliqua dolor amet tempor magna lorem enim eiusmod ad do magna elit aliqua do lorem aliqua ut tempor dolor eiusmod sed tempor sit do aliqua amet tempor ipsum quis lorem ad enim incididunt lorem ut elit ut aliqua labore do lorem consectetur veniam aliqua labore magna dolor ad lorem minim ipsum dolor amet incididunt incididunt enim consectetur quis elit ad magna sit lorem ipsum incididunt incididunt do consectetur incididunt lorem minim eiusmod ad ut dolore adipiscing dolore tempor enim elit minim ipsum ut et dolore ad ipsum adipiscing eiusmod labore quis consectetur aliqua adipiscing adipiscing ipsum adipiscing sit do eiusmod eiusmod quis do amet consectetur aliqua quis ut sed ipsum incididunt sit aliqua et consectetur incididunt elit ut incididunt amet magna ad labore enim veniam tempor ipsum ad enim dolore elit adipiscing dolore incididunt et elit aliqua quis dolor magna sit amet consectetur do lorem eiusmod sit amet enim ad dolore enim consectetur magna ut sed lorem tempor amet ipsum dolore adipiscing veniam labore minim enim sed ut ipsum elit aliqua sed adipiscing eiusmod minim sit veniam sed et tempor do ipsum sed veniam sit tempor do adipiscing eiusmod elit elit adipiscing adipiscing minim incididunt sit magna sed minim consectetur tempor incididunt lorem magna minim ipsum magna ut minim sit sed veniam adipiscing dolore do quis dolor sit dolore ut minim magna tempor consectetur amet sit do quis quis do consectetur et do elit magna aliqua veniam elit dolor et adipiscing tempor ut sed eiusmod sit amet labore ad tempor dolore amet dolor labore incididunt et eiusmod enim do et amet do ipsum ut veniam dolor labore aliqua consectetur dolor sit incididunt elit do ad sit ut ad aliqua do magna lorem aliqua ipsum incididunt quis ut dolore tempor elit elit consectetur ad ad sit sit magna enim amet do tempor et dolore elit ut amet enim enim consectetur eiusmod consectetur minim lorem eiusmod amet sit dolore eiusmod enim elit sit quis quis magna sit quis minim et dolore enim veniam et do eiusmod enim dolor adipiscing ipsum ad incididunt sed dolore veniam eiusmod elit aliqua eiusmod sed quis quis veniam dolor labore amet ipsum minim sit labore veniam sed eiusmod sed labore lorem eiusmod quis eiusmod ad elit tempor quis tempor tempor veniam incididunt adipiscing lorem incididunt ad aliqua magna do incididunt enim lorem magna sit eiusmod lorem ut amet dolor amet ipsum minim magna do adipiscing ad tempor elit aliqua dolor elit minim ut eiusmod ad ad veniam ad do lorem dolor labore sed ad incididunt adipiscing consectetur quis labore lorem consectetur lorem consectetur aliqua minim ipsum dolor tempor ipsum minim elit elit et sed do aliqua tempor quis ad magna eiusmod ad veniam et enim dolore ipsum elit amet quis adipiscing elit sit amet sit ipsum enim magna dolor aliqua quis enim ut labore ad labore incididunt lorem quis enim ad dolore ad eiusmod veniam dolore adipiscing amet enim do magna do minim incididunt enim dolore ipsum labore aliqua aliqua elit incididunt sed magna minim do quis sit eiusmod aliqua veniam sed veniam dolore sed labore labore sit aliqua do eiusmod veniam dolor veniam ut eiusmod quis lorem sit veniam eiusmod aliqua sit magna adipiscing aliqua minim minim lorem lorem lorem do quis labore sit dolore minim lorem et quis elit sed dolore do sed adipiscing quis incididunt enim labore et do eiusmod lorem aliqua enim enim do sit eiusmod veniam dolor amet incididunt labore sed tempor sed enim minim quis elit veniam consectetur magna labore labore sed et sit dolor minim consectetur magna ipsum sed tempor eiusmod tempor ipsum incididunt minim eiusmod consectetur do et ipsum magna dolore do eiusmod sed enim ad eiusmod dolor tempor lorem labore enim consectetur tempor enim do veniam sed et lorem labore consectetur elit tempor elit sit ad ipsum incididunt labore incididunt ad enim ipsum eiusmod lorem minim lorem labore sed adipiscing veniam consectetur labore sit veniam quis incididunt magna ipsum sed sit amet ipsum lorem sed ut dolore minim aliqua do dolor eiusmod et aliqua eiusmod sed eiusmod elit ipsum quis dolore elit labore dolore veniam lorem minim incididunt eiusmod ipsum dolore adipiscing veniam quis sit magna elit lorem quis magna eiusmod amet veniam elit aliqua et magna et ut consectetur labore sed consectetur ipsum tempor minim do quis eiusmod ad sed elit incididunt lorem ipsum minim sit amet enim eiusmod aliqua consectetur lorem ipsum magna consectetur aliqua amet dolor amet amet magna consectetur ut incididunt lorem tempor do tempor aliqua lorem elit aliqua lorem tempor tempor eiusmod dolor aliqua minim minim tempor tempor dolor ut adipiscing tempor aliqua sed amet adipiscing dolore consectetur enim amet dolore sit eiusmod aliqua aliqua magna ad enim sit enim consectetur tempor sed labore sed labore consectetur incididunt ut tempor sit eiusmod amet sed consectetur eiusmod labore dolor tempor tempor sed minim tempor amet ut adipiscing adipiscing consectetur aliqua labore enim ut minim aliqua veniam elit lorem tempor amet lorem labore ut aliqua sit ipsum incididunt do enim veniam adipiscing ut lorem veniam enim elit incididunt enim et et enim ut incididunt incididunt ipsum do consectetur consectetur eiusmod sit dolore enim labore consectetur aliqua incididunt elit veniam amet sed lorem labore eiusmod et tempor dolore elit do aliqua lorem et veniam amet sed adipiscing tempor lorem do enim ut quis elit lorem elit enim incididunt amet sed quis ad aliqua magna adipiscing amet incididunt aliqua minim quis et tempor minim et enim veniam amet lorem quis enim incididunt minim ut tempor do amet do et sed incididunt veniam enim elit labore adipiscing veniam consectetur et labore ipsum quis quis ut adipiscing dolore do veniam ut sit do sed consectetur amet adipiscing do amet consectetur tempor adipiscing aliqua aliqua consectetur ipsum sit amet sed aliqua do do consectetur ipsum sed aliqua tempor sit ut sed eiusmod veniam enim magna dolore ad sed sit magna magna ipsum lorem eiusmod quis ut enim et eiusmod minim aliqua elit sit ut ut minim dolor aliqua sed magna tempor quis amet ipsum lorem amet incididunt tempor lorem tempor eiusmod dolore aliqua aliqua incididunt magna veniam sit sit enim magna adipiscing adipiscing eiusmod ad magna incididunt amet ipsum dolor ipsum ad tempor adipiscing ut aliqua veniam ad sit ut adipiscing eiusmod dolor sit quis amet sed elit minim quis consectetur adipiscing incididunt ut adipiscing adipiscing lorem dolore sed ut veniam adipiscing dolore sit enim tempor ad eiusmod sed dolore elit et aliqua adipiscing aliqua ipsum aliqua minim sit sed ut ad sit ipsum sit eiusmod eiusmod sit enim do sit elit do aliqua dolor sit incididunt et amet eiusmod elit amet elit amet ipsum veniam aliqua ipsum labore et magna magna elit ut elit adipiscing enim sit eiusmod consectetur veniam do sit labore labore adipiscing ad et sed et adipiscing labore minim consectetur enim adipiscing aliqua incididunt tempor ut sed incididunt consectetur amet quis enim magna et sit elit enim tempor minim aliqua sed lorem do labore adipiscing incididunt elit do aliqua eiusmod lorem tempor ipsum ut quis lorem aliqua ad magna lorem quis ipsum ipsum ut enim do labore eiusmod ad lorem magna incididunt sit ut sed quis ipsum eiusmod ipsum ipsum tempor ad minim minim sit enim veniam quis lorem ad dolore ad sit aliqua magna ipsum adipiscing aliqua aliqua ut sed sit sit consectetur sit amet magna elit elit et enim sed incididunt adipiscing ipsum dolore dolore et dolor elit amet incididunt veniam minim labore dolor elit elit sit amet ut minim et ut sed enim adipiscing magna ipsum magna veniam minim incididunt amet veniam et quis aliqua amet ipsum do incididunt aliqua et minim labore enim quis ut eiusmod ad magna elit dolor do magna dolore elit eiusmod quis do elit dolore ut sed do incididunt eiusmod quis eiusmod adipiscing sit do do adipiscing sit elit ut magna consectetur eiusmod adipiscing ut labore elit sed dolore tempor incididunt dolor aliqua et labore minim sit consectetur ad sit dolore sed sed lorem incididunt incididunt adipiscing lorem enim consectetur ipsum ut ut eiusmod ut quis incididunt ipsum sed et minim aliqua aliqua lorem dolor ipsum amet enim magna elit consectetur quis eiusmod ut amet dolore do et elit ad dolor sed amet enim adipiscing elit amet lorem veniam lorem labore sed do quis dolore lorem incididunt labore enim ad consectetur ipsum consectetur ad lorem dolore incididunt tempor dolor eiusmod lorem minim labore quis dolore elit sed incididunt ipsum ut labore et consectetur adipiscing ad sit magna dolore sit dolore adipiscing ad veniam quis magna incididunt eiusmod sed elit quis enim ipsum eiusmod minim sed dolor veniam enim aliqua consectetur sit labore lorem et tempor sed ut consectetur ipsum veniam dolore consectetur consectetur ut quis veniam veniam quis adipiscing do enim magna aliqua ipsum ipsum elit lorem aliqua minim tempor sed labore quis enim do sed dolore eiusmod tempor amet dolor ad labore magna sit quis sit amet consectetur eiusmod ut adipiscing magna sed quis tempor ad tempor sed dolore quis lorem dolore dolor minim ut minim sit aliqua veniam ut labore dolor quis sit quis labore ad amet incididunt quis consectetur ut magna ipsum tempor ut sed do amet dolor ad aliqua labore magna eiusmod enim adipiscing magna minim minim minim ipsum elit labore quis quis ad dolore ut amet magna magna ipsum quis adipiscing do magna adipiscing sed ipsum dolore tempor veniam quis do incididunt amet ipsum adipiscing enim do adipiscing labore do lorem ipsum ut ut magna veniam lorem veniam consectetur minim sit tempor ut elit sed incididunt dolore tempor aliqua amet quis dolor eiusmod adipiscing minim ipsum incididunt incididunt sed eiusmod adipiscing dolore veniam elit aliqua aliqua aliqua ipsum et elit adipiscing sed consectetur consectetur labore elit ipsum dolor elit incididunt incididunt ut ad lorem amet enim do veniam do ad ipsum aliqua adipiscing do et incididunt sit enim minim veniam tempor dolor minim labore tempor elit enim aliqua ipsum sit tempor minim labore adipiscing do do quis sit lorem dolor minim enim ut adipiscing et adipiscing et magna minim quis incididunt dolore et consectetur amet lorem dolore labore dolor ut ad veniam eiusmod quis consectetur dolor aliqua enim eiusmod labore enim adipiscing veniam et minim ut ipsum consectetur sed do magna do minim ut aliqua lorem ad et dolor et do quis labore minim magna tempor et enim dolor ad do adipiscing ut labore tempor labore ad amet veniam et tempor dolor sed ad aliqua adipiscing aliqua amet minim dolore do ad dolor consectetur quis lorem magna aliqua quis minim veniam magna labore magna et sed consectetur lorem ad ut tempor do ut ipsum veniam ipsum ipsum elit ut ad dolor ipsum incididunt et enim adipiscing sit sed lorem consectetur et enim ipsum dolor lorem sed amet labore consectetur adipiscing elit ad elit magna minim lorem ipsum labore et eiusmod amet ad dolor ipsum minim enim ipsum ipsum ad elit incididunt sed lorem et sed sit magna amet dolor tempor quis eiusmod ad dolore ad adipiscing magna minim enim enim adipiscing magna dolore do dolore elit consectetur consectetur tempor dolor sit do ut ut dolore sed dolore amet sit tempor incididunt sit consectetur ipsum ut magna lorem aliqua sed minim quis aliqua quis eiusmod elit lorem ipsum ipsum lorem elit eiusmod sit ad quis ad veniam dolor do sed labore ipsum sit ut elit sed sed aliqua enim quis minim quis sit amet magna sit tempor tempor sed tempor ipsum dolor ut do tempor sed lorem amet et eiusmod elit adipiscing dolor ut tempor magna minim dolor consectetur consectetur ad enim elit magna dolor do veniam dolore incididunt ipsum ad tempor adipiscing labore aliqua aliqua adipiscing labore dolore sit labore adipiscing tempor enim eiusmod eiusmod veniam veniam enim enim lorem ut ad quis sit eiusmod incididunt ut tempor elit amet tempor incididunt incididunt veniam magna do veniam veniam minim adipiscing adipiscing dolor incididunt adipiscing consectetur adipiscing ipsum eiusmod incididunt ad dolor sit enim magna enim ipsum tempor amet consectetur tempor labore ut eiusmod do do eiusmod ut quis consectetur labore minim amet minim labore amet ut magna ad tempor eiusmod sit lorem veniam amet veniam tempor eiusmod minim adipiscing sit veniam do tempor enim ipsum sit veniam labore ad magna ad enim elit lorem labore et dolor elit dolor adipiscing adipiscing ad enim adipiscing tempor tempor lorem do veniam sed minim lorem eiusmod enim ipsum tempor consectetur ut dolor quis minim ut magna veniam amet minim labore et magna minim ad eiusmod lorem tempor lorem dolor labore et veniam lorem elit dolore ipsum minim sit incididunt sit ut amet et aliqua aliqua do aliqua incididunt lorem magna ipsum aliqua adipiscing consectetur enim consectetur labore consectetur labore adipiscing lorem dolore ipsum dolore et et incididunt sed ad labore eiusmod enim enim magna ad ad sit consectetur aliqua minim ad quis ad ipsum labore et et sed ut elit ipsum ipsum quis quis eiusmod incididunt et elit enim aliqua lorem magna ad adipiscing aliqua eiusmod adipiscing veniam amet do magna magna veniam consectetur ut enim sit veniam sed incididunt adipiscing dolor enim tempor incididunt amet minim ut magna et sit sed ut tempor tempor incididunt tempor ut minim sed lorem adipiscing labore tempor adipiscing dolor eiusmod lorem incididunt ut adipiscing aliqua ut elit ad ad quis veniam ad minim ut eiusmod sit lorem adipiscing ad do dolor adipiscing do dolor adipiscing consectetur elit aliqua veniam ut ad eiusmod do consectetur elit ad dolore et lorem amet et ad enim dolor lorem do labore quis sed adipiscing dolore magna amet ad minim eiusmod do amet consectetur eiusmod incididunt veniam do tempor labore et ut consectetur enim labore adipiscing sed ut ad ut et ipsum eiusmod et eiusmod ad ipsum eiusmod et tempor ad ad labore dolor dolor quis magna do ad do enim ut eiusmod ad ad labore sit tempor minim labore ad ut magna ipsum incididunt magna tempor ad lorem tempor lorem aliqua et dolore enim ut elit minim veniam ipsum do dolore magna adipiscing enim magna adipiscing minim sit dolore incididunt sit magna ipsum sed eiusmod eiusmod ipsum labore minim quis amet dolore aliqua lorem enim ad enim enim ut sed et sed enim quis magna minim elit do consectetur elit adipiscing et quis elit elit sed tempor adipiscing aliqua veniam eiusmod amet dolor incididunt adipiscing adipiscing enim enim adipiscing incididunt consectetur et veniam do enim dolor quis sit veniam do adipiscing lorem ipsum tempor dolor labore do do et incididunt sed elit minim magna tempor dolore enim sit ad eiusmod incididunt tempor sit magna amet enim et enim aliqua incididunt dolore tempor sed magna amet minim amet dolor amet dolor sed ipsum veniam ad elit ad dolore tempor quis incididunt magna tempor quis sit veniam quis dolore veniam aliqua quis et lorem adipiscing magna ut lorem ut minim ut aliqua magna magna elit sit dolor eiusmod ut magna do ad adipiscing et labore consectetur ad ad minim aliqua aliqua do dolore magna et tempor sit quis lorem eiusmod tempor labore amet magna sed ipsum dolor labore consectetur et ad do ad dolore amet consectetur aliqua dolor minim elit dolore minim veniam consectetur labore ipsum sit dolore tempor eiusmod ut tempor elit sit ut quis et quis enim aliqua dolore dolor aliqua enim veniam aliqua labore do lorem eiusmod magna et sit minim minim magna quis do sit enim tempor ipsum aliqua consectetur do minim consectetur eiusmod et elit incididunt eiusmod ad veniam magna lorem tempor eiusmod aliqua labore adipiscing labore ut aliqua quis aliqua eiusmod ut ut et adipiscing consectetur et ipsum adipiscing ipsum et lorem consectetur sit consectetur dolore adipiscing minim quis sit minim sit ut adipiscing dolore lorem consectetur incididunt consectetur sit ut eiusmod dolore elit minim labore ipsum sit sit adipiscing dolore sit sit magna veniam dolor enim dolor lorem consectetur sed labore aliqua dolore amet do minim do aliqua aliqua enim elit eiusmod sed amet quis ad et consectetur eiusmod minim adipiscing elit amet quis enim incididunt et incididunt ipsum aliqua consectetur enim lorem do sit ipsum ut ipsum amet adipiscing do dolor minim adipiscing lorem aliqua labore incididunt ut minim et adipiscing sed veniam minim dolor veniam tempor et ut eiusmod enim lorem minim ipsum sit aliqua enim et labore adipiscing veniam labore dolore lorem sed elit ipsum elit eiusmod quis dolore sed ipsum incididunt tempor veniam adipiscing elit adipiscing eiusmod lorem enim veniam sit ipsum dolor elit tempor enim quis sit magna incididunt sed ad adipiscing consectetur labore labore labore elit aliqua dolor dolore magna consectetur veniam sit eiusmod aliqua consectetur dolor veniam sed veniam ut do adipiscing amet sit minim elit ad adipiscing do sed ipsum sit amet magna et sed veniam amet dolor sit adipiscing enim dolore elit elit amet labore adipiscing et enim et sit tempor ut magna consectetur lorem ut ad aliqua enim amet aliqua veniam amet veniam incididunt tempor quis dolore magna adipiscing veniam adipiscing tempor elit enim et ad ipsum ut enim minim labore ipsum magna eiusmod adipiscing eiusmod consectetur lorem eiusmod et dolor magna enim incididunt do aliqua magna consectetur elit minim lorem et sit veniam veniam adipiscing labore do aliqua do do lorem veniam enim tempor sed minim dolor lorem enim lorem labore ipsum amet dolor ipsum sed do quis dolor dolor dolor elit aliqua enim ipsum ad eiusmod eiusmod enim enim labore lorem magna minim enim quis et quis incididunt do enim lorem elit tempor tempor eiusmod consectetur ut adipiscing incididunt ipsum lorem sed enim magna adipiscing do amet enim lorem quis incididunt sed elit lorem ipsum enim consectetur quis labore enim amet elit enim quis eiusmod do ad incididunt dolore veniam dolor dolore et elit sed sed ipsum tempor aliqua do aliqua sed elit veniam do eiusmod dolore do sed incididunt et sit veniam dolor ut adipiscing ad consectetur minim minim adipiscing sed tempor ad dolore labore ad labore veniam enim ut adipiscing magna labore eiusmod sit quis adipiscing enim et do aliqua sit lorem ipsum dolor minim veniam veniam elit magna ut ad eiusmod dolore sit dolor et do quis et labore ipsum do magna ad magna et tempor enim et amet minim magna elit ad ipsum labore aliqua elit elit aliqua elit minim ipsum do sit incididunt eiusmod ut tempor amet aliqua incididunt labore amet ut consectetur elit ipsum tempor labore minim labore sit minim eiusmod sed enim amet quis veniam do ipsum adipiscing dolore do sed elit dolor labore sit magna et quis eiusmod consectetur do dolore sit lorem adipiscing elit tempor magna consectetur tempor eiusmod elit do adipiscing ad dolore ipsum minim incididunt sed do incididunt sed dolor ad ipsum lorem aliqua amet elit magna lorem magna quis amet aliqua enim do minim consectetur consectetur aliqua et ipsum ad et enim do magna elit minim veniam minim tempor sit adipiscing elit amet amet sed dolore et do labore do adipiscing eiusmod adipiscing amet labore et consectetur dolore elit ut incididunt sed adipiscing tempor ad do labore minim amet veniam ut incididunt labore ut quis adipiscing enim ipsum sed aliqua dolore quis ipsum minim et aliqua ipsum ut enim enim consectetur lorem lorem enim sed consectetur dolor ipsum sed eiusmod lorem veniam do incididunt sit incididunt ut quis eiusmod adipiscing veniam elit consectetur enim labore incididunt eiusmod labore magna quis amet aliqua sit quis do labore enim sit amet do magna elit ad dolore eiusmod minim ut aliqua eiusmod dolor tempor et lorem sit amet amet minim sed sed amet elit consectetur tempor eiusmod minim dolore elit elit aliqua ipsum dolore minim adipiscing magna et tempor consectetur incididunt et et adipiscing labore magna incididunt veniam do elit sit dolore quis labore ut enim quis sed eiusmod dolor sit amet ad elit et magna consectetur ipsum labore consectetur dolore quis et amet lorem elit consectetur adipiscing enim ipsum eiusmod tempor et do eiusmod aliqua eiusmod eiusmod dolor do ad incididunt dolor veniam et enim ad eiusmod aliqua tempor ut sit quis sit minim lorem amet magna aliqua tempor elit lorem dolor magna sit minim do ut incididunt do eiusmod et ipsum elit sed veniam eiusmod ut eiusmod ipsum dolor dolor ipsum eiusmod aliqua amet quis ipsum do aliqua sed enim quis elit ad dolore eiusmod magna adipiscing adipiscing amet enim labore incididunt veniam amet dolor sed quis quis quis incididunt eiusmod minim eiusmod do ad ut eiusmod ad adipiscing ad lorem ad ut amet eiusmod dolor amet elit dolore amet et do aliqua ipsum labore veniam labore magna amet ad dolore incididunt dolor enim enim elit labore ad ut magna eiusmod magna elit ipsum lorem sed enim incididunt veniam amet elit elit ad enim ut enim dolore consectetur amet minim ipsum quis elit sit aliqua lorem elit aliqua sit dolor ad ipsum adipiscing elit amet adipiscing eiusmod ut dolore ut dolor tempor aliqua dolor eiusmod ad incididunt sit dolore eiusmod ipsum dolor ipsum do do veniam ipsum consectetur sit tempor tempor do consectetur labore minim ut adipiscing ad quis elit dolor sit ut consectetur et aliqua dolore labore magna veniam dolore consectetur lorem veniam sit lorem et amet adipiscing eiusmod aliqua ipsum dolor dolore dolore sed amet sed eiusmod ut sit quis incididunt veniam lorem quis do magna veniam do ut enim eiusmod et veniam do ipsum lorem et quis amet sit minim ipsum adipiscing sed elit sit ad dolore elit veniam adipiscing dolore do ut do labore quis et enim quis aliqua ut adipiscing adipiscing minim lorem veniam adipiscing tempor quis consectetur do adipiscing ipsum magna ut ut magna eiusmod adipiscing amet do dolor amet ipsum eiusmod ipsum amet ad dolor dolore sit sit quis ipsum sit amet incididunt enim do lorem labore sit adipiscing enim quis enim dolore magna lorem elit consectetur lorem aliqua labore sed do enim enim consectetur dolore enim incididunt eiusmod sit consectetur sit magna amet et amet lorem enim ipsum do adipiscing sit sed sed labore et elit sed minim enim magna ut labore do labore elit enim magna ut sit ut incididunt adipiscing sit quis tempor ad ad ad aliqua elit minim quis lorem dolore minim dolor lorem veniam ut tempor adipiscing veniam adipiscing eiusmod ut sit ipsum quis et ut do sed incididunt enim sed incididunt aliqua amet ad amet lorem dolor consectetur ipsum eiusmod enim tempor dolore labore ut adipiscing incididunt ipsum tempor minim dolore incididunt minim minim labore aliqua enim incididunt aliqua sed enim aliqua sit ad consectetur do consectetur ad minim magna aliqua quis magna incididunt sed amet ad aliqua dolor do ut do ut enim lorem et adipiscing dolore do adipiscing lorem enim minim dolore do sit ut quis tempor quis incididunt tempor quis enim quis incididunt adipiscing eiusmod lorem ipsum sit enim dolore veniam ad ad ut sit ut eiusmod ad elit eiusmod eiusmod veniam tempor tempor amet eiusmod eiusmod dolor sed eiusmod quis ad sed ad do amet ut aliqua incididunt elit enim sit elit sed labore ad amet do amet sed ipsum lorem elit tempor eiusmod minim ad sit veniam labore ad ipsum sed amet et adipiscing elit eiusmod dolor sit sed et labore do ut quis tempor aliqua magna consectetur dolor sit amet incididunt sed dolor magna aliqua veniam do dolore consectetur veniam et ut elit tempor sed sed eiusmod consectetur sed adipiscing sit do incididunt incididunt eiusmod labore quis sit minim dolor magna elit consectetur labore adipiscing aliqua tempor sed ipsum amet lorem veniam tempor labore do minim aliqua do veniam ad eiusmod consectetur minim lorem aliqua do consectetur lorem labore labore sit ut minim veniam ad et dolor ipsum labore minim veniam ut elit sed amet aliqua et labore dolor ad tempor veniam ut magna do et et labore eiusmod aliqua do dolor sit sit dolor elit veniam ut labore dolor aliqua labore ad lorem magna consectetur amet minim ut magna minim eiusmod consectetur veniam eiusmod consectetur elit et incididunt consectetur sit tempor consectetur labore sit veniam amet eiusmod ipsum veniam sit do do sed consectetur aliqua minim adipiscing incididunt et veniam magna lorem sed ad incididunt aliqua sit do tempor magna magna incididunt do do dolor labore veniam elit do et lorem elit lorem sit lorem eiusmod ut amet do ipsum sit sit do labore do quis amet ut sed tempor sed ut ut aliqua dolore do aliqua et ut elit veniam ad consectetur minim labore do amet sit aliqua consectetur amet sed do sed sit eiusmod sit lorem adipiscing dolor aliqua veniam eiusmod dolore ipsum sed enim ad et ut sit consectetur sed minim veniam adipiscing elit enim eiusmod consectetur ipsum ipsum lorem adipiscing sit quis minim ipsum et dolor do adipiscing dolore tempor veniam ad ut consectetur elit veniam dolore veniam magna aliqua consectetur labore incididunt adipiscing ipsum dolore enim enim ut aliqua veniam et amet ut enim dolore dolore minim dolor consectetur aliqua consectetur eiusmod enim consectetur enim ut tempor ad enim ut minim ut do adipiscing incididunt ut aliqua ut dolore veniam elit quis labore dolor lorem minim minim aliqua et dolore ipsum eiusmod eiusmod eiusmod et ad veniam aliqua do aliqua dolore dolor ipsum dolor veniam do consectetur sed quis elit minim dolor sit aliqua adipiscing ipsum do ut sed dolor dolore enim ipsum lorem quis amet minim elit dolor ut amet veniam aliqua quis enim tempor dolor adipiscing elit et aliqua dolor dolore elit dolore labore consectetur labore enim tempor elit amet sit do et dolore minim veniam ipsum incididunt adipiscing ut labore aliqua amet incididunt ipsum magna lorem dolore elit magna aliqua tempor lorem amet dolor dolore sit elit veniam veniam consectetur incididunt tempor adipiscing dolor ut adipiscing veniam amet veniam elit et enim ut do tempor consectetur quis quis sed do veniam incididunt tempor elit elit veniam adipiscing ut minim ad amet incididunt do adipiscing consectetur ipsum eiusmod minim eiusmod minim enim minim ipsum do adipiscing et amet ad ipsum veniam consectetur magna adipiscing tempor labore ut elit dolore amet consectetur minim et dolore enim amet veniam lorem enim ipsum incididunt amet ipsum adipiscing minim aliqua incididunt adipiscing et lorem incididunt dolore magna sit sed labore ipsum eiusmod et ipsum et amet veniam amet veniam enim tempor labore eiusmod minim ipsum dolor adipiscing aliqua minim quis quis lorem dolor ut sed adipiscing sit eiusmod elit eiusmod et magna incididunt magna adipiscing elit incididunt dolor et et lorem labore adipiscing lorem adipiscing elit do enim dolore consectetur dolor et enim incididunt incididunt consectetur lorem ipsum dolor sit et enim sit dolor adipiscing incididunt eiusmod sed ipsum enim ad dolore aliqua minim et dolore magna quis minim eiusmod ad sit minim eiusmod sed lorem sed ipsum minim ipsum incididunt tempor lorem et aliqua tempor consectetur elit quis consectetur dolore enim enim consectetur sed minim labore labore amet adipiscing sed consectetur veniam sit minim elit amet dolor sed labore et tempor ad lorem amet adipiscing tempor veniam elit ipsum lorem sed consectetur aliqua ipsum labore sed tempor aliqua sit ad ad incididunt sed eiusmod incididunt eiusmod incididunt tempor eiusmod adipiscing dolore do minim veniam et quis adipiscing ipsum ipsum sed tempor consectetur enim quis aliqua et do incididunt elit lorem amet magna sed consectetur incididunt do enim minim sed lorem quis tempor sit aliqua tempor minim eiusmod dolore veniam eiusmod veniam enim elit quis quis dolore elit sed lorem dolore consectetur eiusmod veniam dolore quis adipiscing enim sed tempor elit incididunt tempor magna incididunt eiusmod eiusmod minim ut incididunt do elit ad magna veniam sit sed dolor eiusmod labore amet magna et magna dolor sed adipiscing consectetur sit sed eiusmod magna lorem eiusmod dolore lorem amet tempor et sit enim elit incididunt minim elit sed ipsum lorem enim tempor magna aliqua minim sed eiusmod lorem dolore enim lorem aliqua incididunt elit labore minim minim et do amet incididunt lorem adipiscing elit ut minim dolor dolor minim do dolor eiusmod veniam amet ut sit sit adipiscing sit consectetur adipiscing adipiscing magna tempor dolor dolore elit minim minim sed veniam adipiscing dolore adipiscing sed dolor adipiscing enim veniam veniam et elit dolor aliqua tempor lorem aliqua do minim consectetur adipiscing lorem magna ad veniam labore quis veniam enim ad dolore tempor enim elit incididunt quis amet enim tempor dolore dolore consectetur et dolor adipiscing et tempor et quis enim ut sed consectetur quis lorem ad et sed ipsum dolor enim ad quis ipsum tempor magna et ut aliqua ad et sit adipiscing minim ipsum tempor incididunt sed tempor dolore enim eiusmod amet eiusmod amet sit quis tempor dolor sed tempor incididunt sit eiusmod aliqua amet incididunt dolore do sed tempor consectetur aliqua ipsum sit adipiscing et dolor eiusmod elit sed labore consectetur ut ad adipiscing magna dolore magna labore consectetur et veniam amet et et magna quis tempor minim enim minim dolore veniam ipsum adipiscing consectetur dolor veniam eiusmod incididunt et incididunt enim dolore incididunt ipsum aliqua eiusmod amet consectetur sed ad veniam consectetur aliqua consectetur labore sit enim quis tempor enim enim elit magna ad enim incididunt eiusmod amet aliqua dolore enim labore dolore sit incididunt incididunt do eiusmod labore aliqua minim ad eiusmod incididunt labore sit amet et et labore consectetur adipiscing quis consectetur veniam sit ad veniam ad tempor amet aliqua dolore consectetur aliqua ad magna labore elit elit magna ut et ad consectetur ipsum enim dolore sed do dolore sit lorem et sed ut ipsum veniam sed et ipsum magna et et aliqua dolore eiusmod ad minim et ut magna sed enim ipsum dolor dolor elit adipiscing adipiscing sed ad ad eiusmod ad magna consectetur magna incididunt elit elit dolore dolor lorem tempor lorem consectetur do incididunt et ipsum amet minim veniam tempor enim sit do amet incididunt minim veniam ut minim et et eiusmod quis aliqua consectetur elit tempor eiusmod enim consectetur aliqua ut quis incididunt eiusmod eiusmod quis magna ut amet labore dolor ad veniam do incididunt adipiscing lorem veniam dolore labore elit do lorem minim lorem do enim lorem ipsum enim elit dolore eiusmod tempor tempor dolor dolore aliqua sed lorem amet ipsum dolor magna eiusmod dolor aliqua quis sed amet tempor ad aliqua ad elit enim lorem ut magna eiusmod ut quis incididunt minim incididunt labore minim sit dolore lorem labore sed minim sit tempor dolor et eiusmod dolore incididunt incididunt sit magna enim lorem consectetur ut ad incididunt tempor ipsum ipsum quis enim consectetur tempor consectetur labore do enim et ad veniam eiusmod do ad sit tempor consectetur et ipsum veniam labore ut sed tempor adipiscing lorem tempor elit enim dolore sit adipiscing incididunt do consectetur lorem ut enim tempor enim labore consectetur aliqua ut do adipiscing magna minim sed tempor ad enim eiusmod incididunt enim enim veniam veniam aliqua ut quis sit incididunt adipiscing ipsum veniam adipiscing et minim quis eiusmod dolore elit magna quis elit et veniam quis et elit sit amet amet quis sed eiusmod amet ipsum aliqua minim aliqua veniam dolore incididunt consectetur incididunt labore ipsum sed labore tempor ad do incididunt enim ut dolore dolor ut et quis do elit lorem ut do enim labore aliqua aliqua et incididunt minim eiusmod quis tempor enim amet tempor do aliqua labore amet dolore adipiscing do dolor labore amet sed sit consectetur tempor dolore veniam et aliqua ipsum magna enim consectetur minim veniam ut ad adipiscing sed lorem ad aliqua tempor lorem minim ad eiusmod eiusmod sit ad adipiscing dolor magna incididunt ipsum incididunt veniam aliqua do elit amet magna sit tempor magna adipiscing veniam adipiscing ad quis dolore ut ad consectetur ut ad lorem sed sed dolor lorem labore do sed tempor ipsum ut ipsum enim ipsum adipiscing enim lorem amet dolor adipiscing dolore ad incididunt ut veniam eiusmod sit sit ad elit eiusmod dolor ad consectetur consectetur magna elit veniam et dolore enim enim dolor ut aliqua dolore elit dolore veniam eiusmod sed dolore incididunt labore sit aliqua lorem ad ut dolor tempor do magna quis ipsum ipsum tempor consectetur sit elit tempor dolore ipsum eiusmod magna elit aliqua elit eiusmod dolor minim dolor ipsum sit eiusmod et incididunt ad veniam incididunt et eiusmod tempor ad ipsum amet labore dolor magna eiusmod amet ut eiusmod ipsum labore ut enim consectetur tempor labore labore elit tempor aliqua enim lorem minim aliqua elit magna dolor elit dolor enim consectetur ut labore elit minim do enim minim enim sit lorem et ad ipsum lorem amet veniam ad quis ipsum veniam et amet adipiscing ut amet ad ipsum adipiscing consectetur magna veniam lorem adipiscing dolor amet magna lorem adipiscing incididunt ad quis veniam magna minim quis quis et veniam enim lorem veniam labore dolore incididunt sed lorem labore veniam quis consectetur et adipiscing dolor adipiscing labore lorem eiusmod incididunt sit minim incididunt consectetur incididunt enim quis magna quis ut do consectetur ut magna sed ad dolore sit ut quis quis dolor tempor ad lorem magna et do quis incididunt aliqua labore ad et tempor et dolore dolore elit lorem tempor elit elit do sed ut ut sed do amet lorem elit amet enim et adipiscing magna magna minim et et ipsum enim minim magna et ipsum lorem veniam quis enim sit minim elit aliqua dolor magna elit amet ut aliqua minim ad quis do amet ad ad adipiscing sed enim aliqua labore do sed amet minim sed elit magna veniam ut adipiscing elit minim ipsum dolore amet sed aliqua tempor elit sed elit quis lorem dolore enim veniam dolor et sed magna adipiscing lorem tempor incididunt ut labore magna tempor ut veniam quis sed sed magna do minim eiusmod aliqua do lorem quis lorem tempor veniam sit incididunt tempor enim et et veniam tempor magna amet consectetur lorem incididunt aliqua elit sit amet adipiscing do sed labore enim quis sed elit do ipsum dolore amet elit enim tempor eiusmod minim aliqua adipiscing et sit ad et sed adipiscing incididunt ut ipsum ut ipsum incididunt dolor labore adipiscing ad minim eiusmod lorem amet labore dolore quis labore minim ut dolore ipsum amet amet elit et dolor ipsum dolor magna lorem lorem magna eiusmod minim dolor tempor ipsum adipiscing et magna dolor aliqua adipiscing quis do do lorem elit et ad consectetur elit tempor ad ipsum ipsum ad labore eiusmod magna incididunt ipsum ipsum dolore lorem consectetur magna dolor elit et minim magna magna dolore elit ipsum elit labore lorem amet ipsum consectetur labore elit et sit minim veniam quis eiusmod minim magna ut ad aliqua labore aliqua elit elit dolor sed quis incididunt et incididunt minim consectetur do veniam do enim sed lorem adipiscing elit elit labore lorem dolor incididunt eiusmod dolor et lorem veniam tempor labore minim et ipsum sed aliqua minim sed adipiscing consectetur enim magna sed sed dolore dolor sit ut incididunt dolore dolor tempor ad enim sed sed ut labore labore ut ut veniam ut sed sit amet magna dolore magna enim aliqua ut veniam lorem eiusmod consectetur incididunt quis veniam dolore veniam do ipsum veniam eiusmod minim quis lorem lorem lorem elit tempor adipiscing dolor adipiscing ipsum adipiscing ad enim aliqua consectetur tempor magna magna sed ut sit veniam et veniam amet ut enim lorem tempor consectetur aliqua et eiusmod lorem lorem sed adipiscing dolore aliqua magna consectetur veniam incididunt enim et aliqua et adipiscing elit tempor et aliqua tempor magna ut dolore do ut sed do et enim sit quis veniam ipsum lorem sed incididunt dolore ad ad incididunt adipiscing ad veniam ipsum incididunt aliqua dolore minim lorem minim minim eiusmod et elit tempor magna eiusmod ad aliqua enim ipsum veniam ipsum elit labore ut et quis sed do enim dolor lorem sed dolor sit elit magna quis aliqua eiusmod elit consectetur et ut sit tempor ipsum dolor amet sit sit incididunt et incididunt consectetur eiusmod et lorem magna veniam adipiscing do eiusmod minim aliqua ad elit consectetur incididunt dolore aliqua do sed sit incididunt dolore ad tempor dolor quis sed tempor adipiscing aliqua minim ut minim tempor dolor lorem ad et veniam quis lorem elit dolor ad do minim sit aliqua sed ut ipsum sed sit veniam veniam aliqua incididunt adipiscing magna tempor adipiscing amet labore dolor magna incididunt quis sit dolore consectetur dolore sit labore adipiscing consectetur quis quis eiusmod quis consectetur enim lorem veniam veniam do ut ut ipsum adipiscing magna dolor et consectetur quis elit labore veniam quis labore lorem incididunt sit sed adipiscing aliqua incididunt adipiscing dolor sit sit incididunt quis quis quis adipiscing do lorem sit tempor dolore elit aliqua lorem incididunt ipsum amet sit amet veniam dolor labore tempor adipiscing enim consectetur veniam veniam minim sed dolor ad minim ut eiusmod veniam do ad eiusmod aliqua magna sit minim elit adipiscing incididunt ut amet enim dolor quis elit elit do veniam quis ut do ut ipsum ad ipsum incididunt incididunt sed lorem incididunt sit lorem sit aliqua lorem do ipsum lorem incididunt magna enim magna amet et sit dolor aliqua ipsum ut adipiscing elit elit lorem dolor sit minim quis adipiscing ad adipiscing labore tempor magna eiusmod veniam tempor quis minim veniam enim lorem ut incididunt incididunt aliqua et ut eiusmod amet lorem adipiscing adipiscing incididunt incididunt amet quis et sed minim ad magna magna ipsum ut eiusmod adipiscing lorem ut eiusmod consectetur sit ad elit quis eiusmod incididunt minim eiusmod eiusmod consectetur adipiscing aliqua elit incididunt sit incididunt aliqua magna ad enim elit quis do lorem consectetur aliqua do tempor eiusmod dolore eiusmod aliqua ut quis labore consectetur tempor dolor elit tempor minim tempor adipiscing ipsum eiusmod dolore veniam ad sit tempor sed aliqua ad lorem dolor sed amet labore sed do dolor magna dolor enim amet aliqua sit dolor magna elit enim ipsum amet magna eiusmod veniam sit dolor amet aliqua consectetur tempor amet sed dolore et ad veniam sed elit veniam magna lorem do adipiscing tempor adipiscing incididunt et eiusmod sed lorem enim amet sed elit do do amet elit eiusmod amet magna dolor sit et eiusmod enim dolor elit minim minim do consectetur ipsum sed lorem lorem dolor do sed minim eiusmod ut ut amet ad minim enim veniam dolore ad sit enim ut labore tempor do aliqua et sit labore veniam ut sit ut sit do labore sed consectetur incididunt enim amet aliqua ut adipiscing amet consectetur dolore dolor elit adipiscing tempor do do labore et dolore enim ipsum eiusmod sed do ut veniam magna et aliqua lorem do ut lorem adipiscing quis veniam dolor elit do ad dolore ad elit lorem ut dolor aliqua dolore elit dolor sed minim sit aliqua incididunt minim veniam incididunt sit labore dolor lorem sed consectetur tempor et elit adipiscing do tempor adipiscing amet amet ad et adipiscing et consectetur amet eiusmod sed minim lorem amet sed ad dolor ad labore dolore magna elit ut quis elit labore ad magna ipsum minim tempor sit ut sed aliqua aliqua dolor ad et tempor adipiscing dolor consectetur dolor lorem ut sit eiusmod aliqua eiusmod sit minim consectetur eiusmod do dolore sit magna amet eiusmod ut ad consectetur veniam amet ipsum eiusmod do adipiscing elit do labore lorem dolore sed magna sit enim ad lorem adipiscing consectetur veniam aliqua ut lorem amet labore incididunt labore dolore labore eiusmod quis dolore minim quis minim labore et dolor sed dolor ut ad adipiscing quis ad dolor dolor tempor dolor incididunt veniam incididunt veniam ut adipiscing eiusmod et ipsum eiusmod dolore amet do do aliqua quis veniam lorem amet ipsum consectetur magna sed eiusmod incididunt enim sed incididunt veniam elit adipiscing eiusmod et ad eiusmod consectetur lorem sed amet sed elit quis dolore aliqua minim incididunt minim sit ut labore labore ut minim ut dolor lorem adipiscing do lorem elit aliqua minim ad minim amet consectetur incididunt consectetur adipiscing ut quis enim dolore ipsum consectetur lorem ipsum enim veniam amet sed sed sed enim ut adipiscing dolor dolor lorem do adipiscing magna dolore minim lorem minim eiusmod veniam dolore dolor magna eiusmod elit enim magna dolor labore et et magna aliqua veniam ad aliqua labore aliqua aliqua labore elit aliqua sit magna magna ipsum enim ad ipsum ipsum ut enim ut et adipiscing lorem eiusmod lorem ut minim aliqua adipiscing et aliqua dolor aliqua sit sed enim ut amet veniam do adipiscing veniam lorem quis quis consectetur ut magna elit veniam tempor minim ipsum minim amet incididunt labore incididunt sit incididunt do ipsum sed elit amet incididunt aliqua minim consectetur eiusmod elit dolore ipsum tempor incididunt ut lorem elit dolor tempor minim consectetur et quis magna sit veniam magna sit et dolore tempor et ad lorem consectetur dolore consectetur labore adipiscing veniam et amet ut do dolore ipsum labore consectetur adipiscing et dolor amet incididunt aliqua sit elit ut et do ad do dolore sit dolor quis labore ut veniam quis incididunt adipiscing amet tempor dolor lorem sed veniam veniam elit eiusmod dolor amet minim elit dolore ut ipsum do lorem dolor enim ut enim dolor incididunt dolore magna magna ipsum quis enim dolor sit adipiscing dolor veniam ut quis enim aliqua quis aliqua ipsum incididunt eiusmod aliqua veniam incididunt minim lorem et magna do tempor et veniam ipsum sed lorem do do elit consectetur minim do elit magna sit minim adipiscing ad elit labore ut sit lorem dolore amet consectetur sit consectetur incididunt magna magna minim adipiscing labore et adipiscing elit dolor do amet enim ipsum ut sed do dolore magna sed magna enim minim et incididunt veniam sed magna eiusmod incididunt minim quis veniam dolor quis aliqua sit et dolor amet eiusmod tempor dolor ut veniam sed quis incididunt consectetur do dolore do aliqua ipsum dolore do dolore amet et adipiscing ut ad minim ipsum veniam veniam adipiscing do aliqua sed dolor quis ad aliqua labore quis consectetur veniam enim sed dolor eiusmod ipsum consectetur consectetur ut eiusmod eiusmod lorem dolor amet ad minim tempor veniam tempor ut lorem tempor veniam dolor aliqua aliqua tempor ad enim eiusmod amet labore sit quis veniam quis minim enim magna labore amet quis labore incididunt et magna do quis quis minim ad et adipiscing ut eiusmod lorem et ut amet enim do eiusmod adipiscing sit adipiscing amet quis aliqua ut ipsum ad quis et minim eiusmod quis sed eiusmod ad et sit lorem lorem tempor ad sed ut minim aliqua elit incididunt elit aliqua veniam dolor aliqua sit ad sed aliqua lorem incididunt do veniam tempor incididunt consectetur tempor incididunt enim aliqua veniam sit quis magna ut eiusmod sed eiusmod sit enim consectetur labore elit adipiscing sit labore veniam consectetur lorem minim enim consectetur tempor quis ipsum quis do elit consectetur dolor magna quis eiusmod et enim dolor aliqua aliqua eiusmod lorem ipsum eiusmod sit sit ut et incididunt aliqua tempor sed enim aliqua aliqua ipsum elit ad sed adipiscing minim minim quis magna lorem quis ad dolore et minim lorem dolore lorem lorem dolor consectetur ad tempor labore et magna sit incididunt consectetur magna quis quis labore consectetur tempor labore consectetur dolor labore lorem labore incididunt enim dolor ipsum labore aliqua dolor sit aliqua amet minim labore consectetur quis aliqua ipsum sit ipsum ut tempor eiusmod sit tempor ut et enim sit ipsum adipiscing sit consectetur labore lorem et ut magna amet enim ad lorem quis amet veniam quis minim incididunt sit sit dolore magna elit veniam elit enim lorem sit minim dolor ipsum ad enim eiusmod adipiscing tempor sit eiusmod amet eiusmod elit aliqua ipsum minim dolore magna et sit aliqua amet minim minim aliqua eiusmod et amet dolore consectetur et ad tempor amet ipsum veniam consectetur veniam sed tempor eiusmod elit sed aliqua ut do ad eiusmod enim lorem quis minim et do amet minim sed adipiscing lorem et do quis et lorem sit tempor quis elit enim ad magna elit incididunt adipiscing ad incididunt sit consectetur sed sit do consectetur magna amet aliqua quis labore ut incididunt ut elit magna dolor quis dolor ut eiusmod do et ut ut tempor incididunt et incididunt elit ipsum sit do eiusmod dolore ipsum eiusmod do eiusmod et dolor enim aliqua amet ipsum enim veniam sit incididunt elit lorem do elit do sit quis incididunt ut sed dolor veniam lorem elit do magna lorem dolor veniam dolore aliqua sed amet elit adipiscing elit do dolor dolor elit labore dolor lorem et amet tempor ut incididunt ad eiusmod labore amet enim ad enim lorem adipiscing ut adipiscing ad dolor sed dolore sit enim do labore minim incididunt veniam adipiscing incididunt elit sit ut ipsum dolor eiusmod eiusmod do minim consectetur amet enim lorem elit ad veniam ad minim dolor do ipsum elit consectetur ut ad aliqua ut minim eiusmod ad sit et enim dolor minim incididunt ad et quis enim ipsum sit consectetur et veniam ut sed adipiscing veniam ut do consectetur dolor sit labore ad lorem do veniam consectetur ad amet incididunt amet elit veniam sit incididunt amet adipiscing consectetur et tempor aliqua enim sit lorem veniam incididunt lorem ut sed sed eiusmod veniam dolore magna magna dolor adipiscing tempor quis minim dolor tempor ipsum elit dolore ad minim incididunt dolor enim ad dolor dolore dolor enim do ipsum incididunt lorem aliqua dolore incididunt ut sit tempor adipiscing minim et consectetur ipsum dolor adipiscing magna amet magna sed aliqua labore elit consectetur incididunt et sit sed sed quis ad ut dolor ad ad magna dolore incididunt tempor enim sit sit amet ad do elit labore aliqua minim adipiscing adipiscing et adipiscing enim adipiscing adipiscing sed ut sed magna magna elit sed aliqua et enim enim amet minim minim et elit lorem ad eiusmod dolor ad et incididunt consectetur labore ipsum ipsum labore veniam elit elit enim magna veniam ad elit veniam incididunt lorem ad elit et adipiscing dolore magna veniam dolore veniam veniam incididunt ad quis tempor dolore et dolor ipsum et elit sit adipiscing dolore labore amet do minim sit aliqua quis veniam minim minim magna magna quis ipsum minim eiusmod ipsum sed amet ad aliqua dolor dolore eiusmod incididunt dolore labore aliqua enim aliqua do dolor sit et tempor quis adipiscing lorem sit dolore incididunt dolore aliqua adipiscing aliqua eiusmod ut labore eiusmod sed incididunt quis et quis dolor ut magna quis sed quis adipiscing adipiscing amet amet ad tempor ad et minim tempor labore sed do ipsum labore eiusmod tempor enim elit tempor lorem ad amet adipiscing dolor tempor ut consectetur quis adipiscing dolor incididunt aliqua eiusmod sed aliqua tempor consectetur incididunt minim quis lorem aliqua veniam quis amet sit ipsum et lorem elit dolore elit veniam labore quis eiusmod tempor ut magna tempor eiusmod aliqua elit dolor sed do minim amet lorem adipiscing eiusmod do minim labore amet consectetur sit quis dolore veniam minim aliqua dolore dolor amet incididunt ut incididunt ipsum labore minim aliqua do incididunt sit consectetur adipiscing magna tempor incididunt enim ad amet veniam dolore do tempor enim ut dolore ut sed eiusmod ad aliqua lorem ut ipsum do aliqua labore veniam ut veniam elit amet sit quis elit incididunt amet sed aliqua dolore veniam et quis adipiscing sed do adipiscing elit dolore tempor ad elit ipsum quis enim do do enim et magna do magna incididunt ut tempor sit lorem quis amet tempor sit minim dolor adipiscing sed quis ut enim et ipsum ad ut lorem quis tempor dolor labore veniam do adipiscing eiusmod labore incididunt et lorem sed tempor sit et eiusmod enim do enim adipiscing aliqua elit enim dolor adipiscing eiusmod sed lorem elit et ut ut eiusmod dolore quis amet et amet dolor ad dolore veniam aliqua sed veniam adipiscing quis tempor eiusmod do dolor ut magna dolor dolore ad ut elit sed consectetur amet elit consectetur sit elit do sed ipsum magna sit ut do enim enim eiusmod sit dolor incididunt quis dolor magna consectetur minim lorem minim et enim elit consectetur ut ut do enim incididunt ut et dolor amet do dolor sit labore labore veniam do labore ad aliqua eiusmod magna elit sed consectetur et lorem ut sit veniam ad consectetur elit minim lorem et et magna minim ad tempor tempor dolore ad aliqua quis aliqua minim labore eiusmod ad sed elit quis ipsum magna enim eiusmod dolor consectetur labore eiusmod amet enim magna ut labore quis minim dolor ad amet et amet dolor ut sed labore ipsum magna eiusmod dolore ut ipsum dolore do veniam aliqua minim quis sed aliqua quis quis minim lorem aliqua amet sed minim amet quis do labore labore enim et do lorem et sit amet consectetur sit ipsum elit ut ipsum adipiscing incididunt veniam adipiscing dolor dolore adipiscing do magna dolore enim magna adipiscing enim quis incididunt minim amet quis dolor ad sit enim ut aliqua elit do aliqua lorem sit labore minim aliqua et tempor enim ut ipsum ipsum adipiscing dolore sit magna ipsum amet tempor eiusmod adipiscing lorem veniam adipiscing sed amet sit amet tempor sit lorem lorem ad enim dolor sed quis sed ut aliqua et eiusmod amet tempor sit enim dolore adipiscing adipiscing magna ad consectetur amet aliqua minim do amet amet do quis enim ipsum adipiscing adipiscing veniam consectetur incididunt veniam ad sed do magna elit ipsum amet amet adipiscing minim incididunt lorem aliqua labore dolor ut sit veniam magna ad ipsum tempor ut elit sit sit magna labore amet amet magna ut labore quis sed magna veniam ipsum enim eiusmod incididunt enim eiusmod adipiscing tempor tempor magna ut et adipiscing labore ad tempor enim incididunt incididunt adipiscing amet incididunt elit elit adipiscing do lorem ipsum ut magna labore sit quis eiusmod ipsum ad do enim adipiscing adipiscing elit dolor ad sed ad magna lorem dolor ipsum amet do incididunt veniam sit ut elit labore ipsum consectetur dolore ad minim do enim aliqua ad consectetur sed elit lorem magna tempor labore lorem sit ipsum aliqua ad eiusmod sit dolore tempor et incididunt incididunt sed sed incididunt adipiscing aliqua lorem et amet minim incididunt sed ad do dolore sed veniam enim amet incididunt dolore quis dolor aliqua ipsum minim tempor et do elit sit labore magna consectetur minim enim magna amet consectetur tempor quis eiusmod lorem et et adipiscing dolor ad consectetur aliqua elit ut ad veniam amet et lorem ad incididunt consectetur adipiscing enim dolore dolor incididunt ipsum magna aliqua lorem consectetur dolore et do do magna magna et sit ad aliqua lorem lorem adipiscing enim tempor ipsum consectetur dolore aliqua minim labore enim dolor veniam dolore eiusmod eiusmod dolore veniam minim tempor eiusmod minim sed enim tempor labore magna adipiscing minim ut ipsum eiusmod tempor minim ut sit ad ad veniam quis adipiscing amet ad et magna quis labore do magna tempor magna consectetur ipsum elit veniam do veniam eiusmod dolor labore amet ut dolore dolor adipiscing dolor dolore consectetur dolor sed incididunt quis adipiscing dolore dolore dolor eiusmod sit minim lorem aliqua consectetur amet lorem amet amet consectetur do consectetur adipiscing aliqua enim ipsum enim consectetur eiusmod eiusmod ut dolore aliqua veniam quis dolore consectetur ad veniam aliqua eiusmod dolore do amet incididunt minim labore sit consectetur sed consectetur quis ad tempor labore do elit ipsum labore incididunt quis ad lorem do adipiscing dolore ad dolor enim elit tempor elit consectetur sed dolor eiusmod consectetur aliqua ad aliqua dolor sit dolor veniam ad magna dolor aliqua adipiscing incididunt elit magna tempor minim sit dolor enim dolore quis eiusmod magna eiusmod enim ut aliqua incididunt elit labore ipsum dolor quis dolor lorem labore enim minim tempor minim sit minim do sit minim dolor amet minim adipiscing elit magna ipsum ad et sit adipiscing adipiscing labore elit aliqua amet quis adipiscing veniam elit adipiscing minim consectetur adipiscing dolor elit consectetur labore sit minim quis magna do labore tempor amet consectetur consectetur sed sed ad enim do magna do elit amet consectetur lorem ipsum labore consectetur et minim ad ad amet sit eiusmod tempor ut adipiscing dolor sed adipiscing consectetur magna et dolor labore aliqua labore minim do ipsum incididunt tempor lorem sed elit adipiscing quis minim dolor elit amet incididunt ad labore sit dolor enim incididunt do aliqua magna sed ad aliqua quis ad enim magna et ut quis et quis aliqua ad consectetur veniam elit et ipsum sit veniam enim eiusmod quis consectetur sit ut dolor et amet incididunt consectetur ad ipsum quis dolore elit dolore consectetur dolor enim ut veniam quis minim consectetur consectetur eiusmod sed tempor ut elit do consectetur dolore aliqua enim ut dolor veniam elit consectetur adipiscing dolore eiusmod do lorem minim enim ut adipiscing do eiusmod quis enim et consectetur enim elit dolore lorem dolor ad magna enim magna veniam ut ipsum sed veniam quis eiusmod ipsum lorem et veniam minim incididunt ad quis incididunt elit ut veniam enim tempor amet ut do tempor tempor dolor tempor consectetur quis dolor ad tempor enim dolore elit minim incididunt ipsum sit veniam adipiscing amet incididunt consectetur adipiscing sit et veniam quis elit adipiscing incididunt consectetur lorem incididunt dolore ipsum enim ad lorem sit et quis consectetur aliqua aliqua veniam labore lorem consectetur sed et dolore sed dolore quis adipiscing tempor enim ut dolore dolor dolor quis aliqua dolore dolor ut ad tempor dolore dolore elit ipsum dolor labore incididunt et dolore do enim do ad consectetur amet minim quis ipsum quis amet veniam consectetur do ipsum eiusmod aliqua incididunt ut consectetur ipsum consectetur ut dolor enim eiusmod magna et aliqua ad quis eiusmod dolore veniam consectetur amet lorem et consectetur elit lorem veniam quis eiusmod adipiscing et magna aliqua dolore do veniam enim adipiscing dolor adipiscing sed adipiscing amet magna lorem sed sit et ut enim aliqua eiusmod dolore enim minim adipiscing lorem dolore veniam sed sed sit do veniam incididunt lorem dolore incididunt dolor consectetur elit elit amet eiusmod dolore incididunt elit incididunt ut eiusmod tempor elit ad do do tempor consectetur lorem dolor incididunt labore elit elit eiusmod aliqua sit amet tempor consectetur tempor ad aliqua elit incididunt minim elit elit et ipsum ut minim ipsum eiusmod veniam dolore ipsum tempor minim sit sed quis ad lorem enim veniam lorem do labore labore adipiscing ipsum tempor incididunt magna incididunt sit sit ut minim dolor elit amet magna incididunt ad adipiscing lorem quis enim sed dolor aliqua amet do sed quis consectetur ipsum dolor incididunt elit dolor adipiscing incididunt labore lorem lorem quis incididunt sed adipiscing dolor aliqua sit aliqua magna sed sit labore ut eiusmod adipiscing magna incididunt eiusmod ipsum ut lorem ut quis dolore dolore enim enim lorem ipsum lorem elit do ad adipiscing lorem do sit minim elit ut enim elit do magna tempor et incididunt aliqua minim ad tempor veniam lorem quis lorem dolor enim sed aliqua do ut dolore ad consectetur labore ut incididunt do enim sed lorem sed veniam tempor consectetur eiusmod eiusmod labore tempor labore magna minim ut consectetur veniam et tempor do consectetur adipiscing aliqua aliqua et consectetur ut lorem ad lorem et consectetur elit magna elit labore et quis enim incididunt amet veniam ut dolor dolore lorem magna tempor sed dolore dolore eiusmod consectetur ut eiusmod elit quis dolor labore lorem ad sit ad tempor ipsum eiusmod incididunt quis sit aliqua consectetur ad veniam dolor ut magna ut sit ut incididunt consectetur incididunt dolor ut eiusmod aliqua ut amet eiusmod elit incididunt dolore incididunt adipiscing sit adipiscing eiusmod lorem et veniam incididunt ut minim incididunt dolore et ut amet labore eiusmod sed consectetur minim aliqua magna quis elit ad consectetur dolor incididunt consectetur et consectetur adipiscing do quis minim incididunt amet consectetur ad ipsum consectetur adipiscing sit dolor do tempor labore consectetur dolore dolor labore aliqua adipiscing ut incididunt incididunt amet enim dolore enim incididunt sit consectetur elit et ut incididunt lorem sed consectetur ut veniam magna et consectetur sit magna dolore minim sed labore dolore ut enim do veniam amet do magna incididunt incididunt lorem do dolore ipsum aliqua dolore sed elit ipsum veniam labore magna dolor aliqua aliqua aliqua tempor ipsum dolore veniam aliqua consectetur elit elit minim ut eiusmod sed ipsum elit enim dolore do dolor elit enim eiusmod consectetur incididunt eiusmod dolor sed ad do veniam eiusmod quis dolor magna eiusmod lorem aliqua magna do ad labore labore sit labore et ipsum lorem amet ut consectetur sit lorem elit incididunt ut ad quis magna lorem tempor quis enim dolore veniam magna aliqua dolore incididunt amet et ipsum enim adipiscing ad incididunt aliqua sed consectetur ut quis et ipsum ad labore et dolor ad lorem dolore aliqua tempor quis veniam consectetur eiusmod eiusmod labore tempor amet incididunt ut amet ipsum ut eiusmod ipsum quis veniam minim aliqua dolore ad minim minim et ad magna labore amet sit amet incididunt sit tempor aliqua elit lorem tempor lorem sit dolore veniam magna tempor dolore tempor ipsum magna ut sed tempor enim ut sit quis sit consectetur tempor enim tempor ut sit ut sit magna ad minim dolor do minim ad consectetur dolore incididunt amet ut elit do consectetur sit dolore veniam do enim dolore minim ipsum incididunt ut sit elit consectetur tempor dolor amet elit et eiusmod magna labore do labore do labore lorem lorem dolor et sed minim aliqua do ut enim dolor minim tempor elit ad elit incididunt do ut tempor enim eiusmod ipsum labore labore ad consectetur eiusmod elit quis sed eiusmod minim incididunt ad tempor sed sed et dolore minim et enim veniam eiusmod ut sit quis veniam lorem lorem do elit adipiscing enim magna amet consectetur sit et dolore eiusmod ut enim tempor tempor tempor et labore do quis tempor ipsum ut lorem ipsum dolore eiusmod adipiscing magna minim dolore minim amet ut incididunt amet incididunt minim minim do amet amet adipiscing amet enim adipiscing incididunt enim quis magna dolore lorem sit sed et ut ad do elit consectetur amet ad minim quis ipsum tempor tempor do quis sed dolor lorem eiusmod sit ipsum minim ipsum adipiscing amet elit incididunt dolor sed enim magna amet ut ipsum dolore adipiscing consectetur et lorem amet eiusmod eiusmod minim ut eiusmod minim veniam ad labore quis eiusmod ad veniam eiusmod veniam dolore lorem magna aliqua dolore adipiscing elit incididunt consectetur amet incididunt ipsum ipsum aliqua incididunt sed labore elit minim ipsum ut ut labore ipsum incididunt adipiscing ut ut quis lorem et elit amet ad ad dolor dolor quis dolor tempor dolor quis ipsum dolor magna consectetur do amet et elit adipiscing minim elit eiusmod enim minim tempor veniam dolor sed ad aliqua dolore et sit sit sit incididunt elit do dolore consectetur sed et ut adipiscing veniam et aliqua minim labore incididunt lorem amet tempor magna enim sed sit elit lorem sed consectetur enim aliqua do consectetur adipiscing tempor do ipsum dolor veniam quis labore dolore quis ipsum lorem dolor magna lorem elit amet aliqua lorem tempor minim consectetur lorem sit aliqua tempor sit magna amet ad dolore magna veniam eiusmod minim elit ipsum ad quis lorem do et et ipsum ipsum ut magna tempor quis ut ad consectetur dolor adipiscing dolore adipiscing et consectetur magna sit sit labore amet lorem aliqua amet adipiscing sed veniam do elit et aliqua quis eiusmod do amet dolor incididunt lorem dolore et elit ipsum ipsum minim incididunt dolor aliqua enim et dolore eiusmod minim aliqua labore tempor eiusmod adipiscing ipsum ad lorem adipiscing eiusmod eiusmod minim elit magna labore eiusmod eiusmod consectetur amet aliqua amet enim eiusmod adipiscing incididunt tempor ut tempor ad veniam tempor eiusmod lorem lorem ipsum incididunt aliqua dolore incididunt veniam ad dolore ad quis magna consectetur elit elit veniam do adipiscing veniam sit enim lorem et labore sed incididunt sed ut ut ut tempor dolor dolore ipsum sed sed dolor enim ut enim labore tempor quis magna magna magna do magna ut aliqua elit dolor et quis dolor do et consectetur consectetur eiusmod do minim eiusmod enim do enim adipiscing elit aliqua incididunt amet elit sit quis lorem dolor minim enim dolor dolor consectetur et veniam sed veniam ipsum enim amet ut amet dolor enim quis veniam do sed enim aliqua et minim ad ad ad adipiscing dolor aliqua sit quis incididunt sit sed amet magna adipiscing ut elit adipiscing aliqua sit ad labore aliqua adipiscing enim enim adipiscing magna ad ad dolor enim incididunt aliqua ut dolor tempor dolore veniam dolor incididunt lorem consectetur amet ad minim dolor quis veniam lorem incididunt et quis ut magna eiusmod aliqua amet adipiscing dolor enim minim incididunt aliqua et aliqua eiusmod incididunt sit veniam aliqua enim do tempor lorem consectetur sit lorem sed aliqua magna eiusmod sit aliqua magna elit sit veniam labore ad et et elit lorem magna elit do eiusmod elit eiusmod incididunt sit adipiscing consectetur dolore magna enim ut aliqua dolor ut labore lorem amet lorem consectetur magna sit minim dolore ad labore aliqua eiusmod labore sed lorem elit aliqua ut eiusmod do aliqua sed magna labore dolore elit ut ut minim sed quis veniam incididunt ad enim eiusmod adipiscing quis sit quis elit labore amet ipsum elit elit ut magna adipiscing ut dolor lorem ut incididunt et consectetur aliqua aliqua ut ad amet enim consectetur consectetur quis dolor sit ipsum dolore sed magna adipiscing ipsum do ipsum aliqua veniam sed quis veniam minim ad lorem do minim consectetur veniam quis do quis ad ut aliqua sed lorem do elit labore consectetur ad labore ad ipsum incididunt magna minim et sit sed eiusmod ipsum et do magna enim ad lorem consectetur sit adipiscing labore adipiscing veniam veniam amet do dolore enim ut ipsum consectetur sed et ipsum et ut dolore do lorem aliqua sed elit tempor aliqua dolore dolore magna minim veniam consectetur dolore incididunt consectetur enim labore quis eiusmod quis tempor enim magna magna ad tempor quis quis tempor aliqua tempor ad eiusmod amet enim quis aliqua dolor enim lorem veniam incididunt lorem labore aliqua elit veniam dolore ut incididunt do incididunt elit aliqua eiusmod minim labore lorem aliqua labore do sit lorem labore magna incididunt veniam aliqua eiusmod veniam sed lorem incididunt sit ad dolore incididunt tempor lorem enim labore eiusmod dolor dolore magna lorem sed aliqua dolor veniam magna et lorem magna magna elit elit tempor labore magna ipsum lorem ipsum sed incididunt ut incididunt dolore ad quis adipiscing do lorem elit consectetur labore tempor et minim adipiscing incididunt eiusmod sit adipiscing quis veniam ipsum quis enim ipsum quis labore elit sit eiusmod consectetur amet et sed veniam magna tempor lorem sit elit labore ut quis labore enim minim minim labore ipsum incididunt sit labore enim dolor eiusmod ut veniam amet consectetur consectetur veniam tempor ut quis ad minim adipiscing sit ut do minim ut adipiscing ad lorem elit sit lorem aliqua tempor labore enim adipiscing eiusmod lorem sed amet amet minim aliqua do minim ipsum adipiscing sed ipsum do aliqua sed dolor aliqua dolor incididunt ut magna et tempor aliqua quis quis sed veniam sit aliqua consectetur aliqua do aliqua ad veniam lorem consectetur magna incididunt minim dolore elit ut consectetur ut amet enim veniam sit aliqua labore tempor magna quis magna aliqua ad do consectetur ad lorem veniam ipsum quis enim enim incididunt ut eiusmod quis sed ad minim minim labore tempor aliqua sit eiusmod incididunt adipiscing adipiscing tempor veniam lorem minim aliqua tempor tempor dolore adipiscing enim elit tempor amet veniam veniam magna incididunt dolor incididunt et adipiscing incididunt et ut veniam enim sed ipsum veniam consectetur aliqua ut sit aliqua quis ut aliqua elit consectetur incididunt tempor et veniam enim lorem consectetur et magna tempor dolor eiusmod elit ipsum enim minim do veniam ipsum ut sed aliqua tempor magna dolor eiusmod dolor magna dolore ipsum dolore incididunt dolor incididunt veniam quis labore dolore quis adipiscing minim tempor minim lorem dolore ad incididunt dolore quis veniam labore dolore dolore enim aliqua incididunt adipiscing ipsum veniam elit ipsum do do dolore incididunt labore eiusmod ut incididunt sit ut labore do et quis adipiscing lorem aliqua sed incididunt magna adipiscing ad ut dolor eiusmod incididunt et magna lorem incididunt labore elit lorem minim consectetur magna dolore enim magna enim elit dolore adipiscing et do dolore ad et elit sit adipiscing dolore quis magna elit quis tempor adipiscing dolore labore magna tempor do ad minim amet dolor do magna sed do incididunt sit et labore enim veniam ut tempor aliqua aliqua dolore dolor ad incididunt enim sed sit sed ipsum quis veniam sed quis minim dolore sit dolor elit labore amet incididunt adipiscing tempor labore sed tempor amet et consectetur do amet lorem enim sed incididunt veniam magna amet ipsum dolore incididunt incididunt magna lorem adipiscing eiusmod tempor incididunt incididunt eiusmod sed elit veniam quis et eiusmod do aliqua eiusmod consectetur enim lorem lorem veniam sed consectetur ipsum sit incididunt aliqua consectetur consectetur aliqua et sit ad do adipiscing sed amet quis tempor adipiscing magna amet elit aliqua ipsum minim ad elit incididunt ipsum sed aliqua labore et eiusmod tempor veniam et et magna sit do enim sit sed ipsum minim minim sit magna minim ipsum ipsum aliqua quis dolor labore eiusmod et dolore veniam eiusmod et do labore aliqua eiusmod lorem ut incididunt ipsum amet quis et enim aliqua adipiscing do magna aliqua sed adipiscing et tempor tempor veniam enim minim et ut consectetur magna magna sit minim veniam quis magna ut et dolor aliqua adipiscing sed enim ut adipiscing sit ut aliqua ad elit labore enim magna elit aliqua sed lorem veniam quis do ut incididunt tempor do lorem elit sed incididunt adipiscing veniam dolor tempor lorem amet labore eiusmod lorem dolore ad magna veniam dolor ipsum quis quis sed incididunt labore incididunt magna labore ad amet sit lorem enim enim ut eiusmod adipiscing magna sed tempor dolore eiusmod aliqua labore dolore quis consectetur ipsum labore do elit eiusmod adipiscing et eiusmod quis aliqua et ipsum eiusmod veniam aliqua et tempor sed enim minim do labore aliqua ad veniam eiusmod magna dolore quis sit dolor magna amet enim sed aliqua quis et sed dolor dolor consectetur adipiscing ad eiusmod consectetur adipiscing magna sed incididunt adipiscing dolore dolor quis quis sed labore quis ipsum dolore minim eiusmod consectetur amet minim magna ad labore ad adipiscing elit veniam ut lorem consectetur consectetur sed sit lorem magna sed incididunt lorem lorem ipsum consectetur elit adipiscing ipsum ad sed dolore et ut lorem ad dolore ad sit veniam enim tempor aliqua elit consectetur enim elit dolor dolor ad labore tempor ipsum tempor ad consectetur ut quis ad amet adipiscing adipiscing quis et dolore do minim dolore et lorem et dolore incididunt magna do ut adipiscing enim do enim dolor incididunt do lorem ad ipsum sit lorem sit amet ad lorem do consectetur ipsum enim elit dolore quis amet labore veniam ipsum tempor do labore elit quis dolore tempor incididunt enim eiusmod et dolor ad elit aliqua dolor labore ipsum adipiscing sed veniam sit incididunt ad tempor dolore incididunt do aliqua tempor sit enim magna minim dolore aliqua ipsum ad dolor minim elit ad et lorem ut enim adipiscing elit ad sed tempor ipsum eiusmod sit lorem tempor minim consectetur ad dolore adipiscing eiusmod do ipsum elit dolore elit quis minim enim et sit magna labore sed quis ad magna ad eiusmod sed aliqua enim elit ipsum labore eiusmod ut labore consectetur et magna sit elit eiusmod ad dolore eiusmod tempor veniam aliqua ut eiusmod veniam magna magna dolor dolore magna do amet tempor quis elit et ad amet ad sed dolor amet eiusmod dolor sed magna ipsum lorem labore dolor ipsum tempor ad magna ipsum ut veniam magna dolore sed lorem et elit minim dolor ad magna minim lorem adipiscing eiusmod tempor quis consectetur veniam dolore amet quis ad incididunt ipsum labore sit aliqua labore enim eiusmod ipsum consectetur quis do consectetur dolore adipiscing elit enim incididunt amet adipiscing eiusmod enim labore ipsum consectetur tempor ut sit minim sit aliqua ut tempor labore magna enim eiusmod sit tempor ad ipsum labore tempor do ad dolore amet do minim sit ut sit quis dolore dolor aliqua consectetur sit minim veniam veniam adipiscing consectetur ad elit aliqua ut do do quis lorem labore ipsum adipiscing labore labore lorem dolor amet dolore tempor aliqua ut sed consectetur enim dolore ipsum eiusmod eiusmod do amet do consectetur minim magna veniam tempor quis ad quis consectetur elit ad et lorem labore consectetur quis sed incididunt lorem aliqua dolore veniam ut dolor consectetur minim incididunt dolor sit adipiscing ad ut sed amet minim sed veniam quis quis quis et quis quis consectetur incididunt adipiscing do incididunt ad dolore adipiscing lorem dolor labore tempor adipiscing incididunt ut ipsum dolor tempor magna adipiscing dolore adipiscing tempor labore elit et dolor adipiscing tempor ad tempor incididunt labore elit dolore amet dolore sit minim incididunt ut do enim tempor labore dolore magna elit sed minim tempor tempor adipiscing dolor tempor enim amet dolor sed adipiscing sit ipsum ut dolore tempor quis enim ut consectetur dolore consectetur sed lorem elit dolore et lorem incididunt dolor magna elit elit quis incididunt quis quis labore elit consectetur tempor amet amet eiusmod consectetur incididunt veniam magna tempor tempor eiusmod incididunt consectetur labore incididunt amet veniam quis labore eiusmod ad consectetur enim ipsum ipsum labore sed elit sit lorem enim et et elit elit enim amet veniam aliqua minim adipiscing minim do veniam quis veniam magna ipsum incididunt ipsum quis magna ad dolore ad labore veniam ut elit veniam dolor quis tempor enim eiusmod elit enim adipiscing magna labore dolor sit ipsum consectetur magna magna eiusmod dolore et ut labore veniam minim amet dolor do quis enim sed veniam labore amet magna amet enim enim eiusmod minim do magna incididunt dolore et eiusmod labore magna aliqua labore consectetur labore ut aliqua sit ut enim amet enim ad lorem adipiscing ad adipiscing amet elit minim minim ad magna labore dolore tempor consectetur elit aliqua ut minim incididunt eiusmod adipiscing dolore ut enim sit adipiscing tempor sed aliqua ad magna incididunt veniam lorem adipiscing ipsum et eiusmod do tempor ad amet ut sed ut magna consectetur consectetur ut enim minim ipsum labore minim elit minim do enim minim do amet adipiscing sit incididunt adipiscing dolor ut ad sed quis elit do magna enim dolore incididunt consectetur lorem et dolore aliqua ut dolor labore consectetur minim magna dolore amet ut ut lorem magna minim minim lorem do eiusmod eiusmod ad quis eiusmod dolor elit sed tempor sed consectetur consectetur ad aliqua elit amet do ad minim et ut labore amet ipsum veniam tempor ipsum eiusmod sit adipiscing enim lorem amet do veniam ipsum elit dolore enim lorem minim sit ad ipsum consectetur lorem et enim consectetur amet do do adipiscing tempor dolore sit enim tempor ipsum eiusmod enim dolor lorem eiusmod do sed et consectetur consectetur dolore veniam sed labore quis sit dolor labore tempor do amet do eiusmod dolor do tempor eiusmod incididunt ipsum consectetur consectetur aliqua incididunt ipsum do ipsum ut eiusmod adipiscing ut sed ad adipiscing consectetur labore elit eiusmod ut ipsum lorem elit ut et elit eiusmod ad elit eiusmod veniam sed aliqua do incididunt do amet sit magna veniam ad ut lorem incididunt do dolore sit minim dolor magna aliqua elit consectetur eiusmod veniam enim tempor aliqua aliqua enim veniam lorem ut labore quis sit minim consectetur labore ipsum elit ad magna elit dolore consectetur quis et tempor dolor veniam sed enim minim consectetur et minim sit aliqua ad aliqua sit quis magna magna magna labore quis quis amet dolore veniam tempor dolore dolore eiusmod aliqua magna incididunt sit amet tempor et elit ut labore minim dolore ipsum veniam quis enim ipsum consectetur ad ut consectetur dolore aliqua ut consectetur veniam incididunt dolor sed elit amet minim adipiscing veniam labore consectetur et minim elit ipsum quis adipiscing ad ipsum sed ipsum magna ipsum tempor magna adipiscing minim quis labore dolore adipiscing veniam sit tempor do adipiscing elit consectetur sit ut tempor aliqua labore do do amet ipsum quis ad quis aliqua ad veniam ad lorem magna enim dolor ut tempor labore minim ipsum ut do incididunt consectetur adipiscing labore ipsum sit veniam amet ad magna do et aliqua tempor labore do dolor minim elit aliqua veniam aliqua magna incididunt incididunt quis eiusmod sed minim incididunt amet ut labore incididunt minim do elit minim do quis adipiscing veniam ipsum consectetur lorem tempor incididunt magna amet ad adipiscing magna ad incididunt adipiscing lorem sit sit veniam veniam adipiscing sit ipsum ut sed enim enim amet amet amet enim elit tempor et enim magna labore tempor labore tempor amet minim do elit tempor dolore lorem ad et adipiscing consectetur enim incididunt sit veniam magna ipsum enim dolor sit do eiusmod magna tempor magna dolor ut sit enim consectetur magna et sed adipiscing ad tempor enim quis sit aliqua ut ipsum sit labore veniam veniam enim et ad consectetur ut dolor sed enim do lorem consectetur eiusmod veniam magna eiusmod sed aliqua et adipiscing enim labore aliqua adipiscing et dolor dolor sed elit veniam et tempor elit dolor enim eiusmod et amet eiusmod magna minim aliqua adipiscing et amet eiusmod adipiscing ad sit adipiscing incididunt labore amet et sed veniam lorem lorem adipiscing amet elit enim eiusmod minim elit adipiscing ad labore do et dolore ut elit do elit incididunt dolor consectetur dolor incididunt eiusmod dolor lorem veniam lorem amet incididunt do ut adipiscing dolore labore sit dolor ad magna consectetur sit tempor elit aliqua ad do tempor adipiscing minim ipsum incididunt consectetur do labore veniam consectetur ut ad sed dolor enim aliqua do amet amet consectetur incididunt ipsum ut do aliqua sit incididunt eiusmod magna dolore lorem do consectetur ipsum dolore lorem magna tempor do aliqua et et lorem dolor dolore do magna enim aliqua ipsum sed et labore consectetur veniam elit lorem et ipsum eiusmod minim labore amet sit sit consectetur ad ut amet ipsum ut ut elit ad consectetur tempor consectetur dolor ad ipsum amet enim amet enim do elit magna consectetur sed adipiscing lorem lorem ut sit aliqua magna eiusmod adipiscing adipiscing ipsum magna lorem do ad veniam sit eiusmod aliqua tempor dolor adipiscing eiusmod aliqua dolore elit sed sed eiusmod veniam labore lorem aliqua magna do tempor tempor magna elit ipsum ut veniam quis minim aliqua ipsum amet do enim et quis lorem do magna dolor sit elit et ipsum ipsum enim incididunt adipiscing eiusmod ut et incididunt ad sit labore amet magna magna magna sit lorem magna sed ad lorem et quis sit tempor sit minim amet ipsum adipiscing ipsum consectetur dolor magna et aliqua minim veniam incididunt labore consectetur eiusmod dolore aliqua adipiscing ad aliqua aliqua adipiscing labore labore dolore adipiscing lorem lorem minim magna ad lorem elit incididunt lorem veniam quis amet ipsum veniam consectetur do quis tempor elit ipsum et eiusmod labore aliqua enim elit dolore quis eiusmod eiusmod sed tempor amet aliqua sit lorem labore ipsum eiusmod quis sit minim adipiscing amet consectetur enim aliqua sit magna sit tempor do amet aliqua sed veniam sed ad ipsum tempor incididunt labore eiusmod amet consectetur quis dolore dolore amet ut veniam adipiscing elit dolor magna aliqua aliqua amet et dolore minim sed dolor consectetur enim lorem amet do et elit consectetur minim quis aliqua enim tempor dolore ipsum elit magna ut magna et lorem veniam veniam aliqua lorem minim veniam ad ad sit elit ipsum lorem tempor adipiscing ipsum veniam et ipsum magna veniam quis elit sed quis amet eiusmod do elit labore lorem lorem aliqua aliqua sed labore eiusmod incididunt ut eiusmod sed ipsum sit lorem eiusmod et ad labore consectetur ad elit sit dolor amet sed ut veniam lorem sit dolore et sed magna aliqua enim ad do elit quis sit minim consectetur minim ad amet enim veniam quis lorem consectetur sit enim dolor ad ad aliqua magna do dolore labore incididunt lorem amet elit aliqua adipiscing et ut enim aliqua ut sed elit aliqua quis dolore lorem ad incididunt incididunt amet ad eiusmod sit elit dolor adipiscing tempor ut amet ad consectetur sit dolor et ipsum eiusmod dolor et dolor incididunt magna sit veniam elit sit tempor magna dolore do ad ut minim do sed veniam amet ipsum ut eiusmod sit eiusmod aliqua eiusmod incididunt adipiscing dolor consectetur tempor consectetur magna enim minim minim ipsum eiusmod dolor ut quis veniam veniam dolore magna minim lorem incididunt ad ad amet magna sit et adipiscing minim tempor do lorem quis et incididunt ipsum lorem incididunt enim ipsum elit sed dolore sed labore do sit adipiscing ad ut sit labore quis veniam veniam eiusmod consectetur labore sed dolor eiusmod et dolore elit magna dolor sed veniam adipiscing dolor et lorem ipsum do adipiscing quis veniam sit incididunt do et sit magna do consectetur eiusmod ipsum sit lorem et adipiscing enim eiusmod ut elit lorem minim quis elit elit et quis et eiusmod dolore aliqua consectetur ad eiusmod elit et ipsum minim minim enim eiusmod veniam aliqua veniam enim ipsum ad lorem eiusmod lorem quis et minim magna magna veniam ut dolore dolore do veniam lorem dolor magna dolor enim ut lorem dolore sed lorem lorem elit ipsum eiusmod lorem ad ad sed et enim minim ut sed enim lorem et quis sed ut dolore et et incididunt ut dolore magna adipiscing do et tempor tempor enim consectetur minim aliqua tempor elit ipsum sit aliqua lorem tempor amet tempor magna quis magna quis do incididunt veniam ad labore labore et lorem ad dolore veniam sit ad ut lorem quis aliqua magna tempor do eiusmod ut ipsum ipsum veniam do amet sed et eiusmod adipiscing enim elit aliqua aliqua ipsum lorem magna dolore magna tempor magna amet ad ad quis magna labore ut veniam magna magna dolore minim adipiscing amet incididunt enim minim ut veniam lorem elit ut lorem dolore elit elit quis eiusmod quis et sit magna enim dolore consectetur magna sed ad et et eiusmod do ut minim minim tempor amet veniam aliqua ad labore eiusmod eiusmod sit et quis amet amet do enim veniam do incididunt consectetur quis adipiscing adipiscing ut eiusmod quis amet enim ipsum lorem magna enim quis lorem veniam lorem incididunt magna ut elit labore dolore dolor lorem ut consectetur labore tempor dolore dolore do labore minim sed ut enim lorem quis dolor eiusmod et amet ut et quis sit sit minim veniam adipiscing sit lorem dolore sit amet sit minim ipsum elit sit sit dolor amet dolor adipiscing magna do enim aliqua minim et aliqua consectetur enim aliqua sed consectetur adipiscing lorem veniam eiusmod veniam aliqua minim adipiscing labore tempor dolor et labore tempor ipsum dolore adipiscing dolor adipiscing quis et quis consectetur sit veniam labore dolore lorem minim lorem sit adipiscing dolor incididunt lorem amet tempor minim do elit ipsum sit eiusmod elit adipiscing sed aliqua incididunt enim veniam tempor veniam magna elit elit sit enim aliqua enim magna amet do enim enim labore tempor tempor quis ad labore elit tempor minim quis veniam eiusmod incididunt enim lorem quis amet veniam incididunt ad veniam labore tempor et do dolor magna sit ad tempor quis consectetur magna quis enim ad quis ipsum tempor ut tempor amet veniam elit veniam lorem aliqua magna et ad amet veniam enim ut elit dolore aliqua sed amet ipsum minim enim magna et et ipsum aliqua ad incididunt amet magna ut sit ipsum lorem dolor minim adipiscing quis amet aliqua dolore veniam eiusmod sed labore quis minim amet minim do eiusmod veniam veniam eiusmod adipiscing sed tempor dolor ipsum incididunt dolore magna et labore aliqua adipiscing enim dolor et sit minim sed labore enim elit enim ut magna tempor incididunt aliqua minim magna sed ipsum sit eiusmod amet dolore minim consectetur ipsum dolor elit elit magna veniam magna quis magna et et dolore sit tempor sit eiusmod adipiscing labore dolor dolor sit ad et minim amet et do dolor labore amet enim dolore sed dolore amet veniam aliqua adipiscing quis dolor ipsum ad ipsum do sed quis et elit labore dolore enim adipiscing tempor quis incididunt adipiscing ut dolore enim amet dolor aliqua elit ad ut do adipiscing do amet consectetur sed tempor sit amet tempor minim veniam magna ut sit adipiscing tempor incididunt consectetur consectetur incididunt sed enim quis amet ipsum tempor quis minim lorem sed et eiusmod enim consectetur veniam enim adipiscing minim dolor veniam ipsum adipiscing lorem et enim tempor incididunt et dolor dolor sit veniam quis elit sit elit ut adipiscing adipiscing quis labore adipiscing lorem elit ad consectetur lorem incididunt consectetur ipsum magna tempor ut dolor tempor ut minim lorem ut ut tempor eiusmod eiusmod quis ut tempor tempor ut veniam eiusmod sit minim dolore magna magna sit enim quis magna ut ad dolor elit sit do dolor sed sed adipiscing labore aliqua amet sed lorem aliqua et do ad magna quis sit sit minim do dolor adipiscing sed elit quis eiusmod aliqua ad do minim adipiscing aliqua veniam sit dolore ad magna labore et tempor amet ut sit incididunt ut minim sit ipsum elit labore eiusmod magna do incididunt quis veniam et lorem lorem quis aliqua ipsum elit tempor eiusmod elit minim amet labore et quis lorem magna sed ipsum lorem do magna amet dolor ut ad magna ut eiusmod incididunt adipiscing elit sed dolor ad sed ut consectetur ad amet consectetur aliqua consectetur sed minim sed quis eiusmod incididunt do consectetur sit eiusmod labore adipiscing amet labore et magna elit ad do dolor minim ipsum eiusmod veniam eiusmod enim elit lorem consectetur veniam et quis ad eiusmod minim magna labore elit tempor dolor do sit enim do sit amet dolore enim quis aliqua lorem et consectetur et labore elit consectetur ad enim quis quis elit enim consectetur incididunt lorem aliqua ad amet minim veniam elit labore incididunt tempor enim minim et dolor dolore aliqua dolore eiusmod sit adipiscing labore sit lorem quis labore lorem amet et adipiscing dolor amet ut enim aliqua dolore lorem consectetur ut tempor tempor enim minim enim eiusmod quis ipsum quis elit quis sed amet do consectetur do do labore dolore lorem lorem eiusmod ut minim aliqua ipsum enim adipiscing eiusmod elit eiusmod quis dolore magna sed dolor aliqua dolor amet eiusmod dolore consectetur tempor aliqua amet lorem sed enim eiusmod eiusmod amet dolor eiusmod sit do ipsum minim dolor dolore ad et lorem tempor aliqua aliqua sit sed incididunt tempor aliqua aliqua veniam consectetur sit ad amet veniam do sed lorem ad tempor veniam eiusmod et magna minim eiusmod dolore consectetur elit labore adipiscing tempor enim lorem dolor sit dolor incididunt magna elit ad et dolor incididunt et elit lorem quis enim incididunt consectetur dolore aliqua tempor eiusmod ad adipiscing veniam do ipsum ad enim ad dolor eiusmod et ipsum consectetur sed dolor incididunt et amet aliqua sed incididunt amet magna ipsum consectetur sed veniam amet quis incididunt lorem elit labore aliqua dolor ipsum aliqua adipiscing aliqua consectetur enim magna magna tempor consectetur quis adipiscing lorem et magna enim labore magna tempor aliqua tempor minim et dolor ut et labore enim labore incididunt enim amet incididunt minim ut ipsum incididunt sed ipsum do adipiscing ipsum enim do do tempor enim eiusmod adipiscing elit amet sed aliqua aliqua ut dolor dolor incididunt consectetur veniam labore sed veniam veniam ipsum magna eiusmod lorem labore ipsum lorem tempor tempor ipsum sed amet sed sit sed sed eiusmod minim incididunt labore amet eiusmod elit amet ad aliqua dolore do minim ipsum adipiscing sed ut sed quis ipsum eiusmod elit labore dolor ad labore sit magna eiusmod aliqua adipiscing amet dolor eiusmod ipsum sed aliqua ipsum veniam dolor lorem ad adipiscing adipiscing sit amet quis dolor enim ipsum aliqua aliqua et lorem magna sit ut labore labore dolor incididunt elit labore minim ipsum ad dolore labore eiusmod do labore dolore ipsum aliqua tempor enim et et elit enim ad adipiscing quis dolor do sed ipsum eiusmod dolor eiusmod et dolore consectetur consectetur ad elit magna ipsum incididunt minim consectetur sed tempor dolore adipiscing elit veniam dolore magna dolor adipiscing aliqua adipiscing quis enim amet amet amet amet dolore adipiscing incididunt dolore ut dolore do elit ipsum et amet tempor minim sit ut sit magna enim dolor aliqua ad enim eiusmod veniam elit dolore elit minim sit ipsum ipsum minim veniam do ut tempor sit lorem lorem sit ipsum sed tempor dolore enim minim tempor sed sit dolor elit do sed aliqua ad dolore ipsum incididunt ad dolore amet sit do veniam ipsum minim sit ad elit ad aliqua amet ipsum labore dolor eiusmod sit tempor veniam eiusmod ipsum ut dolor sed magna incididunt sed ut sit dolore magna tempor tempor eiusmod tempor incididunt magna minim adipiscing veniam amet minim quis ut labore et labore et dolore ad eiusmod enim ipsum consectetur amet amet dolore ipsum magna veniam labore tempor aliqua ut adipiscing ad elit do eiusmod do ad amet et dolore do eiusmod consectetur sit sed sit ipsum amet sed aliqua quis magna amet minim ipsum lorem quis consectetur ipsum veniam sit ipsum labore tempor ipsum lorem sed sed veniam incididunt ipsum consectetur aliqua quis incididunt sed dolor aliqua sed dolor labore consectetur elit et sed et minim ad labore sit incididunt consectetur eiusmod aliqua veniam incididunt adipiscing tempor ad tempor ipsum et adipiscing dolore sed ipsum sit ipsum sed dolor do dolore ut incididunt sit magna dolore lorem lorem ad quis enim ad et lorem incididunt quis magna consectetur enim adipiscing ipsum dolor enim veniam et elit incididunt tempor sed do veniam dolore ipsum labore consectetur tempor dolor quis adipiscing enim dolore consectetur tempor sit et minim et sit dolore ad dolor minim minim tempor sed sit consectetur consectetur do elit dolore enim ipsum quis incididunt magna sit veniam tempor ad amet consectetur tempor aliqua dolore tempor ut tempor lorem veniam ad consectetur labore adipiscing ipsum ad do veniam minim consectetur consectetur amet lorem elit adipiscing magna dolor dolor veniam labore et dolore ad sit aliqua ad dolor aliqua incididunt ut aliqua quis consectetur ad amet adipiscing aliqua quis incididunt magna adipiscing lorem enim aliqua labore labore incididunt tempor minim enim incididunt dolor adipiscing magna ipsum eiusmod dolor sit eiusmod ad tempor sed consectetur dolor minim ad tempor ipsum enim minim aliqua ut veniam elit ut consectetur quis tempor sed magna veniam aliqua aliqua enim aliqua consectetur dolore consectetur elit lorem elit elit magna adipiscing labore amet tempor incididunt ut lorem aliqua sit magna dolor dolore sed sed lorem amet labore do quis labore lorem sit amet dolor aliqua adipiscing dolor ipsum ipsum sit ipsum elit ipsum labore ad dolor labore sed quis sit sed consectetur amet tempor veniam incididunt ipsum lorem tempor adipiscing magna et dolore labore sed minim veniam adipiscing sit amet ad dolor do do dolore ut labore aliqua et adipiscing do incididunt dolore enim tempor minim magna dolore sit elit labore ut ut dolore enim do sed dolore sed ad elit lorem do adipiscing veniam eiusmod eiusmod et sed adipiscing aliqua elit minim consectetur ut magna do ipsum dolor minim dolore lorem elit tempor ut adipiscing dolor incididunt consectetur aliqua ad amet dolore adipiscing sed dolore ut minim et ut veniam adipiscing dolor ipsum magna eiusmod aliqua eiusmod lorem veniam et incididunt enim do enim elit minim et veniam enim enim consectetur elit enim enim tempor magna consectetur dolor quis eiusmod magna magna eiusmod incididunt ut aliqua labore tempor lorem dolor dolor labore ad sed adipiscing quis dolore et enim magna et dolore ad do dolore quis adipiscing ut dolor enim enim adipiscing labore veniam ipsum elit sit veniam tempor eiusmod sed aliqua ad incididunt eiusmod adipiscing labore amet dolor adipiscing quis magna adipiscing labore elit elit consectetur magna ut dolor elit consectetur labore dolor elit ad eiusmod ad sed dolore lorem minim amet quis lorem consectetur dolor enim ipsum elit tempor labore quis ut minim et sed consectetur quis do et labore et enim adipiscing do magna dolore aliqua magna ad ad consectetur enim adipiscing consectetur sed sit ut tempor aliqua et ut aliqua minim aliqua magna sit ut sed enim consectetur incididunt ipsum veniam lorem sed dolore eiusmod consectetur minim do aliqua ad labore enim quis tempor eiusmod magna magna ut enim incididunt dolor consectetur veniam amet veniam elit sit quis magna dolore minim do elit dolor et et lorem quis do consectetur lorem et magna sit amet incididunt do enim elit veniam et tempor aliqua incididunt do dolore ipsum ipsum lorem tempor sit consectetur quis minim minim sed lorem sed ipsum ut ut lorem dolor ut dolore sed adipiscing minim elit do sit dolor tempor incididunt dolore ipsum tempor elit do minim incididunt lorem labore aliqua eiusmod quis amet sit veniam lorem incididunt dolore minim amet ad veniam veniam do lorem tempor do enim ad enim labore consectetur sit sed enim veniam dolore amet magna et do lorem sit magna elit ut aliqua ad ut aliqua ipsum incididunt ad tempor dolor tempor magna adipiscing magna dolore veniam tempor amet incididunt et adipiscing lorem labore ad sed et elit do do consectetur ipsum consectetur consectetur aliqua consectetur eiusmod enim veniam elit do ipsum labore aliqua amet ad eiusmod aliqua ipsum quis magna dolore quis amet ut magna adipiscing labore veniam enim ut adipiscing tempor eiusmod enim do aliqua incididunt magna enim elit ut eiusmod ipsum amet consectetur sit incididunt et elit lorem veniam enim ut quis veniam dolor enim et do consectetur veniam ad dolor labore tempor ut minim tempor ad eiusmod sit ad lorem dolore dolore dolore labore labore ipsum dolor lorem amet minim aliqua dolor incididunt sed do amet sed et et labore incididunt tempor aliqua eiusmod tempor dolore consectetur incididunt dolor minim dolore et eiusmod et dolor ad dolor elit ad dolor tempor sed consectetur aliqua adipiscing ad veniam minim minim ut dolor ad enim incididunt magna lorem adipiscing eiusmod consectetur veniam ipsum quis dolor veniam incididunt ad dolore ipsum tempor dolor aliqua enim consectetur dolor do enim magna labore lorem dolore enim sit consectetur ad lorem adipiscing do sit minim eiusmod consectetur tempor elit ipsum aliqua quis incididunt minim quis veniam incididunt sit sit incididunt aliqua dolore dolore sit ipsum dolore enim dolore labore veniam dolor tempor elit ut quis enim quis sed dolore enim ipsum lorem amet incididunt enim adipiscing minim labore amet ad ad aliqua eiusmod do do veniam ipsum lorem dolor elit aliqua dolore sit ut ipsum eiusmod sit eiusmod sed eiusmod labore minim lorem magna sed quis aliqua eiusmod dolore eiusmod eiusmod dolor enim amet amet veniam sed eiusmod amet ut eiusmod sit quis labore incididunt amet dolore tempor dolore adipiscing quis et incididunt adipiscing minim consectetur eiusmod enim quis quis magna et aliqua do dolor consectetur elit minim eiusmod elit lorem elit dolore sit veniam quis aliqua dolore sit lorem quis eiusmod et aliqua lorem aliqua amet veniam consectetur et quis ad sit aliqua eiusmod magna dolor sed elit dolore ipsum veniam ipsum quis consectetur dolor adipiscing et consectetur consectetur ad sit amet magna enim sit lorem aliqua do enim do adipiscing enim adipiscing enim sit amet eiusmod dolor veniam enim sit ut dolor consectetur veniam consectetur dolore aliqua magna dolor veniam elit amet consectetur do magna tempor ad enim aliqua adipiscing quis veniam lorem eiusmod veniam ut sed do et labore labore incididunt dolor dolor amet magna sit elit ipsum dolore veniam enim sit dolor ut tempor sit tempor et enim incididunt minim et adipiscing veniam dolore ipsum veniam quis incididunt labore adipiscing do et ad dolore labore tempor ipsum incididunt elit lorem sed quis dolore consectetur enim incididunt ut tempor do sit do sed elit ut tempor tempor labore et elit et consectetur ut do dolor labore eiusmod amet veniam magna sit sed amet aliqua sed consectetur eiusmod elit labore labore quis sed sit do eiusmod lorem lorem aliqua lorem quis et eiusmod ipsum ad incididunt lorem ipsum ipsum dolor dolor quis do eiusmod ipsum eiusmod veniam magna ipsum quis veniam eiusmod elit veniam incididunt amet tempor dolore magna veniam quis quis magna ipsum sed veniam minim dolor tempor labore dolore dolor veniam enim labore elit eiusmod quis elit ut quis incididunt sed sit eiusmod aliqua labore tempor ad labore magna enim ad elit enim lorem ut sed labore tempor incididunt sed elit et lorem sed eiusmod elit consectetur ad eiusmod amet ad ut eiusmod adipiscing dolor labore et quis ut et ipsum veniam sed magna ad magna adipiscing lorem ipsum do consectetur do quis amet veniam minim dolore amet magna et dolore tempor adipiscing aliqua dolore lorem amet et lorem magna lorem elit veniam ipsum aliqua minim minim ut minim magna dolor eiusmod ad incididunt dolor eiusmod amet amet ad aliqua minim ipsum ut aliqua ut ad dolore ad adipiscing ad lorem amet magna labore elit sed lorem enim minim eiusmod tempor do dolore enim aliqua elit eiusmod et consectetur dolor magna ipsum minim do veniam dolore aliqua adipiscing labore labore magna magna amet lorem dolore quis do consectetur sit ut labore ut dolor sit do amet dolor quis ad adipiscing enim do consectetur ipsum enim veniam sed amet labore adipiscing veniam elit ad veniam enim dolor ut dolore lorem quis eiusmod lorem magna sed eiusmod adipiscing veniam et do veniam incididunt incididunt ad labore dolor minim ipsum quis enim sed veniam quis lorem aliqua labore elit veniam minim enim labore minim elit minim tempor do et quis elit quis enim dolor ipsum ipsum eiusmod adipiscing ipsum adipiscing ut et ipsum dolor veniam lorem do amet amet tempor veniam minim enim ad dolor sit eiusmod ad quis aliqua tempor minim quis dolor dolor magna ad adipiscing minim elit ut amet ipsum do lorem labore dolor ut veniam aliqua dolore lorem ut ad consectetur elit eiusmod ipsum tempor sit magna sit adipiscing eiusmod et eiusmod lorem amet adipiscing do magna magna tempor dolore minim sed sit tempor lorem quis labore magna sed ad dolor do veniam sit tempor labore lorem dolor ut quis consectetur lorem veniam minim eiusmod labore et magna minim minim sed consectetur aliqua aliqua ad veniam ipsum amet tempor dolor tempor adipiscing dolore do consectetur ad minim sit quis eiusmod consectetur aliqua amet et enim aliqua ad quis amet sed labore do enim labore ut veniam consectetur ut enim incididunt adipiscing dolor dolor magna enim consectetur minim lorem incididunt sed elit eiusmod et quis consectetur lorem eiusmod eiusmod adipiscing lorem magna ut magna magna lorem lorem aliqua dolore adipiscing adipiscing quis ipsum dolore incididunt et quis do ipsum sit et aliqua ipsum consectetur sit dolore dolore quis labore do ipsum minim incididunt quis tempor tempor incididunt sed incididunt labore elit et elit enim do magna quis minim veniam ut incididunt consectetur sit dolor ad lorem ipsum dolore minim adipiscing enim magna do sit aliqua elit veniam aliqua consectetur incididunt dolor minim do adipiscing sit sit ut minim dolore veniam veniam lorem consectetur et sed enim eiusmod eiusmod incididunt enim veniam aliqua eiusmod amet aliqua dolore eiusmod consectetur veniam minim ipsum quis tempor aliqua et do sit veniam amet incididunt elit eiusmod veniam dolore amet minim lorem consectetur dolore eiusmod ipsum sed magna quis incididunt incididunt minim aliqua magna veniam do quis et consectetur quis labore eiusmod et elit adipiscing aliqua consectetur sit dolore lorem tempor elit sed magna enim consectetur sed ipsum incididunt consectetur amet minim ad minim ut tempor et quis dolor ut adipiscing consectetur do enim incididunt et aliqua magna labore consectetur eiusmod incididunt labore quis tempor lorem aliqua magna ad magna labore eiusmod magna veniam quis tempor sed dolor dolor amet enim tempor sit ad veniam sed eiusmod amet incididunt veniam consectetur ipsum labore sed elit veniam incididunt enim adipiscing dolore consectetur magna do ad consectetur labore elit ad dolor minim tempor adipiscing magna quis adipiscing et incididunt ipsum quis lorem dolor elit veniam ipsum consectetur labore magna dolore dolore incididunt lorem dolor consectetur ad quis veniam elit amet sed eiusmod amet dolor minim aliqua consectetur sit minim lorem elit sed ad elit sed minim ut minim consectetur sed do sed sed adipiscing ipsum aliqua dolore veniam et sit lorem aliqua minim tempor ad dolore sit labore ipsum enim magna enim tempor minim quis elit do consectetur et amet lorem adipiscing elit adipiscing ut amet tempor magna lorem sit adipiscing enim aliqua ipsum eiusmod magna incididunt do veniam lorem veniam enim ut labore dolor sit aliqua sit lorem aliqua ipsum labore elit sed et elit dolor consectetur sit ut et minim incididunt et veniam sit aliqua ut do sed ipsum adipiscing minim elit do aliqua aliqua aliqua eiusmod labore sed labore sit ut do dolor sed ipsum dolore elit tempor enim lorem incididunt quis quis labore adipiscing tempor sed veniam sed lorem dolor veniam tempor ipsum et incididunt lorem dolor veniam sit eiusmod minim dolore lorem minim ipsum incididunt tempor sed ad elit lorem et eiusmod lorem tempor veniam aliqua ipsum dolor ut sit ad dolore consectetur aliqua enim consectetur labore labore adipiscing dolore veniam dolor et aliqua tempor sed ut magna aliqua minim et eiusmod magna et eiusmod et magna ad ipsum tempor ad dolore tempor labore sit magna minim quis amet enim incididunt dolore sit elit dolore incididunt dolore sed aliqua labore enim ad eiusmod sit quis adipiscing et sit dolore sed adipiscing do consectetur sit aliqua ipsum ipsum eiusmod dolor dolore aliqua elit sit minim eiusmod incididunt quis minim elit aliqua incididunt ipsum lorem lorem tempor lorem incididunt magna ipsum ut consectetur elit quis lorem adipiscing sit sit ipsum eiusmod eiusmod aliqua do dolor incididunt consectetur lorem ad veniam elit ut eiusmod eiusmod do adipiscing enim adipiscing incididunt sit aliqua enim minim aliqua magna do et dolor et eiusmod elit aliqua tempor elit lorem elit dolor lorem consec \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_escape_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_escape_rar300.rar new file mode 100644 index 00000000..957dd034 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_escape_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lorem_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lorem_rar300.rar new file mode 100644 index 00000000..333c1217 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lorem_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_match.txt b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_match.txt new file mode 100644 index 00000000..5143781a --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_match.txt @@ -0,0 +1,257 @@ +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +alpha beta gamma delta epsilon zeta eta theta +0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_match_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_match_rar300.rar new file mode 100644 index 00000000..0bd29d41 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_match_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_repeat_rar3.cbr b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_repeat_rar3.cbr new file mode 100644 index 00000000..259afa77 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_lz_repeat_rar3.cbr differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_mixed_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_mixed_rar300.rar new file mode 100644 index 00000000..6cf2c0bc Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_mixed_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_solid_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_solid_rar300.rar new file mode 100644 index 00000000..51baec36 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/ppmd_solid_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/solid_lorem_a.txt b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/solid_lorem_a.txt new file mode 100644 index 00000000..b537ae1f --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/solid_lorem_a.txt @@ -0,0 +1 @@ +amet aliqua dolor sed sit et labore et ad incididunt adipiscing sit et lorem incididunt ut enim lorem veniam labore sed quis elit aliqua sit eiusmod lorem lorem lorem ad magna lorem incididunt minim adipiscing ut quis lorem dolore elit labore et magna elit tempor elit minim elit labore do lorem ut magna ad sit consectetur ad quis do sit quis eiusmod quis veniam dolore ut dolore minim adipiscing do do aliqua et dolore incididunt aliqua ipsum et elit quis incididunt ut minim consectetur tempor magna veniam minim quis tempor dolor labore minim dolore sit consectetur dolore incididunt tempor et quis lorem et ipsum do veniam enim aliqua aliqua incididunt ad consectetur consectetur dolore elit lorem adipiscing magna magna elit incididunt dolore tempor aliqua tempor labore sed minim magna enim quis lorem incididunt quis dolore amet dolore magna adipiscing ut ipsum et tempor aliqua magna adipiscing dolore ut et tempor ut tempor lorem magna magna enim enim eiusmod labore enim lorem elit ad consectetur magna aliqua consectetur dolor magna sed ipsum minim dolor dolor lorem labore lorem sed elit sed sit enim consectetur tempor do dolor consectetur consectetur sed dolore consectetur minim sed ad veniam do labore veniam eiusmod et et sit lorem do incididunt eiusmod ut adipiscing sed sit sed quis dolore adipiscing enim ut lorem elit lorem incididunt amet ipsum quis consectetur labore veniam dolore minim ut magna elit ad veniam dolore labore elit dolore ad lorem incididunt minim aliqua eiusmod minim ad ut ipsum quis do amet adipiscing ipsum do dolor dolor do do quis consectetur ut aliqua sed amet lorem magna ipsum aliqua adipiscing aliqua labore consectetur veniam enim dolore ipsum incididunt adipiscing tempor sit adipiscing aliqua minim ut aliqua adipiscing et sit minim incididunt do dolore et lorem eiusmod enim incididunt do lorem consectetur adipiscing eiusmod aliqua amet eiusmod ut adipiscing sed minim sit incididunt magna tempor minim magna et magna elit dolor quis ipsum dolor amet consectetur consectetur magna adipiscing sed eiusmod enim dolore sed tempor eiusmod eiusmod sit do elit enim veniam et amet aliqua magna sit eiusmod ipsum ut dolor incididunt amet amet eiusmod sit enim aliqua incididunt dolor aliqua magna elit aliqua dolor sed tempor do aliqua magna sit labore sed sit ipsum do lorem enim minim lorem dolor ut sit ipsum adipiscing elit aliqua ut consectetur sit labore consectetur minim elit consectetur quis sit ut incididunt magna do magna sed veniam et eiusmod sit adipiscing ad eiusmod ipsum lorem lorem do quis enim eiusmod labore incididunt eiusmod incididunt dolor dolor eiusmod enim labore sit sed adipiscing enim magna veniam et minim tempor sed consectetur magna adipiscing do adipiscing elit tempor dolor sed dolor labore dolor ad aliqua ad eiusmod elit incididunt do ipsum eiusmod consectetur eiusmod aliqua do elit eiusmod sit magna enim aliqua enim dolor elit elit lorem elit incididunt dolor sed magna dolor quis dolor lorem ad lorem do tempor et et amet sit dolore eiusmod dolor dolore minim consectetur consectetur amet amet eiusmod do sit veniam dolore enim do amet adipiscing amet magna quis ipsum eiusmod enim minim magna quis veniam adipiscing consectetur do ut magna consectetur ipsum veniam minim elit sed dolor minim labore ut magna sed magna labore magna labore lorem incididunt eiusmod consectetur sed et lorem ad ut aliqua lorem ipsum veniam tempor aliqua amet aliqua amet amet sed sed incididunt aliqua incididunt consectetur enim dolor elit et lorem consectetur dolore eiusmod dolore ad labore minim ad quis elit elit eiusmod et minim et elit veniam ut eiusmod magna enim quis ad sed ad elit ipsum dolor dolore ad tempor consectetur dolore adipiscing do do veniam do magna tempor consectetur veniam veniam quis labore enim dolor sit enim dolore aliqua incididunt consectetur amet sed ut adipiscing aliqua quis ipsum et minim incididunt veniam ad tempor incididunt dolore consectetur magna quis ipsum dolore dolor sed ad sit sed quis dolor amet enim minim minim veniam dolor labore elit incididunt ut incididunt consectetur eiusmod labore amet enim et adipiscing sit ut enim magna ut sit minim do sed elit incididunt quis magna lorem adipiscing dolore labore aliqua lorem lorem ad enim elit sed adipiscing consectetur do amet magna adipiscing sed do aliqua sed minim labore consectetur magna tempor et ut sit adipiscing aliqua incididunt adipiscing do sit lorem sit aliqua quis lorem magna do minim quis ad amet dolor dolore tempor aliqua do ut dolore minim tempor dolore eiusmod lorem sit labore veniam labore tempor do magna incididunt eiusmod quis minim aliqua et sit ad incididunt incididunt adipiscing magna lorem sed ad enim quis quis quis dolore adipiscing labore enim dolore ut quis veniam do veniam consectetur labore enim minim dolore adipiscing tempor dolore lorem minim incididunt aliqua ut incididunt eiusmod enim aliqua quis veniam quis dolor et quis elit ad ad do ad lorem ut quis ad amet ad incididunt sed consectetur dolor enim lorem tempor sed veniam ut minim magna do amet labore sed et consectetur labore dolore ipsum sed dolore sit quis aliqua ut dolor tempor dolor minim labore lorem consectetur dolore veniam consectetur veniam dolor incididunt ad veniam sed enim do adipiscing dolore adipiscing elit eiusmod sed dolor dolor veniam dolore minim tempor labore dolore magna quis ipsum consectetur do ad quis veniam magna sed tempor enim quis elit incididunt magna incididunt consectetur et sed enim eiusmod veniam elit sed enim veniam elit minim lorem enim incididunt eiusmod ut elit sed adipiscing dolor ad quis consectetur aliqua labore aliqua quis amet enim sed labore dolore consectetur amet amet veniam labore tempor do incididunt elit sit veniam adipiscing veniam minim do dolor sit elit incididunt eiusmod et sit consectetur ipsum ipsum enim lorem adipiscing minim ipsum et veniam dolore quis enim labore eiusmod minim sed sit enim veniam consectetur sit elit incididunt elit et labore incididunt consectetur elit elit do labore magna aliqua incididunt adipiscing labore veniam sed eiusmod et aliqua sit adipiscing dolor ipsum lorem lorem et eiusmod incididunt aliqua do adipiscing incididunt consectetur ad amet lorem lorem incididunt amet minim magna ipsum aliqua incididunt sed amet dolor labore ad do lorem ipsum magna ipsum dolore amet ipsum sed sit ut dolor adipiscing lorem et ad amet quis sed minim adipiscing minim labore incididunt eiusmod ad sed sed ad ad elit elit ipsum aliqua aliqua consectetur tempor ut enim veniam magna ad dolore ipsum tempor magna ut magna adipiscing veniam magna ut minim dolor veniam sed quis enim quis dolor sed consectetur sit amet ipsum adipiscing ut ipsum ipsum ad dolor dolore et dolore tempor sit eiusmod ipsum amet magna ipsum labore minim amet incididunt veniam labore lorem quis dolore sed dolor sed eiusmod dolor do ipsum incididunt ipsum quis sed eiusmod quis amet sed incididunt sit minim do sit ut elit dolore magna adipiscing eiusmod eiusmod dolore incididunt aliqua et sit amet ad labore dolore magna quis aliqua veniam dolore magna lorem do quis consectetur adipiscing tempor incididunt dolore eiusmod sit ut tempor amet aliqua dolor ipsum do ad magna eiusmod ut do eiusmod tempor sed eiusmod quis quis dolore dolore lorem dolore sit amet eiusmod quis eiusmod eiusmod aliqua dolor labore sed et labore tempor quis incididunt dolor aliqua ipsum amet ipsum dolore et aliqua sed elit veniam aliqua quis eiusmod tempor ad tempor incididunt do labore enim eiusmod magna dolore consectetur lorem amet sed minim elit aliqua amet sit consectetur ut quis enim ipsum sit magna minim sed veniam sit adipiscing sed dolor ad aliqua dolore ad dolor dolor adipiscing ad consectetur dolore ut lorem aliqua tempor et veniam do elit adipiscing enim et elit ut labore minim tempor magna adipiscing et quis dolor sed ut adipiscing lorem quis magna incididunt dolore et dolor incididunt enim dolore aliqua aliqua ut ipsum tempor labore lorem adipiscing do veniam veniam ad lorem magna sit do dolore quis eiusmod magna ad aliqua magna do dolore ut magna dolore ut enim ad aliqua do labore do amet dolore labore aliqua amet magna consectetur sed ad lorem ut quis minim aliqua ipsum tempor ut incididunt do minim minim lorem dolor dolor lorem incididunt sed labore sed tempor ad quis et eiusmod incididunt labore sit et tempor amet ut amet lorem consectetur sed tempor amet aliqua do ut sed dolore do quis ut veniam sed ut eiusmod et adipiscing veniam et incididunt veniam ut dolor dolor amet adipiscing amet elit quis lorem sit sed amet et sit incididunt ad quis consectetur lorem dolor ut enim ipsum magna adipiscing magna ut tempor ipsum ad sit quis magna minim ut minim quis sit sed minim sed consectetur et veniam ipsum adipiscing minim ad dolor incididunt sit minim labore do minim dolore et incididunt sit enim et sit amet incididunt enim veniam adipiscing consectetur dolore sed ut quis magna do et ad magna adipiscing enim eiusmod et sit lorem quis minim tempor veniam sed ipsum magna ad labore do sit elit dolore sed sed veniam elit ut amet amet sed adipiscing ut magna ad enim ipsum magna enim dolore amet ut sed sed et veniam do sed et adipiscing et tempor enim et elit eiusmod consectetur enim consectetur quis aliqua veniam labore magna amet ipsum dolore eiusmod dolore veniam amet ad adipiscing eiusmod enim et et eiusmod sit amet amet veniam sed elit dolor ad magna veniam ipsum aliqua consectetur minim sit elit aliqua adipiscing dolore aliqua minim do ut eiusmod lorem lorem do enim elit dolor quis elit sed minim ad eiusmod sed enim quis dolore incididunt lorem sit eiusmod tempor amet sit sed amet minim aliqua ipsum tempor dolor dolor quis sit do eiusmod elit sed dolore ipsum tempor lorem dolor amet incididunt tempor quis ad veniam elit sit minim eiusmod sed lorem dolore eiusmod sit tempor ad quis amet enim sed incididunt dolor minim aliqua enim quis dolore et aliqua ut magna incididunt do elit ad do magna amet ipsum enim dolore sit consectetur elit adipiscing ut sed magna lorem sed magna sed dolore sed et amet incididunt veniam sit quis tempor dolor ad magna tempor magna magna quis dolore minim aliqua lorem enim do labore minim amet amet dolor aliqua amet minim adipiscing et eiusmod tempor do consectetur amet incididunt labore incididunt sit enim amet sed do minim minim ad enim lorem magna lorem ad amet incididunt quis magna sit labore lorem ut enim minim ut sed tempor ut incididunt enim labore ipsum sit et ipsum ad veniam veniam lorem ipsum sit aliqua amet dolore dolore tempor magna sed aliqua ad tempor et veniam elit enim elit sit magna tempor consectetur sit ipsum veniam eiusmod ut quis tempor sed minim ad ipsum enim ut ut incididunt tempor do eiusmod labore veniam elit ad enim dolore amet ipsum eiusmod minim sit dolore consectetur magna ad ad et eiusmod veniam sit aliqua lorem et adipiscing incididunt ad consectetur incididunt veniam elit sit elit eiusmod eiusmod minim elit minim labore quis et tempor et ad minim quis adipiscing ut labore incididunt magna sit aliqua et sed amet amet lorem incididunt ut sit lorem ad dolor consectetur labore incididunt minim dolore do amet amet dolore sit sed lorem labore incididunt ad veniam quis elit magna veniam incididunt lorem magna elit ut consectetur minim consectetur eiusmod minim elit dolor magna magna consectetur consectetur incididunt aliqua lorem dolore adipiscing ut elit ipsum dolore quis adipiscing veniam dolore veniam enim ad magna dolor elit incididunt labore sit aliqua ad ipsum incididunt dolor magna sit ad et ipsum dolore elit lorem lorem do labore sed quis ut consectetur enim amet magna veniam eiusmod magna ad labore dolore ut magna consectetur veniam incididunt veniam incididunt adipiscing et sed tempor amet sed aliqua sed consectetur quis enim dolor quis tempor eiusmod amet sed sed sed tempor incididunt sed aliqua labore lorem amet amet sed elit adipiscing dolor aliqua magna enim adipiscing magna ut veniam elit aliqua amet magna labore incididunt veniam adipiscing dolor ad dolor amet minim ipsum lorem quis incididunt incididunt ut minim amet aliqua enim amet minim magna magna dolor elit incididunt amet do adipiscing minim quis incididunt tempor quis consectetur elit do veniam amet tempor et magna do dolor dolore do adipiscing veniam labore lorem do enim aliqua sit enim tempor labore sed enim ipsum ipsum eiusmod consectetur amet ad sit sit ut ad aliqua elit quis adipiscing dolore dolore incididunt sit veniam adipiscing incididunt minim dolore amet veniam aliqua sed quis lorem veniam sit adipiscing aliqua incididunt minim et magna enim elit sed ipsum ad consectetur minim minim magna dolore elit ut sed minim ut incididunt sed et sit minim amet consectetur magna lorem labore ipsum et adipiscing incididunt quis magna eiusmod elit sit dolor minim quis ipsum ut labore adipiscing consectetur enim dolore adipiscing dolore incididunt dolore tempor adipiscing elit tempor minim aliqua dolor eiusmod ipsum labore ipsum enim consectetur amet do et ipsum aliqua dolore dolor aliqua incididunt dolor incididunt dolore aliqua ad do incididunt sed tempor et ipsum magna et lorem ut do aliqua quis eiusmod amet enim aliqua magna sed dolor enim tempor ut incididunt dolore lorem aliqua aliqua sit ipsum aliqua dolore lorem sit eiusmod eiusmod tempor magna ipsum ad tempor aliqua dolor et ad dolor magna labore eiusmod dolore magna lorem consectetur eiusmod tempor adipiscing amet aliqua amet aliqua sit incididunt eiusmod dolore ut tempor eiusmod sed enim tempor ipsum veniam dolor ad elit sed incididunt magna do aliqua enim dolor dolor veniam consectetur sed ut dolor amet do magna quis ad sed elit adipiscing sit sed quis et ipsum quis dolore do adipiscing magna dolor magna eiusmod eiusmod do dolore amet ipsum labore tempor quis ipsum lorem eiusmod ut quis consectetur magna ipsum veniam aliqua veniam minim ad dolore ut consectetur adipiscing elit sit aliqua amet aliqua dolore sit quis sed labore adipiscing ipsum tempor labore eiusmod enim quis tempor elit ad lorem lorem et ipsum consectetur sed magna ipsum lorem elit dolor dolore consectetur ipsum dolore adipiscing adipiscing labore do elit et dolore tempor eiusmod incididunt ad dolor adipiscing enim consectetur adipiscing minim enim do aliqua ut enim et tempor lorem et lorem sit minim ad aliqua minim enim ut veniam aliqua eiusmod eiusmod dolor ad ut adipiscing veniam dolore et enim aliqua minim magna dolore et enim minim quis aliqua labore enim et consectetur sed minim dolore do aliqua incididunt enim magna sed sed do lorem enim ipsum labore labore tempor elit dolore labore adipiscing veniam et eiusmod veniam ad amet incididunt ut ipsum ad sit tempor lorem sed magna quis ipsum do incididunt lorem eiusmod eiusmod do aliqua ipsum adipiscing veniam dolor eiusmod sit minim ad dolor amet veniam do ut enim eiusmod elit lorem ad veniam veniam consectetur dolore quis aliqua ad tempor do do incididunt ut dolore labore dolor adipiscing ut elit enim ipsum enim elit ad elit elit veniam incididunt incididunt adipiscing enim amet quis do quis quis tempor lorem veniam veniam minim do labore et consectetur minim amet lorem tempor ut magna eiusmod dolore et eiusmod enim sit aliqua ad do magna minim sed ut lorem do dolor ad et sit dolore elit enim quis ad quis sed ut tempor elit ipsum sit enim dolore dolore dolore consectetur amet do ipsum dolor adipiscing lorem minim ipsum ut quis veniam lorem dolor ipsum lorem ipsum magna eiusmod eiusmod lorem enim lorem magna adipiscing et adipiscing sed do aliqua magna dolore sed elit consectetur adipiscing incididunt ipsum elit magna veniam labore ipsum eiusmod eiusmod ut sit lorem aliqua consectetur dolore ad dolor consectetur adipiscing elit consectetur do sit ipsum eiusmod quis amet dolor labore amet elit ipsum quis do tempor ipsum aliqua dolor labore adipiscing elit minim consectetur sit ipsum adipiscing ipsum quis quis sit dolor quis elit do veniam sed dolore ut elit quis ipsum quis sed adipiscing eiusmod tempor tempor labore minim enim incididunt minim incididunt dolor ut elit et eiusmod consectetur enim ad sit elit dolor ut sed magna do eiusmod tempor ut labore tempor tempor eiusmod incididunt et dolore lorem tempor amet do consectetur do aliqua amet magna veniam quis amet consectetur labore ad ad amet amet consectetur dolor enim sed elit tempor ad eiusmod consectetur sed et do dolor ut amet magna tempor labore sit amet minim eiusmod dolor minim consectetur et magna ipsum ipsum quis adipiscing ad tempor quis tempor dolore tempor dolore ad minim tempor eiusmod ad sit consectetur incididunt ipsum sed enim veniam adipiscing ipsum elit do eiusmod aliqua incididunt elit tempor ipsum elit do veniam aliqua lorem adipiscing sit amet elit tempor dolore sed amet consectetur elit dolor do aliqua dolore dolore magna enim magna ut labore aliqua dolore et consectetur dolore tempor adipiscing ut dolor sed adipiscing elit amet amet adipiscing lorem consectetur et tempor consectetur ipsum tempor dolor enim elit minim veniam adipiscing dolor labore ad ad adipiscing enim eiusmod consectetur aliqua veniam minim veniam lorem adipiscing eiusmod et magna ipsum ipsum tempor et magna tempor amet et dolor dolore eiusmod minim quis aliqua minim do enim eiusmod aliqua dolor et eiusmod ut dolor sed dolor minim ad eiusmod lorem consectetur eiusmod elit eiusmod sed sed do et ut lorem do consectetur ad do ipsum sit ut ut enim adipiscing sed tempor ad quis aliqua et aliqua do enim sed minim consectetur eiusmod amet tempor sit incididunt tempor dolore quis aliqua veniam adipiscing incididunt labore amet et veniam elit ipsum quis ad elit dolor quis dolor ipsum dolore dolore et aliqua et veniam eiusmod dolore consectetur aliqua veniam et incididunt lorem incididunt magna quis magna quis labore consectetur aliqua aliqua tempor ipsum quis tempor tempor labore elit veniam ad minim magna do dolor labore tempor adipiscing consectetur amet labore ipsum tempor aliqua eiusmod consectetur aliqua et et lorem aliqua elit enim ipsum labore ad consectetur dolore adipiscing incididunt labore sit eiusmod sed amet consectetur eiusmod amet consectetur quis enim dolore do elit magna veniam ut labore labore dolore magna do consectetur dolore enim dolore do aliqua adipiscing do minim amet minim lorem eiusmod sit ut incididunt veniam ad dolore quis consectetur enim labore labore magna labore tempor adipiscing ipsum dolor quis sit sit magna incididunt amet labore incididunt consectetur et labore dolore aliqua ipsum aliqua adipiscing aliqua labore et incididunt do tempor consectetur enim sed consectetur lorem magna ipsum minim dolor magna elit labore eiusmod labore eiusmod quis sit incididunt ipsum quis labore sed ut labore eiusmod dolore sit consectetur incididunt magna ut enim quis et dolore amet eiusmod amet tempor amet enim adipiscing elit adipiscing labore ad amet sit veniam sit ut ipsum labore amet tempor magna eiusmod sed incididunt lorem incididunt et veniam labore do quis veniam do ad aliqua incididunt eiusmod do consectetur sit et consectetur labore amet labore sit magna sit magna eiusmod eiusmod et minim magna ad eiusmod quis aliqua eiusmod magna aliqua labore eiusmod et veniam incididunt magna adipiscing consectetur elit magna adipiscing enim elit ipsum eiusmod enim ipsum eiusmod ut lorem tempor tempor tempor enim enim minim ut adipiscing do elit eiusmod incididunt veniam incididunt minim consectetur lorem incididunt ad tempor enim enim elit elit dolor enim eiusmod incididunt adipiscing veniam do sit ut lorem tempor dolor ut amet sit magna quis consectetur eiusmod amet incididunt ut eiusmod magna ad veniam dolore sed adipiscing adipiscing consectetur consectetur magna consectetur amet sit labore aliqua dolore amet ut amet eiusmod enim quis veniam minim eiusmod enim amet lorem tempor consectetur elit elit veniam et aliqua et ipsum ad dolor amet magna et aliqua amet adipiscing tempor veniam amet sed quis tempor dolor incididunt et lorem dolore labore adipiscing quis elit adipiscing veniam lorem quis veniam do ipsum sed dolore adipiscing dolor sit sit incididunt eiusmod sit labore veniam aliqua dolore veniam ad et minim sed amet ut tempor ad tempor incididunt ut ut tempor magna adipiscing adipiscing dolor amet elit elit lorem elit minim incididunt labore enim labore aliqua sit ipsum consectetur dolore lorem ipsum ut sed ut amet elit veniam minim tempor ut eiusmod aliqua quis ipsum dolore labore amet veniam dolore tempor aliqua ipsum tempor sit elit ad ad sit ut amet lorem tempor amet amet do lorem et ad lorem et dolor aliqua ut dolor et magna enim dolore sit amet magna minim veniam incididunt ad enim magna ut elit dolore incididunt et quis eiusmod labore sit dolor adipiscing aliqua enim veniam tempor sit sit tempor sit adipiscing sit veniam ad aliqua dolor lorem dolore ut elit dolor do et enim ipsum aliqua ut magna do incididunt ad ipsum minim enim lorem sed enim et labore elit sed eiusmod et labore magna ipsum sed dolore consectetur quis veniam labore labore do aliqua aliqua consectetur eiusmod dolore minim incididunt minim veniam ut minim magna enim incididunt et ad elit do lorem dolor amet et sit tempor sed do magna do amet sit dolore amet labore ipsum labore et quis aliqua eiusmod magna tempor amet veniam lorem magna adipiscing sed enim dolor labore do lorem ad sed quis dolore veniam lorem aliqua incididunt sit sit minim eiusmod enim enim ad veniam veniam aliqua labore dolor enim et dolore eiusmod aliqua minim ipsum adipiscing consectetur ipsum enim sit ipsum sit magna dolore do adipiscing consectetur magna amet elit adipiscing dolor dolore tempor veniam aliqua ut sed enim amet do aliqua elit dolor enim sed ipsum lorem ut enim do et ut ut dolor consectetur adipiscing minim ipsum ad ut ut tempor tempor dolore amet consectetur elit elit ipsum tempor dolor labore eiusmod adipiscing elit sed amet veniam veniam dolore incididunt sit et minim quis enim lorem et do sed veniam do adipiscing amet veniam ad incididunt minim ipsum incididunt labore magna lorem amet elit et ad sit do veniam enim ut adipiscing dolore eiusmod sit elit elit et aliqua sit consectetur et tempor veniam ad enim ad enim ut incididunt magna ut lorem ad incididunt amet ut amet ipsum do incididunt enim ut ad sit adipiscing enim sed et enim ut sed dolore sit eiusmod amet magna veniam magna sed minim minim lorem magna minim quis sit tempor labore sed sit do amet dolor ut veniam incididunt lorem et aliqua quis amet magna incididunt et elit dolore lorem incididunt ipsum ut enim dolor elit minim ipsum labore dolor do enim ipsum tempor ipsum dolor dolor ipsum aliqua do tempor do dolor magna et enim tempor eiusmod consectetur ad tempor dolore elit eiusmod enim elit elit ad veniam adipiscing do do magna eiusmod veniam do aliqua lorem minim et sed minim elit amet elit consectetur dolor sed incididunt adipiscing amet consectetur magna enim dolor eiusmod incididunt veniam adipiscing consectetur ipsum labore adipiscing incididunt sit veniam do elit quis ad do dolore ad labore eiusmod dolor dolor dolor elit sit dolore labore veniam magna labore lorem enim consectetur labore ut magna sit adipiscing lorem elit do adipiscing dolore enim do do sed tempor sed do ipsum lorem lorem ad labore ipsum adipiscing dolor eiusmod labore minim do sit elit minim sit adipiscing lorem adipiscing ad amet enim enim minim minim lorem labore quis lorem magna elit et consectetur magna lorem elit amet dolor lorem amet eiusmod aliqua dolor dolore magna sed adipiscing incididunt lorem magna sed tempor sed magna incididunt incididunt dolore dolore magna labore sed dolor consectetur et aliqua incididunt amet enim adipiscing dolore lorem dolore ipsum eiusmod amet elit eiusmod incididunt ipsum ut quis aliqua et dolore dolor veniam ipsum amet magna ut magna incididunt magna sed aliqua ipsum adipiscing adipiscing do veniam incididunt do dolore lorem aliqua sed adipiscing magna dolore quis magna consectetur elit dolor adipiscing et consectetur ipsum minim incididunt do lorem amet sit ipsum veniam aliqua ut et consectetur adipiscing aliqua labore minim sit minim incididunt elit dolor amet eiusmod dolore et et dolore minim tempor ut aliqua elit labore sed incididunt tempor incididunt aliqua elit incididunt enim sit consectetur minim enim ad tempor dolor lorem ut aliqua et ipsum labore sit ad ad elit labore tempor dolore dolor eiusmod minim ipsum sed aliqua dolore enim eiusmod amet aliqua consectetur ut minim do veniam labore quis elit et quis incididunt lorem dolore sed sit do sed lorem aliqua dolor eiusmod ad dolore minim consectetur elit do quis dolor consectetur labore tempor incididunt ad labore minim et minim minim sit aliqua et aliqua dolor minim ipsum ipsum lorem sed ipsum sed do consectetur magna et enim veniam minim eiusmod lorem labore eiusmod elit elit tempor quis veniam ipsum lorem labore dolore adipiscing incididunt amet consectetur elit dolor incididunt ipsum consectetur eiusmod lorem labore magna enim dolore consectetur ipsum ut elit sed minim dolore labore adipiscing ipsum enim veniam incididunt ut incididunt dolore ut sed labore eiusmod aliqua lorem dolor et quis quis ut consectetur ut consectetur magna dolore dolore quis dolore enim consectetur sed ut quis et do tempor veniam labore incididunt magna incididunt do elit tempor magna magna veniam veniam dolore elit sed lorem minim dolor sed veniam incididunt consectetur sed aliqua sed et lorem consectetur et sit elit amet sit incididunt ipsum consectetur dolor sit labore magna quis ad labore lorem ipsum sed ipsum dolore et quis ad adipiscing tempor enim labore sit eiusmod eiusmod incididunt ad incididunt do dolor elit quis labore magna tempor ut ut veniam quis quis ut aliqua sed consectetur amet ipsum eiusmod tempor incididunt dolor ad aliqua eiusmod aliqua consectetur amet quis ad sit magna adipiscing et veniam elit tempor enim dolore ad veniam consectetur adipiscing do consectetur quis amet ad incididunt ut et tempor veniam ipsum magna dolor lorem tempor elit amet adipiscing incididunt labore dolore aliqua sed ut enim eiusmod et eiusmod dolor aliqua enim ipsum amet magna quis et consectetur dolor lorem dolor lorem consectetur sed adipiscing quis labore incididunt veniam magna dolore sed veniam minim sed magna incididunt sit veniam incididunt labore elit dolor quis quis eiusmod amet minim enim lorem ad veniam incididunt ad ipsum do tempor minim lorem veniam enim labore eiusmod aliqua lorem magna eiusmod quis incididunt veniam quis ipsum aliqua minim labore minim veniam ad sit ut incididunt quis sit aliqua lorem lorem magna enim ut tempor consectetur incididunt quis ipsum amet do dolore veniam enim ut ad consectetur aliqua et quis do aliqua enim sed quis minim ipsum incididunt magna aliqua ut amet eiusmod consectetur labore incididunt aliqua magna minim amet dolore ad dolor enim aliqua enim incididunt sed incididunt et quis ipsum ad quis do consectetur ad sed incididunt sed sit sed lorem sit minim sit labore amet labore elit elit ipsum elit dolor sit sit quis ipsum aliqua minim sit ipsum sed ut amet tempor sit ipsum incididunt enim enim elit consectetur magna aliqua et consectetur tempor enim incididunt dolore aliqua minim consectetur eiusmod dolore dolor ad ipsum lorem aliqua do sit labore dolor lorem minim ipsum quis sed magna do aliqua enim sed labore incididunt sit ad elit do ad minim amet dolore dolore quis lorem tempor veniam labore sit ut minim amet sed sit tempor sed adipiscing eiusmod enim amet magna elit enim lorem elit veniam et tempor ad amet ut minim eiusmod ut enim labore sit sed ipsum dolore do veniam dolore eiusmod adipiscing adipiscing elit quis elit incididunt tempor sed lorem et dolore amet ut et dolor dolore sed sit elit sit ut incididunt amet sit minim labore dolore minim adipiscing consectetur adipiscing sed tempor veniam eiusmod tempor quis sed aliqua amet lorem elit sed et enim magna lorem eiusmod lorem consectetur veniam adipiscing sed ad elit dolor ut veniam tempor veniam tempor adipiscing sit lorem incididunt eiusmod aliqua eiusmod minim ut eiusmod aliqua veniam sed incididunt enim sed tempor enim dolor ut elit enim et tempor do veniam lorem sit enim dolore ipsum consectetur enim elit magna labore do ut incididunt enim lorem dolor incididunt amet quis aliqua adipiscing et minim incididunt et sit ut ad consectetur veniam minim et adipiscing ad do magna ipsum do do amet sed ad dolore do et amet ut eiusmod dolore eiusmod adipiscing sed ipsum do dolore aliqua do et do sed consectetur do sed eiusmod amet sed incididunt minim labore minim et quis consectetur incididunt ipsum dolor aliqua adipiscing eiusmod ipsum dolore quis do ipsum ut sit enim ad veniam eiusmod amet lorem tempor elit enim tempor dolore ut veniam elit dolore dolor ipsum eiusmod lorem ad labore lorem consectetur quis sed minim enim adipiscing ut do ad consectetur ipsum ipsum et incididunt magna minim minim sit incididunt do ut ipsum elit eiusmod ut aliqua aliqua et enim adipiscing aliqua dolore minim dolor eiusmod ad incididunt ad consectetur elit dolore et dolor ad ut minim incididunt adipiscing sed lorem do ipsum sed dolor consectetur enim sed labore veniam ut do sit do ipsum et consectetur sed magna adipiscing amet ipsum minim incididunt magna lorem aliqua dolore do lorem quis incididunt eiusmod sit sed consectetur enim veniam adipiscing dolor consectetur veniam minim aliqua ad incididunt dolore aliqua quis lorem elit incididunt minim lorem minim lorem dolore ut enim consectetur ipsum quis incididunt ad ut adipiscing consectetur elit dolor enim labore magna magna eiusmod minim minim sed adipiscing dolore enim sed incididunt elit minim do enim sed veniam amet veniam ad sed tempor aliqua quis sed dolore ad elit adipiscing magna minim lorem sit adipiscing sed consectetur quis eiusmod elit consectetur ad ad ipsum enim elit incididunt sed sed adipiscing ad sed incididunt ipsum ipsum amet quis et ut do tempor incididunt tempor enim adipiscing do sed sed et enim amet aliqua tempor amet incididunt ipsum dolor sed dolor et adipiscing labore do ipsum sed eiusmod lorem veniam minim enim et ut ut ut tempor enim quis et adipiscing quis ut incididunt do sit dolor quis consectetur veniam eiusmod tempor aliqua ut veniam incididunt sit incididunt ipsum ut enim adipiscing sit elit veniam et incididunt consectetur minim amet elit enim sit veniam tempor eiusmod dolore labore consectetur incididunt ad et aliqua consectetur ipsum dolore adipiscing minim elit amet sit sed magna lorem lorem tempor do adipiscing ipsum do minim minim amet amet dolor veniam consectetur aliqua ut sed amet dolor adipiscing consectetur enim ut adipiscing incididunt magna et consectetur enim dolor et elit adipiscing dolor minim amet elit adipiscing enim enim veniam amet magna sed dolor aliqua tempor dolor tempor dolore sed consectetur minim aliqua et ut magna aliqua veniam magna elit amet magna aliqua sit ut quis ut tempor elit labore minim magna incididunt eiusmod minim aliqua consectetur ipsum ipsum tempor enim labore consectetur minim quis labore aliqua tempor tempor amet labore adipiscing magna et magna do enim adipiscing amet enim elit do sit minim dolor ad elit ut dolore adipiscing minim do et veniam ipsum incididunt adipiscing ad veniam ipsum do quis do enim adipiscing ut lorem labore eiusmod ut elit ad sit consectetur dolore ipsum quis incididunt consectetur lorem dolore et et tempor magna ut lorem ad enim minim ut incididunt elit dolore lorem magna ad ipsum ad adipiscing ad eiusmod ipsum amet labore veniam consectetur labore amet amet sed dolore incididunt veniam dolor dolore sit dolor veniam incididunt et tempor adipiscing ipsum dolore ut elit et adipiscing consectetur elit adipiscing dolore minim eiusmod ad veniam do et enim aliqua dolore elit adipiscing do sit aliqua lorem ipsum enim eiusmod dolor dolore ad consectetur labore magna dolor ut quis amet magna ipsum veniam amet eiusmod minim quis tempor labore quis adipiscing incididunt labore dolor incididunt tempor lorem do adipiscing aliqua tempor tempor ad lorem sit labore incididunt do veniam consectetur do quis ad elit quis eiusmod eiusmod adipiscing ipsum ipsum lorem veniam consectetur enim labore eiusmod veniam ipsum sed aliqua dolore sed dolor veniam ad elit lorem amet ut tempor sed et ipsum incididunt sit do veniam ut elit elit dolore aliqua ut aliqua sed lorem lorem amet et veniam amet tempor dolor enim elit magna ad magna amet amet incididunt amet enim veniam quis eiusmod adipiscing amet amet enim veniam sit amet ipsum enim sed sed tempor lorem amet veniam lorem dolor labore veniam ut veniam incididunt do minim amet incididunt ut tempor labore tempor do et consectetur sed lorem do elit veniam ipsum et ipsum lorem dolor enim labore lorem ad elit amet veniam incididunt quis enim incididunt elit aliqua enim sed consectetur quis adipiscing consectetur dolor eiusmod tempor dolor sit magna elit adipiscing eiusmod magna magna labore veniam dolor ut aliqua tempor consectetur consectetur enim sit tempor consectetur minim et enim dolor labore ut adipiscing dolor dolor minim ad sed eiusmod incididunt tempor aliqua eiusmod ut dolore enim enim ad dolor adipiscing ut tempor dolore quis et tempor veniam sit labore eiusmod lorem elit do ut minim minim amet adipiscing sed veniam dolore enim ipsum consectetur aliqua do ipsum minim sit sed ad aliqua sit consectetur veniam ad magna labore aliqua elit labore ut ipsum amet et tempor dolore do quis incididunt dolor aliqua ut amet minim veniam dolore elit ut et dolor aliqua quis tempor magna dolore consectetur ipsum veniam adipiscing adipiscing lorem tempor elit elit veniam dolore dolore ad ut magna ut consectetur elit lorem elit dolore magna ipsum ad amet magna dolor lorem amet magna sed elit tempor eiusmod quis amet sit sed ut tempor enim ipsum magna ipsum minim labore ipsum quis ad eiusmod do do minim quis incididunt do incididunt et do minim sit aliqua minim ad lorem veniam sit ut dolor adipiscing sit ad lorem elit et dolor adipiscing eiusmod adipiscing do do labore labore veniam magna aliqua veniam dolore adipiscing veniam labore incididunt dolor quis lorem dolor do quis enim labore adipiscing quis do ut consectetur ad enim minim incididunt incididunt quis labore elit elit et lorem do sed et et tempor sit aliqua quis veniam minim sit adipiscing veniam labore incididunt adipiscing ut ipsum quis consectetur veniam minim incididunt ut tempor dolore amet dolor dolore veniam consectetur ipsum aliqua adipiscing dolore magna quis labore ad do do et amet lorem veniam labore ut minim minim aliqua tempor ut veniam tempor adipiscing sed adipiscing minim labore aliqua et sed minim enim ut do sed labore dolor sit eiusmod labore minim minim do dolore elit dolore eiusmod elit amet consectetur sed veniam elit ut lorem minim ut incididunt elit amet dolor dolor consectetur labore enim quis incididunt elit do enim incididunt sed lorem do amet sit quis ut do quis minim do veniam ut magna ipsum minim quis minim amet sit consectetur dolore et ut sit ipsum tempor ad eiusmod do ipsum do labore ipsum tempor do magna ad adipiscing sed quis sed consectetur do dolore eiusmod magna dolor veniam ipsum amet amet incididunt eiusmod eiusmod et consectetur do lorem sed lorem magna ad enim aliqua lorem ut magna labore lorem enim dolore incididunt sit sit aliqua aliqua lorem incididunt dolor et adipiscing tempor aliqua ipsum ut et magna eiusmod adipiscing lorem amet et et sed ut ad dolore sit ut labore magna dolore do dolor ipsum ut amet tempor adipiscing dolor labore tempor sit aliqua enim eiusmod minim sit adipiscing enim eiusmod consectetur consectetur eiusmod dolor adipiscing quis do magna aliqua dolor et aliqua labore dolore labore incididunt tempor dolore ad dolore et consectetur minim amet lorem consectetur do consectetur ad ad amet adipiscing amet veniam elit labore amet dolor et dolore magna incididunt incididunt enim ad quis ut magna ad dolore minim minim ut et sed et amet minim minim adipiscing incididunt ipsum sed enim amet labore adipiscing amet incididunt labore ad minim ipsum tempor veniam elit minim amet do aliqua minim aliqua et eiusmod amet enim ad quis dolor minim enim incididunt minim dolor dolor lorem lorem minim dolor dolor amet magna sed ipsum adipiscing ut eiusmod minim sed minim tempor adipiscing minim consectetur ut dolor tempor sit ut labore eiusmod dolore sit lorem minim ipsum amet ut enim veniam adipiscing adipiscing dolor minim consectetur labore dolore lorem eiusmod veniam enim do minim do amet labore ipsum ipsum do consectetur lorem enim eiusmod lorem amet sed sit elit sed ad aliqua et et adipiscing dolor amet do lorem ad elit consectetur minim consectetur elit aliqua enim labore sit lorem adipiscing aliqua tempor ad ad consectetur sed sit dolor do elit incididunt do magna amet do amet do magna sit do dolore sit adipiscing labore incididunt ad sit lorem incididunt et ad lorem do veniam et et tempor consectetur adipiscing et magna adipiscing dolore quis magna ad elit amet ut veniam quis adipiscing veniam tempor et elit ipsum elit sit tempor dolor veniam ipsum adipiscing ut eiusmod quis ut labore labore labore minim enim quis enim ad ad labore tempor ipsum adipiscing minim sed amet dolore quis sit ut adipiscing eiusmod sit lorem quis ad elit adipiscing veniam incididunt adipiscing do do minim tempor elit lorem minim elit enim sed do consectetur minim veniam sit lorem tempor quis amet veniam aliqua incididunt et labore minim sit elit enim tempor ipsum dolor elit consectetur adipiscing ut amet quis incididunt incididunt aliqua tempor dolor ipsum magna labore aliqua tempor do veniam tempor quis eiusmod tempor lorem sit incididunt do sed ipsum minim dolore et sed ipsum et eiusmod veniam ut quis labore aliqua magna dolore elit consectetur dolore ipsum incididunt sed adipiscing eiusmod adipiscing quis sit enim consectetur ut ut aliqua sed enim quis amet dolor sed elit veniam sed aliqua consectetur labore ut lorem amet do dolore amet amet ut ipsum labore dolore quis et dolore quis ipsum incididunt sit veniam quis do ut labore sit ad magna ut ut lorem sed enim ipsum do sed eiusmod dolore lorem amet magna ipsum adipiscing eiusmod quis sit consectetur do quis ut dolore amet dolor dolore et dolore dolor ut tempor ad amet labore incididunt enim eiusmod minim do et magna dolor dolore amet lorem ad dolor adipiscing enim enim veniam tempor et dolore amet consectetur veniam amet ut ipsum dolor tempor ad quis veniam do minim sed quis tempor tempor do ut incididunt et labore tempor eiusmod enim veniam ut amet amet veniam et sed ad veniam sed ut aliqua et enim ipsum do dolore et tempor et amet labore amet ad veniam et amet elit eiusmod dolor minim magna aliqua aliqua tempor consectetur ut ad ut do sed elit enim lorem ad et tempor dolor sed et quis incididunt labore ipsum ut sed et dolore amet eiusmod amet adipiscing quis incididunt sit sit eiusmod amet et ad dolore labore dolore amet et amet ipsum adipiscing veniam incididunt eiusmod ad quis sed et do ipsum ut dolor adipiscing lorem aliqua eiusmod incididunt do elit do ut tempor labore eiusmod minim quis sed sit minim tempor ipsum do adipiscing quis aliqua sit enim quis sed lorem sed eiusmod sit incididunt labore do magna sed tempor amet et ipsum sit incididunt labore eiusmod lorem labore ad elit amet eiusmod amet adipiscing et consectetur do tempor elit ad veniam minim adipiscing enim ipsum enim labore sed minim minim labore labore et tempor ut ut dolor elit tempor ipsum eiusmod sit minim incididunt magna veniam veniam et eiusmod amet adipiscing dolor sed et et labore eiusmod labore magna dolor et enim sed quis magna ad sit veniam aliqua amet ut dolor quis amet incididunt sit ut quis magna adipiscing amet lorem magna minim enim sit sit dolor ad eiusmod sit minim aliqua magna ad tempor ut minim incididunt dolor ut ad labore do quis ut labore incididunt enim magna enim amet labore enim et incididunt sit eiusmod sed ad amet sed minim ipsum sit consectetur quis minim ipsum lorem eiusmod eiusmod ipsum consectetur ut labore enim lorem ut magna aliqua incididunt sit minim incididunt ipsum lorem amet elit dolore et ad incididunt eiusmod amet do enim amet aliqua aliqua amet magna amet amet aliqua elit quis adipiscing lorem adipiscing et magna labore tempor et ad ut dolore lorem ut elit veniam incididunt sed enim lorem dolore eiusmod sit dolore elit ut sed amet dolore labore aliqua magna tempor sed sit quis quis quis dolor veniam tempor incididunt labore tempor ut minim dolore labore et incididunt magna lorem ipsum et eiusmod elit sit lorem tempor dolor veniam consectetur veniam et sit elit amet labore quis dolor consectetur ad sed et veniam tempor consectetur elit ipsum do ad labore magna magna labore enim do elit lorem do et amet adipiscing adipiscing consectetur do dolor sed consectetur incididunt ad aliqua do quis consectetur labore incididunt minim dolor tempor enim tempor elit lorem magna enim consectetur labore enim amet do incididunt amet eiusmod eiusmod sed lorem amet adipiscing sed adipiscing ad lorem ipsum ad ipsum labore do elit dolore ad sit quis dolor enim amet adipiscing consectetur lorem ut consectetur consectetur labore eiusmod ipsum labore enim dolore tempor veniam do aliqua amet magna enim dolore ipsum elit sit labore et elit dolore eiusmod sit minim do amet sed ad ut sed sit lorem lorem tempor dolore consectetur ipsum eiusmod consectetur ipsum ipsum lorem sed elit lorem quis do et quis quis enim ad et eiusmod aliqua dolor adipiscing consectetur tempor veniam quis dolor tempor amet tempor adipiscing labore incididunt labore tempor enim aliqua dolor elit elit sit elit dolor quis do dolore tempor dolor sit aliqua enim tempor elit magna do ad do sit consectetur veniam ut adipiscing labore amet adipiscing dolor ut dolor incididunt amet aliqua elit do sed enim dolore quis aliqua et incididunt sit sit minim tempor labore et minim incididunt adipiscing aliqua eiusmod amet ad sed eiusmod ad adipiscing enim ipsum minim incididunt sed eiusmod sit ad amet adipiscing tempor et ad eiusmod aliqua ipsum ipsum veniam labore amet labore et dolore adipiscing sed sed enim amet do sit eiusmod elit eiusmod eiusmod adipiscing sit ut adipiscing ut amet dolore magna dolor incididunt lorem incididunt amet incididunt tempor enim dolore incididunt enim amet quis minim dolor minim dolore elit adipiscing et et incididunt veniam eiusmod sed magna lorem dolore dolore veniam quis quis sed lorem incididunt quis dolore incididunt amet consectetur enim sed sit ut sit veniam et incididunt ipsum et dolor aliqua dolor elit eiusmod incididunt enim minim minim eiusmod tempor veniam minim sed elit ut amet elit minim incididunt enim quis ipsum elit adipiscing aliqua amet aliqua eiusmod consectetur sit adipiscing ut ipsum dolore eiusmod magna veniam tempor tempor minim veniam ut elit tempor incididunt dolor elit labore amet tempor ad tempor dolor labore dolore ut incididunt sed dolor labore labore minim amet sed veniam veniam lorem dolore tempor ut elit tempor veniam labore eiusmod labore dolore lorem amet veniam elit veniam sit elit sed tempor elit aliqua consectetur labore tempor tempor veniam ipsum veniam minim consectetur incididunt magna tempor tempor minim magna aliqua ut do dolore sed labore dolor ipsum sit magna sed quis do adipiscing ut magna aliqua adipiscing elit sed dolor ut incididunt minim dolore do do enim et dolor ut elit veniam adipiscing et sit dolore tempor aliqua magna dolor ipsum dolor eiusmod elit labore dolor adipiscing ipsum labore amet dolor minim et dolor ipsum amet et amet veniam et ut et do lorem enim sed magna ad sit enim dolor ad magna magna enim tempor ipsum elit magna minim magna amet lorem incididunt ipsum amet eiusmod minim incididunt magna dolor enim labore do eiusmod adipiscing ut et dolor incididunt sed lorem ad minim ad sit do quis elit aliqua aliqua incididunt lorem incididunt dolor ad aliqua dolor elit dolor ut dolore labore magna do dolore ipsum amet et et enim tempor amet magna sit adipiscing sed magna lorem et ad minim aliqua labore amet amet eiusmod incididunt labore sed enim labore aliqua tempor veniam incididunt minim enim aliqua dolor lorem adipiscing magna elit elit enim minim tempor ut magna sit dolor lorem elit ad eiusmod elit quis adipiscing dolore consectetur magna dolor dolore consectetur ipsum do ad sed aliqua veniam minim eiusmod tempor ut tempor adipiscing lorem enim aliqua et amet consectetur incididunt lorem consectetur lorem sed tempor adipiscing ut minim adipiscing elit ut minim aliqua adipiscing ut aliqua incididunt ipsum consectetur aliqua lorem ad adipiscing sit ipsum do amet magna incididunt enim tempor consectetur aliqua minim consectetur consectetur dolore amet et ad dolor labore consectetur do sed tempor labore elit aliqua veniam ipsum quis elit ad adipiscing sed elit quis veniam ut quis veniam sed veniam elit tempor et ut adipiscing labore dolore minim dolore eiusmod ipsum elit elit magna amet ut sit magna dolor ut ut dolore incididunt enim labore sit adipiscing ipsum incididunt adipiscing magna veniam adipiscing ut dolore quis dolore veniam enim dolore enim et tempor ad sit incididunt ad minim do dolore tempor eiusmod consectetur ut minim magna elit adipiscing dolor elit dolor sed minim ut quis elit et ad sed ipsum incididunt tempor incididunt labore ut adipiscing ipsum do elit amet amet eiusmod minim minim quis magna minim do enim aliqua labore ut minim do tempor quis eiusmod sed elit labore incididunt quis eiusmod ipsum aliqua adipiscing minim sit minim ad amet elit tempor amet veniam dolore labore incididunt sed lorem ut eiusmod amet eiusmod dolor adipiscing incididunt adipiscing incididunt enim dolore magna dolor minim ad labore adipiscing sit amet incididunt sit incididunt dolore aliqua quis elit labore et magna dolor consectetur lorem tempor ipsum aliqua labore amet et incididunt adipiscing quis adipiscing enim ipsum eiusmod adipiscing et sed quis consectetur magna et dolore quis dolore ut ipsum amet adipiscing lorem do ut veniam dolor consectetur ad lorem incididunt do dolore amet do amet consectetur do dolore elit minim do labore enim adipiscing dolore adipiscing veniam ipsum adipiscing enim incididunt eiusmod tempor minim incididunt ipsum minim incididunt dolor consectetur et amet tempor do consectetur ipsum veniam et labore tempor tempor tempor enim dolore quis magna adipiscing magna amet elit ut minim dolor elit sed do magna labore eiusmod ad labore lorem et veniam et amet lorem ut incididunt consectetur do magna veniam incididunt adipiscing aliqua minim incididunt quis veniam lorem elit ad et eiusmod tempor labore do dolor ipsum dolore eiusmod consectetur amet ipsum tempor et elit enim ad veniam ipsum ipsum lorem et labore labore ut elit incididunt consectetur sit elit do do elit lorem sit adipiscing ut minim do et sed do sit ad labore amet sed eiusmod do aliqua magna eiusmod ipsum do adipiscing sed labore dolore quis dolore elit lorem eiusmod labore sit dolor ut tempor sed ipsum amet veniam veniam ad dolore do tempor eiusmod incididunt consectetur ipsum labore eiusmod consectetur enim elit et consectetur enim ut magna incididunt sit do dolor enim labore consectetur aliqua ut et ad amet quis dolor lorem magna veniam magna et aliqua enim magna amet elit ipsum magna do ut minim aliqua labore sed aliqua tempor magna eiusmod dolore tempor aliqua ipsum consectetur ut elit et ad tempor minim sed quis quis elit dolor enim sed do dolore ipsum do et et veniam consectetur dolor minim aliqua dolore magna adipiscing ipsum aliqua aliqua dolore eiusmod do dolor magna enim elit aliqua ad consectetur amet minim quis ut ad enim eiusmod ad veniam adipiscing enim labore minim consectetur lorem adipiscing ipsum lorem dolor incididunt tempor ut adipiscing consectetur lorem ut ipsum elit ipsum incididunt sit tempor tempor magna enim amet incididunt dolore ad sit quis eiusmod et ad lorem labore elit incididunt magna dolor ut enim aliqua incididunt lorem tempor dolore do elit incididunt et incididunt tempor ipsum ipsum ut enim ut aliqua consectetur veniam amet eiusmod adipiscing amet et ut eiusmod lorem tempor tempor tempor quis dolor incididunt amet veniam ad ipsum amet ipsum quis do sed eiusmod ipsum dolor amet amet elit eiusmod adipiscing consectetur labore adipiscing ut tempor amet do aliqua consectetur labore lorem aliqua consectetur quis ad sit ipsum quis amet ut minim do dolore adipiscing ipsum dolor enim dolore dolore magna veniam ut quis sed veniam et magna labore labore ad aliqua enim dolor aliqua eiusmod sit do elit ipsum elit sit adipiscing consectetur ad incididunt incididunt ad amet aliqua elit magna ad adipiscing et veniam aliqua ad amet minim quis ad sit incididunt labore tempor minim lorem enim et ad et ipsum elit enim do consectetur dolore enim et dolor quis eiusmod dolor sed veniam dolore do minim amet magna ad ipsum eiusmod dolor amet dolore veniam amet consectetur minim lorem enim magna lorem ipsum dolor magna tempor et ipsum minim dolore ad minim magna dolor labore labore aliqua minim lorem sed tempor labore dolor tempor dolor consectetur consectetur aliqua lorem aliqua incididunt adipiscing aliqua minim elit amet et sit tempor dolore minim et enim sit quis enim magna minim consectetur dolore do sit do dolor enim sit aliqua eiusmod magna labore aliqua quis aliqua do labore do dolore labore magna labore aliqua amet do aliqua sed dolor minim do ipsum aliqua incididunt veniam amet et aliqua ad quis veniam eiusmod minim sed sit aliqua do ut elit elit incididunt magna aliqua dolor incididunt minim amet sed ipsum minim sed elit eiusmod dolor sed incididunt sit amet elit et eiusmod labore ut dolor dolore aliqua labore ipsum ut adipiscing incididunt aliqua aliqua minim sed enim enim amet amet incididunt ipsum lorem ad ipsum labore lorem tempor dolore labore eiusmod consectetur veniam minim adipiscing quis labore enim adipiscing minim adipiscing incididunt eiusmod veniam labore sed sed consectetur do minim dolor enim enim magna dolor dolore enim et aliqua aliqua magna elit sit et sit labore dolore magna consectetur sit veniam ad eiusmod dolore veniam incididunt aliqua elit dolore sit labore elit sit dolor et sed aliqua dolore do lorem eiusmod enim minim incididunt elit ut amet dolore aliqua labore dolor et sed ut quis magna aliqua magna dolor veniam sit incididunt sit sed incididunt ipsum dolor magna tempor dolor incididunt minim et et quis minim ad ad consectetur incididunt ipsum sed incididunt minim incididunt incididunt labore lorem veniam minim elit magna amet dolor et incididunt adipiscing sit veniam veniam labore adipiscing incididunt ad incididunt ut lorem sed lorem ipsum et quis ut amet enim consectetur quis adipiscing dolore incididunt do dolor magna ipsum ut veniam amet labore consectetur do ipsum ad do aliqua do ipsum enim dolore eiusmod do ipsum et dolor consectetur quis ad minim enim incididunt veniam ad et sed labore lorem ut ipsum do ipsum tempor adipiscing eiusmod consectetur elit amet dolor ipsum sit amet do eiusmod et aliqua aliqua enim et lorem incididunt aliqua magna incididunt elit quis dolore amet magna sit ut magna incididunt elit magna lorem enim aliqua magna eiusmod ut incididunt magna ad minim adipiscing amet ut consectetur sit aliqua sit quis do consectetur tempor et et dolore consectetur aliqua sed ad veniam elit sed dolor veniam dolore eiusmod elit ad enim magna elit sed consectetur ad amet ipsum labore tempor amet enim elit do quis ipsum ut labore et quis ipsum consectetur magna incididunt minim minim et ad quis eiusmod do sed consectetur tempor minim dolore quis quis tempor dolor ut elit veniam tempor sit tempor elit veniam dolor tempor minim magna minim minim minim do ipsum aliqua tempor dolore lorem incididunt minim sit lorem lorem aliqua consectetur labore sit adipiscing tempor dolor dolore amet incididunt consectetur labore enim sit dolore sed veniam et consectetur tempor minim amet sit ad amet dolor ad incididunt consectetur consectetur elit do sit ipsum enim adipiscing lorem enim veniam adipiscing lorem ad labore veniam adipiscing enim incididunt do amet eiusmod tempor enim incididunt sit enim enim adipiscing eiusmod lorem ut labore quis adipiscing consectetur lorem dolore sed labore elit dolor dolor adipiscing enim quis amet elit elit veniam magna tempor dolore dolore lorem aliqua dolore adipiscing labore ipsum sit consectetur ut amet minim et dolore elit elit ipsum tempor consectetur magna minim consectetur ut magna elit sed tempor ad enim magna adipiscing do aliqua elit sit magna ipsum eiusmod do magna consectetur quis amet ut eiusmod magna do enim lorem ad magna elit lorem eiusmod et ipsum adipiscing incididunt sit labore veniam ad incididunt et magna magna adipiscing labore sit minim ad adipiscing ad consectetur ad sed enim veniam sed lorem dolor minim minim incididunt lorem adipiscing dolore eiusmod sed ad aliqua adipiscing et magna amet labore ipsum incididunt dolore incididunt incididunt ipsum ad ipsum do tempor elit quis adipiscing labore magna et et ipsum magna adipiscing sit eiusmod veniam do ut quis elit veniam quis consectetur sed aliqua sed minim magna aliqua tempor sit minim veniam tempor ut dolore ut labore et elit sit consectetur dolore enim labore enim labore ipsum sit dolor incididunt incididunt sed adipiscing ad minim et aliqua minim minim tempor enim incididunt magna dolore sit aliqua lorem elit et incididunt sed enim sed dolor consectetur incididunt sit ipsum adipiscing tempor consectetur consectetur do aliqua sit elit ad lorem ut do consectetur amet aliqua amet enim eiusmod magna adipiscing sit eiusmod minim et adipiscing do enim veniam ipsum veniam enim adipiscing consectetur eiusmod minim dolore adipiscing adipiscing ad ut sit eiusmod aliqua aliqua veniam sed consectetur lorem ad veniam veniam dolore do incididunt magna incididunt incididunt dolor eiusmod eiusmod amet sed sed enim labore consectetur labore adipiscing incididunt et dolor labore veniam sit magna consectetur sed do ut do quis do adipiscing tempor ad veniam amet minim enim adipiscing magna ad sit ut adipiscing sed magna magna minim ad elit amet minim aliqua minim dolor eiusmod ipsum quis labore sed ad veniam ipsum minim ipsum enim elit ipsum quis tempor dolor consectetur minim consectetur adipiscing adipiscing ad enim ipsum dolore enim enim veniam ad dolore lorem minim ut et ad ipsum magna elit dolore elit minim sit eiusmod ut consectetur ipsum dolore enim incididunt do tempor lorem labore eiusmod incididunt labore et et tempor quis ad consectetur ut ipsum minim veniam dolore eiusmod sed aliqua ipsum lorem incididunt amet et adipiscing incididunt magna enim quis labore consectetur elit dolore ipsum enim consectetur ut labore quis aliqua ad quis lorem adipiscing ipsum aliqua enim incididunt dolor veniam dolore incididunt lorem amet elit veniam incididunt eiusmod adipiscing sit ad incididunt eiusmod ipsum enim dolor adipiscing tempor tempor veniam ipsum ad ut aliqua do ut do ad dolore consectetur quis consectetur dolore aliqua amet minim veniam lorem aliqua lorem ut adipiscing elit labore sit lorem ut consectetur magna do dolore quis labore quis minim consectetur enim aliqua incididunt adipiscing veniam do eiusmod sit ad sit magna sit et ipsum magna incididunt tempor magna ipsum sit labore sed ipsum aliqua eiusmod quis labore amet do labore ipsum amet do magna sit aliqua sit ad dolor ut ut dolore dolor ut tempor magna aliqua consectetur veniam aliqua et et veniam ad veniam enim dolore ut do labore labore elit adipiscing aliqua magna adipiscing amet ipsum sed quis lorem labore labore minim consectetur adipiscing do magna do amet dolor consectetur dolore dolor minim consectetur consectetur ut consectetur adipiscing ut sed eiusmod minim amet magna elit ut eiusmod quis lorem quis quis elit consectetur ad do ipsum sit elit enim minim lorem ut aliqua lorem dolore ipsum consectetur minim enim lorem magna labore quis quis ut ad amet tempor incididunt elit minim aliqua et veniam magna ad sed sit dolore quis do lorem do ad dolore ipsum do ipsum dolor tempor tempor sit minim dolore labore sed veniam ipsum ipsum magna sed eiusmod tempor elit sed et sed dolor elit ut incididunt et consectetur enim ut labore ut elit labore dolore ad dolor quis dolore incididunt incididunt adipiscing adipiscing enim amet ipsum ad amet labore tempor tempor minim sit sed elit eiusmod do lorem minim sed ut labore sed ad eiusmod veniam veniam dolor veniam quis lorem sed ut veniam amet elit enim tempor enim incididunt lorem elit magna veniam quis consectetur elit minim sit adipiscing sit adipiscing quis ad elit ipsum ut quis eiusmod amet quis minim lorem labore ad enim adipiscing ipsum enim ipsum veniam sit ipsum adipiscing tempor dolor incididunt aliqua enim adipiscing minim eiusmod incididunt do consectetur enim do do sed aliqua tempor elit elit sit quis incididunt amet tempor do tempor enim dolore aliqua tempor ipsum dolore ipsum dolor ut eiusmod dolore tempor magna elit do dolore do incididunt dolore magna eiusmod consectetur amet magna sit minim enim amet minim elit veniam do ut ut sed dolore ad enim minim ad tempor do labore labore ipsum labore magna ut lorem aliqua sit do enim dolore consectetur ad dolor sit elit tempor labore tempor minim do amet sed minim sit quis amet minim et eiusmod magna elit tempor ipsum ipsum sit consectetur et labore minim dolore dolor incididunt ipsum aliqua quis amet dolor do lorem ad enim labore elit enim quis elit adipiscing veniam amet magna aliqua incididunt amet aliqua veniam amet consectetur tempor minim ipsum tempor lorem quis minim minim et labore dolor ipsum veniam veniam labore amet incididunt ad sed ipsum ipsum dolore ipsum consectetur elit enim quis do dolore sit aliqua do consectetur ipsum aliqua sit et amet sed sit magna dolor minim eiusmod amet elit elit eiusmod aliqua labore tempor aliqua veniam minim lorem aliqua dolor quis sed aliqua magna consectetur lorem quis magna ad tempor enim do magna consectetur ad lorem et magna consectetur tempor aliqua dolore tempor veniam magna ut et et tempor ut sit minim dolor sit elit eiusmod adipiscing ipsum veniam enim aliqua veniam ipsum aliqua tempor elit enim ut incididunt ipsum minim ipsum ut quis aliqua sed minim aliqua tempor veniam dolor sit eiusmod consectetur tempor incididunt labore consectetur quis consectetur ut veniam et adipiscing lorem dolore quis consectetur adipiscing minim ad consectetur minim dolor enim quis quis enim enim lorem dolor adipiscing labore sed ipsum incididunt sed ut quis elit labore sit dolor adipiscing adipiscing adipiscing lorem consectetur ut ad et labore eiusmod consectetur minim adipiscing ut dolor incididunt veniam tempor tempor dolor dolore minim labore dolore minim magna veniam lorem adipiscing sed tempor tempor lorem adipiscing elit ut eiusmod tempor minim enim labore labore veniam quis quis sit quis magna ipsum ad adipiscing eiusmod dolor ut sed veniam minim elit ipsum do sit do aliqua amet ut tempor dolore dolor do dolor labore et consectetur elit aliqua lorem magna lorem adipiscing adipiscing eiusmod enim dolor aliqua incididunt ut dolore ad incididunt minim aliqua eiusmod incididunt labore do dolore tempor adipiscing amet magna ad do magna sit quis et tempor incididunt veniam consectetur minim ipsum dolor et minim ipsum incididunt amet aliqua quis amet consectetur dolore dolor quis incididunt magna adipiscing dolore incididunt tempor veniam aliqua ut consectetur ut aliqua sed elit quis aliqua tempor enim dolore lorem veniam do do dolore sed amet adipiscing labore ipsum sed sit quis minim dolor ipsum ad ad dolor et amet et labore enim incididunt elit minim ipsum ut consectetur ipsum do labore ipsum amet dolore ad ad adipiscing adipiscing ad veniam consectetur adipiscing amet lorem incididunt ad lorem incididunt enim tempor tempor eiusmod amet eiusmod eiusmod magna veniam eiusmod ad ipsum eiusmod labore consectetur dolor lorem quis lorem et minim sit adipiscing incididunt enim dolore ut enim sit sed dolore ut ipsum labore adipiscing ipsum magna lorem veniam quis do ut dolor eiusmod tempor aliqua incididunt ipsum enim minim sed amet adipiscing lorem consectetur labore ipsum enim aliqua ut veniam sit magna ipsum dolore consectetur veniam ad minim ad labore dolor lorem magna lorem amet elit lorem magna ipsum dolore ad aliqua minim elit enim veniam ut veniam sit et tempor eiusmod consectetur quis minim adipiscing elit dolor ad adipiscing consectetur veniam do dolor dolor veniam adipiscing amet quis dolor labore sed tempor elit amet consectetur aliqua elit ut dolor tempor dolore magna consectetur ipsum magna minim ipsum sed lorem magna quis aliqua et ut adipiscing aliqua sit tempor veniam eiusmod dolore dolore aliqua quis enim minim sed consectetur quis aliqua do do et dolore eiusmod dolore ad enim ut sit ipsum dolor dolor do enim ut ad incididunt tempor eiusmod aliqua dolor veniam amet magna aliqua quis labore dolore ut elit magna sed quis ut eiusmod ad do consectetur ipsum sit quis sit labore sit quis et veniam tempor sed sit eiusmod incididunt sit consectetur dolore quis magna labore quis dolore dolor et eiusmod aliqua sit eiusmod sit sit magna elit et elit et eiusmod ipsum consectetur ipsum lorem ut minim eiusmod do magna enim quis elit tempor amet minim quis dolor ad adipiscing ipsum consectetur veniam tempor ut minim sed adipiscing consectetur adipiscing incididunt dolore consectetur consectetur elit labore amet enim enim enim do aliqua eiusmod consectetur minim eiusmod magna aliqua enim do elit adipiscing consectetur do ipsum amet amet lorem ut ipsum tempor veniam labore amet ut adipiscing consectetur magna tempor quis consectetur quis dolor veniam sed tempor eiusmod ipsum dolore do minim amet quis enim et ad ipsum labore ad ut enim dolor lorem lorem ipsum lorem lorem adipiscing tempor adipiscing labore ipsum veniam ipsum elit do adipiscing magna consectetur adipiscing aliqua adipiscing tempor magna adipiscing quis amet aliqua veniam sit eiusmod consectetur magna ad labore minim et lorem magna et dolor lorem labore eiusmod aliqua tempor tempor ut amet veniam sit sit labore minim eiusmod consectetur ad dolor incididunt lorem dolore labore lorem minim enim amet do sed do ipsum et et ipsum aliqua amet lorem et enim ut ipsum elit sit quis incididunt minim magna dolore lorem aliqua do lorem lorem elit dolore veniam ad incididunt sed ad tempor quis lorem adipiscing enim quis ut adipiscing quis labore dolore ipsum amet magna enim sed ad ipsum sit incididunt sit ipsum et enim et amet sit veniam et sed consectetur ipsum sit magna labore dolor consectetur eiusmod labore magna consectetur veniam et quis tempor sed veniam do et consectetur minim do dolore lorem consectetur elit sit quis incididunt dolor sit do veniam sit sed minim elit consectetur minim tempor sit labore sit eiusmod dolore magna dolor labore aliqua ut dolor dolore enim sit labore enim et sed incididunt labore et incididunt incididunt sit incididunt eiusmod ad veniam lorem dolor consectetur et ut lorem dolor quis incididunt minim dolore enim eiusmod ipsum lorem dolore ad lorem labore quis dolore consectetur labore do veniam ut dolor dolore quis enim adipiscing ut eiusmod et ut ad incididunt minim ad lorem aliqua veniam quis dolore ad magna sit consectetur incididunt magna amet sed adipiscing et quis amet eiusmod quis quis magna ad sit consectetur enim veniam ad et lorem do ipsum minim magna sit sed et quis consectetur ut sed tempor do eiusmod sed consectetur dolore aliqua adipiscing enim enim magna sit enim consectetur sed sit adipiscing ipsum minim minim amet dolore ad aliqua minim ad quis minim labore quis ad magna sit magna magna elit consectetur ut ut enim consectetur ipsum eiusmod veniam lorem eiusmod veniam consectetur amet tempor elit minim aliqua ipsum veniam quis ut ad consectetur eiusmod aliqua adipiscing ad elit et do sed aliqua ad dolore dolore veniam aliqua dolore et elit consectetur veniam dolor aliqua elit labore amet incididunt consectetur lorem veniam sit quis ipsum ipsum magna ipsum enim et labore quis lorem et sed do sed quis incididunt et sit dolor do ad incididunt ut adipiscing elit dolore amet dolor veniam veniam quis do adipiscing sit tempor sed amet elit tempor tempor consectetur quis sit sit quis adipiscing do elit ad dolor ipsum eiusmod enim quis incididunt adipiscing tempor enim minim veniam dolor labore labore eiusmod minim do eiusmod minim dolore lorem dolore dolor amet consectetur lorem lorem elit adipiscing veniam minim quis quis ut do ad consectetur sit incididunt dolore dolore lorem enim adipiscing dolor adipiscing consectetur eiusmod adipiscing quis eiusmod elit sed tempor incididunt veniam eiusmod ipsum labore adipiscing minim sit labore amet incididunt incididunt adipiscing eiusmod eiusmod elit et labore incididunt quis lorem labore tempor ad ad et eiusmod ipsum labore incididunt minim enim veniam aliqua minim et et veniam minim do ipsum dolore labore elit dolor ad do ad consectetur labore adipiscing sed consectetur magna do quis incididunt lorem adipiscing aliqua enim et lorem do ad quis sed magna sed dolor incididunt ipsum aliqua magna veniam ipsum adipiscing veniam et adipiscing incididunt eiusmod do et enim dolore sed veniam sed enim consectetur incididunt incididunt et magna elit elit ut enim labore dolor veniam incididunt ipsum lorem veniam adipiscing tempor adipiscing sit veniam ipsum et adipiscing ipsum minim consectetur eiusmod ipsum minim dolore sed adipiscing adipiscing ut dolore dolor aliqua consectetur sed ipsum amet lorem veniam ipsum aliqua magna adipiscing quis consectetur eiusmod amet consectetur veniam incididunt aliqua lorem lorem aliqua eiusmod dolore ad lorem adipiscing dolor quis et labore dolor consectetur sed dolore lorem ut do aliqua magna aliqua adipiscing consectetur eiusmod sit quis aliqua do veniam do adipiscing aliqua dolore sit lorem sed adipiscing adipiscing elit magna adipiscing et eiusmod incididunt enim et quis sit tempor dolor labore sed et quis ipsum lorem eiusmod et veniam lorem elit magna amet ipsum incididunt sit elit ipsum do enim tempor sit sit sed sed quis amet elit magna quis enim amet et ipsum ut labore veniam lorem lorem eiusmod ad ut do et ipsum quis labore eiusmod ut ad lorem consectetur sed tempor elit dolor amet enim veniam eiusmod incididunt magna magna aliqua veniam do dolore eiusmod tempor et amet enim adipiscing ut adipiscing sit dolor consectetur lorem ad lorem dolor labore dolore aliqua dolor consectetur tempor dolore magna ad consectetur consectetur quis ut dolor labore sit lorem et ad tempor incididunt quis amet aliqua tempor amet enim elit incididunt eiusmod adipiscing dolore ad magna sit do labore adipiscing aliqua minim labore amet sed ipsum consectetur adipiscing veniam et magna adipiscing ipsum incididunt ad tempor et dolore ut elit adipiscing ad dolor lorem amet et enim veniam adipiscing et sed incididunt sit dolore dolor sit tempor consectetur adipiscing minim ut do ut sed dolore ad elit incididunt ut ipsum ipsum quis quis adipiscing tempor incididunt eiusmod ut sed labore veniam incididunt consectetur amet veniam ut tempor veniam minim tempor aliqua tempor dolore ut ipsum tempor sit eiusmod ipsum adipiscing quis enim tempor consectetur magna aliqua quis do et magna consectetur ut tempor tempor lorem consectetur consectetur enim tempor veniam veniam et ad labore minim magna adipiscing sit eiusmod ad quis lorem ut elit labore do ut sed dolor magna adipiscing labore sit ad ipsum veniam lorem magna aliqua elit elit ipsum enim et minim eiusmod adipiscing adipiscing sit lorem veniam do ad sed veniam sit amet tempor labore ut eiusmod veniam ipsum dolore do dolor labore elit consectetur aliqua aliqua aliqua adipiscing dolore do aliqua ad aliqua et do minim sed sit magna ad sed consectetur consectetur dolore quis enim consectetur ut veniam do incididunt incididunt elit consectetur ut consectetur magna magna eiusmod elit adipiscing amet do veniam minim dolor ad dolor veniam elit sit ad do lorem amet incididunt incididunt ad ipsum sit enim ipsum amet labore incididunt do elit consectetur et elit ad aliqua elit enim sed dolore ut labore aliqua aliqua veniam do amet ad minim dolore amet ipsum amet ad magna amet aliqua lorem dolore sed magna magna consectetur veniam ipsum tempor et magna enim consectetur elit eiusmod ad ad consectetur veniam adipiscing minim consectetur aliqua et minim quis elit amet adipiscing ipsum dolore eiusmod elit incididunt labore veniam sed veniam consectetur sit ipsum dolor ipsum do et ipsum sit ad do enim tempor tempor elit adipiscing sed adipiscing consectetur eiusmod sed minim quis eiusmod tempor ipsum dolor adipiscing consectetur incididunt ad labore adipiscing enim lorem labore incididunt incididunt tempor minim tempor et enim enim lorem ad sed quis dolor minim ipsum incididunt ut tempor veniam incididunt consectetur do ad adipiscing enim dolor do lorem ut ipsum labore lorem incididunt adipiscing magna enim et sed lorem enim veniam quis adipiscing elit minim ad amet enim veniam eiusmod elit ad sed eiusmod ipsum incididunt do lorem ad labore lorem ut lorem sit consectetur minim elit lorem incididunt enim labore amet dolor adipiscing adipiscing ut aliqua aliqua veniam tempor tempor dolor quis elit adipiscing sit amet enim sed tempor veniam quis eiusmod ut sed ut tempor minim elit dolor labore elit ad sed ad lorem dolore enim eiusmod adipiscing tempor aliqua minim do tempor veniam adipiscing minim tempor eiusmod magna magna sed ut dolor incididunt adipiscing quis labore quis aliqua adipiscing adipiscing eiusmod et labore lorem ipsum do amet dolore incididunt dolor do magna aliqua amet eiusmod eiusmod incididunt et ad minim aliqua amet sed elit elit incididunt veniam aliqua tempor enim enim magna quis ipsum do labore dolore veniam tempor ad lorem amet et et lorem ipsum incididunt ad veniam ut quis sit adipiscing minim ut quis tempor ut labore incididunt ut dolor et consectetur dolore minim adipiscing consectetur sit quis ut tempor elit consectetur incididunt veniam aliqua tempor veniam enim tempor amet magna minim consectetur elit sed magna elit ad dolor dolor adipiscing aliqua ut et ipsum eiusmod elit dolor ut dolor eiusmod consectetur et elit tempor quis aliqua aliqua adipiscing do consectetur labore quis sit minim lorem sit sed magna do sit quis do adipiscing labore quis elit quis lorem ut dolor eiusmod labore labore labore magna ad elit ut sed incididunt enim dolor aliqua incididunt minim incididunt sit sit tempor tempor tempor incididunt et et adipiscing ad adipiscing amet adipiscing sit quis lorem aliqua veniam consectetur et eiusmod sed do dolor adipiscing veniam enim magna amet minim sed veniam elit sit aliqua do dolore ut eiusmod aliqua magna tempor amet dolore labore dolore elit ad eiusmod adipiscing magna quis eiusmod amet magna amet incididunt veniam do ipsum lorem veniam ipsum ut tempor ipsum dolore labore aliqua quis minim labore ad dolore do magna incididunt incididunt minim eiusmod sed et dolor lorem amet eiusmod dolor ad tempor ipsum ipsum ad incididunt eiusmod adipiscing amet ut elit do eiusmod lorem elit adipiscing sed incididunt do et incididunt et enim labore sed lorem labore labore labore dolore ut quis enim adipiscing enim tempor ad ut consectetur consectetur minim aliqua labore magna tempor enim dolore lorem adipiscing quis sit tempor quis ipsum ad veniam labore dolor eiusmod ad sit dolor dolore tempor elit veniam dolor eiusmod amet elit tempor elit sed elit dolore incididunt aliqua ipsum lorem sed quis sed et sit et minim ut sed sit ipsum tempor et tempor magna enim sed adipiscing ut quis incididunt consectetur aliqua ad tempor dolor magna labore do et quis tempor adipiscing eiusmod incididunt tempor consectetur labore tempor sed magna quis amet elit quis consectetur eiusmod minim veniam lorem sit amet adipiscing amet magna dolore ipsum incididunt amet amet eiusmod et labore sit lorem incididunt amet incididunt dolor quis enim amet dolor dolore sed do amet aliqua quis quis eiusmod minim veniam sit aliqua labore dolor dolore ad lorem adipiscing dolore tempor veniam labore sit adipiscing lorem aliqua lorem minim magna sit do ad minim lorem ut ad magna consectetur magna enim enim sed quis consectetur ipsum lorem aliqua adipiscing adipiscing consectetur quis veniam sit ad tempor ipsum minim veniam minim minim do sit enim consectetur adipiscing lorem ipsum elit consectetur ut aliqua elit amet lorem dolore labore et ipsum amet et sed labore magna veniam tempor aliqua eiusmod enim minim labore ut elit quis tempor labore et et lorem adipiscing eiusmod adipiscing sit eiusmod labore ipsum veniam consectetur sed minim magna elit labore veniam ut amet consectetur aliqua ipsum dolore aliqua eiusmod et quis minim veniam do magna sed sed dolore dolore incididunt elit dolore elit eiusmod do sed aliqua eiusmod aliqua incididunt sed elit enim enim dolor ad et consectetur tempor magna minim aliqua consectetur magna enim dolor labore et minim tempor eiusmod dolor magna dolore adipiscing elit minim veniam amet tempor eiusmod magna dolore dolore magna consectetur consectetur ad adipiscing aliqua adipiscing adipiscing magna ipsum aliqua ad ut adipiscing labore ut eiusmod minim lorem eiusmod consectetur amet elit ut magna et consectetur dolore sit sit aliqua sit lorem enim labore consectetur dolor magna dolor amet veniam lorem ut sed adipiscing tempor ad labore ad et labore enim et labore ad aliqua sed sit eiusmod eiusmod dolor labore aliqua minim incididunt sed incididunt eiusmod dolor do amet sed ad minim lorem amet consectetur consectetur enim incididunt sit elit incididunt consectetur veniam do tempor enim dolore amet incididunt sit enim incididunt tempor elit lorem sit adipiscing veniam incididunt dolor lorem aliqua tempor tempor incididunt sit amet enim consectetur adipiscing adipiscing labore elit dolore consectetur labore sed et elit incididunt lorem lorem sit consectetur ipsum elit eiusmod eiusmod sit ut et amet ad ipsum sed aliqua tempor veniam adipiscing magna eiusmod et amet do quis do lorem tempor et lorem enim elit elit ipsum magna minim magna do consectetur et do magna dolor adipiscing sit consectetur aliqua lorem eiusmod amet dolore quis amet quis ipsum adipiscing consectetur elit amet minim elit magna ad elit elit veniam quis aliqua do dolore enim ut elit quis aliqua amet incididunt aliqua minim minim quis aliqua amet minim consectetur incididunt incididunt dolore amet minim sit aliqua dolore amet magna adipiscing minim consectetur et elit et dolor eiusmod ipsum et amet labore amet dolore magna incididunt incididunt sit lorem consectetur adipiscing enim aliqua elit sed enim incididunt sed ipsum ipsum et labore ad dolor ut amet ut adipiscing enim lorem lorem sed quis lorem elit amet veniam adipiscing minim et aliqua sed ad consectetur dolore quis adipiscing sed tempor incididunt enim lorem incididunt dolor magna quis ut do lorem do sit adipiscing et sit tempor ad elit minim enim consectetur enim lorem ad incididunt incididunt aliqua ipsum consectetur ad magna amet ad sed et enim quis incididunt amet sit tempor quis labore enim elit sed enim consectetur consectetur labore ad labore adipiscing incididunt ad dolor adipiscing veniam ut consectetur labore ipsum dolor ipsum minim amet amet ut adipiscing enim incididunt dolore sit eiusmod ut minim aliqua incididunt do veniam adipiscing ut ipsum ipsum eiusmod sed incididunt et do dolor eiusmod aliqua amet ipsum dolor ipsum consectetur et eiusmod ut quis ipsum labore sit quis et sed adipiscing incididunt consectetur ut elit quis tempor incididunt veniam sit veniam et enim incididunt eiusmod ad quis quis ad minim elit ipsum dolor sed quis ipsum aliqua adipiscing ut adipiscing magna enim quis adipiscing ipsum dolor do ad dolor enim lorem dolore minim lorem minim amet labore consectetur ipsum ad ipsum ad elit elit adipiscing magna lorem labore elit ut elit ad amet tempor ipsum dolore magna tempor ut ipsum tempor quis lorem elit et aliqua incididunt adipiscing veniam et sed consectetur tempor ad do sit lorem aliqua dolore magna incididunt magna incididunt enim aliqua aliqua ad sit amet quis consectetur sed ad tempor ipsum do adipiscing incididunt ad ad dolor labore ipsum amet minim ipsum amet amet labore consectetur amet consectetur aliqua amet eiusmod aliqua elit enim sit tempor sed sit ipsum consectetur sit ipsum enim veniam elit ipsum do amet magna lorem veniam minim consectetur labore ipsum ut lorem sit consectetur dolor ipsum ipsum eiusmod et enim labore sed quis et magna sit dolore minim magna enim elit elit do magna ut et lorem do aliqua veniam et labore adipiscing quis minim incididunt amet consectetur consectetur dolor eiusmod adipiscing et veniam lorem sit eiusmod quis lorem dolore eiusmod enim minim elit amet adipiscing enim dolore tempor adipiscing quis eiusmod consectetur ut labore veniam consectetur incididunt tempor tempor do sed adipiscing tempor sit tempor enim minim veniam consectetur quis aliqua amet sed consectetur dolore dolore sed aliqua elit elit ad labore adipiscing ad minim sed minim sed ad minim et eiusmod consectetur tempor do lorem ad lorem dolore enim incididunt ut do veniam tempor amet amet minim aliqua sit dolor aliqua aliqua enim dolor sit consectetur elit tempor ipsum elit tempor ut ut magna sed labore incididunt ad lorem quis magna minim veniam tempor labore veniam aliqua incididunt dolor tempor aliqua ad sit tempor labore do sit ut do minim et ut labore amet adipiscing tempor enim quis adipiscing incididunt amet dolore eiusmod ipsum consectetur dolore sit dolor ad enim veniam adipiscing lorem ad et eiusmod et ut consectetur enim eiusmod quis elit ipsum do dolor lorem ipsum lorem minim quis labore elit enim eiusmod eiusmod enim eiusmod sit amet do sed ad magna dolor enim incididunt eiusmod et ad eiusmod consectetur aliqua do ipsum do ut lorem magna magna enim adipiscing amet minim sit quis magna aliqua labore sed eiusmod sit sit tempor ut quis ad labore ut incididunt ipsum enim magna eiusmod lorem sit adipiscing incididunt elit veniam do et ut minim tempor incididunt labore labore ut dolor aliqua do et do et dolor amet minim dolor sit minim ad quis dolore incididunt consectetur tempor consectetur dolor ut dolore lorem elit ut aliqua consectetur dolore dolor labore ipsum adipiscing tempor eiusmod sit amet elit ad lorem et consectetur et do adipiscing aliqua sed enim aliqua sed labore ipsum do eiusmod dolor et labore et ipsum tempor labore et tempor lorem tempor adipiscing veniam amet dolor dolor do ut ut eiusmod sed minim minim dolore minim sed sit incididunt elit quis dolor et amet consectetur magna dolore magna tempor elit do dolor consectetur labore tempor labore adipiscing incididunt ipsum do sed lorem eiusmod ad consectetur do enim incididunt do adipiscing do veniam tempor veniam et do minim sed quis labore ad adipiscing adipiscing adipiscing do incididunt ipsum lorem magna ad sit tempor incididunt tempor elit minim dolor quis ipsum elit ad dolore ad magna quis tempor ut veniam eiusmod eiusmod magna do ipsum sed sed labore ad incididunt enim aliqua incididunt ipsum magna enim minim sed ipsum dolore minim elit sed ipsum quis incididunt minim adipiscing eiusmod labore sed incididunt do sit magna dolore minim adipiscing consectetur dolore labore sed lorem ut magna ipsum eiusmod consectetur eiusmod ipsum adipiscing ut et elit et magna minim ad dolor tempor veniam ut consectetur veniam aliqua magna minim tempor sit eiusmod dolor adipiscing ipsum sit amet sit adipiscing magna eiusmod enim amet labore incididunt do lorem labore amet tempor et magna labore ad et minim quis labore incididunt quis dolor ipsum magna magna tempor ad amet eiusmod et ut lorem tempor sit ipsum tempor elit adipiscing minim ipsum quis ad incididunt do quis dolore quis incididunt ad minim ipsum amet ipsum adipiscing veniam elit aliqua incididunt quis enim eiusmod tempor labore enim tempor ipsum ipsum sit dolor quis aliqua dolor amet ipsum lorem sit lorem ut eiusmod aliqua minim elit magna dolore elit adipiscing labore labore enim labore incididunt lorem ipsum do adipiscing do veniam elit sed eiusmod sed aliqua aliqua minim veniam tempor dolor sed magna adipiscing ut consectetur aliqua ut amet aliqua elit incididunt lorem eiusmod incididunt dolor veniam ut ad aliqua sit dolore et elit et dolor consectetur do aliqua minim veniam sed aliqua eiusmod tempor veniam minim elit magna enim incididunt sit do elit veniam sed incididunt magna aliqua labore sed magna adipiscing adipiscing quis consectetur magna adipiscing elit do ad dolor dolor quis sed elit adipiscing aliqua tempor adipiscing amet ad amet incididunt labore dolor dolore aliqua amet adipiscing adipiscing et labore amet eiusmod ipsum ipsum minim adipiscing dolore veniam amet adipiscing minim aliqua amet enim labore ipsum aliqua eiusmod labore elit adipiscing dolor incididunt sed dolor lorem tempor dolore veniam consectetur eiusmod do et et sed sit labore do tempor dolore adipiscing ut dolor quis do elit quis ut sed sit adipiscing aliqua adipiscing sed ut tempor lorem lorem ad et eiusmod sit tempor veniam elit magna veniam dolor sit sit ipsum ut do ipsum aliqua enim ipsum tempor consectetur eiusmod labore eiusmod lorem ad labore ipsum veniam dolore dolore ad ad ad ipsum incididunt sit ut ad minim dolore sit minim tempor incididunt ut sit amet do incididunt elit incididunt labore enim ad minim quis aliqua ad incididunt quis magna et ut consectetur sit ad quis minim ipsum elit sit ipsum et tempor consectetur quis elit aliqua ad ad et aliqua veniam ipsum ut labore tempor aliqua quis aliqua sed veniam consectetur adipiscing enim et elit labore eiusmod adipiscing sit consectetur quis consectetur ipsum sit veniam ut dolore sit minim adipiscing enim enim incididunt elit incididunt dolor ut tempor labore labore minim magna ut enim sit veniam labore lorem veniam magna minim ut ipsum ipsum incididunt minim ipsum ad labore ut eiusmod ipsum aliqua enim labore et sed minim ut minim quis et veniam consectetur aliqua aliqua dolor amet eiusmod minim dolor incididunt elit ipsum elit eiusmod enim do minim adipiscing tempor labore quis minim adipiscing dolore adipiscing sit sit sed ut ad ut ad minim do elit amet sed sed elit lorem sit minim ipsum minim eiusmod dolore amet sit enim minim quis labore enim eiusmod aliqua sed dolor enim elit do minim ut do consectetur consectetur ad do elit consectetur amet elit labore minim et tempor minim lorem aliqua eiusmod incididunt ipsum tempor quis elit eiusmod et lorem et tempor dolore dolor dolor amet ipsum labore dolor tempor veniam sed elit amet sed dolore quis ad tempor incididunt et ad consectetur et eiusmod magna quis ut adipiscing sed aliqua incididunt labore magna dolor lorem adipiscing tempor enim dolor ipsum consectetur enim et enim et incididunt ad elit ut et tempor quis labore elit lorem dolore consectetur ad consectetur labore consectetur aliqua elit sit ipsum incididunt ad dolore ipsum veniam adipiscing eiusmod amet aliqua veniam adipiscing minim quis amet aliqua incididunt dolor amet labore minim sed quis et tempor magna veniam magna elit veniam ipsum consectetur tempor sed magna amet dolore quis eiusmod dolor dolore et amet elit elit adipiscing quis tempor elit enim enim veniam eiusmod sit veniam eiusmod quis enim amet dolore sed amet labore tempor sed incididunt ipsum labore quis dolore sed ut enim eiusmod ipsum minim ad tempor do elit quis ipsum quis ad ipsum dolore minim adipiscing ad sed adipiscing sit tempor et aliqua sit magna lorem adipiscing adipiscing tempor incididunt sit adipiscing elit enim consectetur sit labore elit elit aliqua lorem tempor lorem tempor ipsum consectetur dolor et adipiscing magna do ipsum sed minim labore lorem enim elit consectetur dolore amet ut tempor aliqua consectetur ad et labore eiusmod enim dolor sit enim aliqua enim incididunt ipsum tempor sit adipiscing labore veniam ut ut minim adipiscing minim ipsum aliqua incididunt amet elit tempor ad ipsum quis tempor eiusmod do adipiscing ipsum eiusmod labore adipiscing do lorem et ipsum ut ipsum tempor magna do ut do ut dolore minim sed ipsum incididunt ad adipiscing ad ipsum quis adipiscing sit sit quis ipsum eiusmod quis adipiscing do quis ut aliqua et ipsum minim eiusmod ad sit dolor enim aliqua lorem veniam tempor magna tempor minim ipsum amet ut consectetur quis sit sed veniam ut amet lorem adipiscing aliqua ut ut do eiusmod ut consectetur magna veniam sed veniam enim sit amet ipsum ipsum labore dolor veniam dolore magna sed do enim veniam magna consectetur et elit consectetur lorem veniam do magna sed sit ipsum aliqua quis consectetur dolore lorem tempor ad magna dolor enim enim aliqua aliqua enim lorem tempor dolore amet dolore lorem dolore consectetur ut incididunt labore lorem ut dolore dolor dolor lorem consectetur eiusmod dolore incididunt do consectetur veniam elit lorem labore lorem ad adipiscing elit dolore minim amet aliqua ipsum dolor enim do adipiscing dolor do sit eiusmod et dolor ipsum enim lorem magna ut lorem consectetur ad amet magna magna sit do incididunt amet consectetur do quis incididunt consectetur amet do eiusmod dolore minim incididunt enim enim et sed dolore do ad labore do lorem lorem tempor sed consectetur incididunt ut ut et et dolore ipsum magna elit elit labore elit ipsum sit labore aliqua ut ut magna ad minim tempor sed minim ipsum tempor magna ut eiusmod incididunt ad incididunt lorem aliqua et ipsum amet ad aliqua eiusmod do lorem veniam incididunt dolor ut ad dolore incididunt minim quis dolor quis adipiscing eiusmod dolor dolor ipsum magna dolore veniam ipsum quis do eiusmod dolor minim veniam sed dolor sit ad lorem consectetur sed magna enim ipsum dolore dolore ipsum magna eiusmod consectetur aliqua adipiscing lorem sed enim ut ut quis incididunt ut adipiscing enim consectetur et consectetur dolor adipiscing aliqua elit dolore lorem ad magna adipiscing elit labore minim aliqua dolore consectetur et elit magna dolore labore dolor quis consectetur tempor eiusmod labore adipiscing dolor aliqua labore sit ipsum minim enim ipsum aliqua tempor sit do amet eiusmod tempor elit consectetur sit dolor eiusmod adipiscing dolor minim amet tempor consectetur ad sit elit veniam et labore consectetur et ad sit dolor et sit dolore minim tempor minim ut ut eiusmod tempor ipsum tempor dolore eiusmod labore ipsum consectetur consectetur amet sit consectetur do ad minim tempor elit minim ad elit enim tempor lorem aliqua eiusmod aliqua amet dolor dolor tempor aliqua ad dolore tempor et labore tempor ad dolor labore ad quis ut ipsum incididunt adipiscing quis magna do ipsum eiusmod elit minim incididunt quis lorem incididunt et ut do labore adipiscing ad incididunt aliqua sed dolor amet lorem elit et eiusmod eiusmod dolor do minim amet consectetur lorem dolore adipiscing dolore quis dolore ipsum sed et sit sed eiusmod do sed tempor ad elit minim magna adipiscing enim labore dolor magna aliqua veniam minim sed incididunt amet magna amet sit eiusmod ad sit do amet ipsum amet ad aliqua lorem veniam ad ut veniam veniam consectetur elit dolor adipiscing sed labore amet et consectetur ut tempor do lorem incididunt dolor sed amet aliqua et lorem do minim consectetur tempor consectetur dolor sed lorem amet eiusmod minim tempor dolore magna ad minim amet sit adipiscing labore sed minim ad adipiscing adipiscing labore adipiscing dolore dolor dolor ut ut ut enim magna minim labore consectetur et aliqua quis sed sit adipiscing do sit ad tempor do tempor dolor lorem adipiscing eiusmod sit enim adipiscing labore minim incididunt incididunt magna adipiscing elit ut ut amet tempor sed ad tempor sit ipsum veniam veniam ad incididunt veniam tempor et aliqua consectetur eiusmod dolor enim labore elit do quis magna ut magna aliqua veniam veniam enim aliqua magna incididunt incididunt amet adipiscing adipiscing lorem consectetur tempor amet do eiusmod dolor minim lorem ipsum ipsum veniam sed dolore adipiscing adipiscing magna elit eiusmod dolore aliqua incididunt adipiscing dolor do sit adipiscing sit sed quis tempor aliqua magna magna amet sed dolor adipiscing minim veniam eiusmod lorem tempor ut dolor adipiscing minim aliqua ipsum lorem labore do magna elit dolore lorem incididunt labore consectetur labore quis magna ipsum lorem amet do et enim et ad ad ad magna veniam ipsum quis labore elit adipiscing dolor lorem ut amet lorem ipsum magna sed eiusmod dolor elit lorem magna amet sit tempor ad adipiscing sed et consectetur quis minim labore enim amet magna eiusmod consectetur incididunt aliqua adipiscing do enim minim minim sed labore labore amet consectetur consectetur consectetur et ut quis eiusmod consectetur enim amet sed magna ad quis minim ad lorem incididunt magna sed ad consectetur magna tempor quis quis sit magna sed ipsum consectetur ad veniam ut lorem dolor ad lorem aliqua magna lorem labore magna ut tempor enim amet consectetur ipsum lorem tempor quis aliqua ad veniam dolor ipsum sit amet amet lorem sed elit aliqua quis aliqua adipiscing minim dolor tempor ut tempor quis et incididunt dolore adipiscing sed quis magna adipiscing quis ut elit incididunt adipiscing sit eiusmod consectetur incididunt minim elit eiusmod magna tempor ipsum adipiscing consectetur eiusmod do enim labore sit lorem consectetur veniam magna tempor adipiscing consectetur sed consectetur sit adipiscing incididunt magna ut do adipiscing dolor do et dolore labore elit enim quis ipsum sed eiusmod incididunt aliqua quis amet ad eiusmod eiusmod labore consectetur dolore sit labore amet amet lorem incididunt consectetur amet sit magna ad adipiscing ad sit ipsum aliqua et elit magna incididunt veniam tempor do dolor elit tempor quis lorem sit sed sed do magna do consectetur ut veniam et eiusmod incididunt sed magna incididunt adipiscing ut tempor dolor veniam magna labore do do incididunt tempor veniam minim lorem do amet sit quis eiusmod elit minim elit dolore aliqua minim eiusmod amet enim ipsum tempor consectetur aliqua sed dolore ad incididunt veniam ipsum quis enim elit do eiusmod eiusmod dolore quis enim quis ipsum lorem ut do lorem sit amet aliqua do sit ipsum sed ipsum enim et et ipsum adipiscing adipiscing quis ipsum tempor sed tempor dolore elit eiusmod eiusmod et ipsum aliqua ad amet amet aliqua adipiscing ad elit veniam ipsum dolor ipsum ipsum adipiscing tempor ut quis consectetur adipiscing adipiscing tempor minim elit elit lorem ut eiusmod magna ad incididunt amet eiusmod minim ad amet sit labore do quis incididunt incididunt enim sit adipiscing veniam ad eiusmod veniam aliqua tempor veniam incididunt incididunt labore do dolor et ipsum et quis sit eiusmod consectetur consectetur eiusmod ut amet aliqua consectetur veniam consectetur quis enim tempor incididunt dolor enim adipiscing enim quis enim quis quis enim amet quis quis et tempor sed veniam consectetur do et ipsum incididunt amet enim labore do eiusmod enim aliqua sit tempor ut elit sit lorem labore sit eiusmod enim et enim adipiscing sit dolore lorem magna do ut adipiscing eiusmod ut ad enim adipiscing consectetur quis consectetur ipsum et ut sit consectetur lorem minim aliqua ipsum dolor labore aliqua ut tempor adipiscing dolore ipsum tempor et dolor tempor consectetur incididunt elit veniam dolor dolore sed magna aliqua ipsum aliqua ut amet dolore sed ut elit eiusmod ut adipiscing magna sit adipiscing tempor magna et magna consectetur eiusmod incididunt sit elit quis incididunt ut labore incididunt enim do veniam amet adipiscing eiusmod lorem magna dolor ad dolor eiusmod ut ipsum et enim veniam dolore sit lorem ad labore minim elit et minim enim do incididunt tempor sed magna adipiscing veniam veniam veniam magna ipsum veniam magna ad sit minim sit magna magna sit ipsum dolor consectetur quis do ut ipsum dolor sit adipiscing enim amet labore lorem consectetur adipiscing adipiscing adipiscing consectetur quis lorem tempor dolore ad lorem elit ipsum enim amet consectetur sit elit ut amet adipiscing ad tempor ipsum dolor minim tempor do do ipsum adipiscing et lorem dolor dolore ipsum dolor et adipiscing sit ad sit lorem ipsum ad ut labore sit ipsum lorem magna lorem enim sed sit minim sed do consectetur tempor ipsum sit magna eiusmod eiusmod amet dolore magna ut dolor magna sed magna enim dolor labore sit sit ipsum veniam ipsum dolor ad ipsum amet adipiscing ut minim do ad magna magna quis elit do ad dolor dolore dolor enim sed dolore do labore et enim sed ut ut elit lorem tempor sit quis consectetur minim minim enim veniam amet dolore veniam amet labore consectetur enim lorem enim lorem ut incididunt eiusmod dolore et enim aliqua do lorem quis lorem lorem dolor quis amet veniam elit veniam et adipiscing sit dolor sed sit adipiscing quis amet do amet eiusmod amet magna amet quis sit aliqua veniam incididunt et elit ipsum labore elit magna magna veniam dolore et aliqua adipiscing quis do adipiscing tempor do quis lorem ad enim et dolore quis consectetur ad incididunt aliqua minim minim sed veniam amet minim sed labore incididunt consectetur aliqua enim lorem labore do ut ut aliqua ut ut incididunt consectetur quis magna enim incididunt adipiscing adipiscing dolor ad labore minim minim dolore aliqua eiusmod elit quis veniam incididunt dolor do sit sit incididunt lorem adipiscing enim incididunt magna labore incididunt sed dolor elit incididunt sit dolore do labore minim do sed ut dolore labore elit tempor eiusmod elit tempor minim lorem sed ad dolore sit et minim eiusmod do ipsum ipsum sed magna incididunt incididunt consectetur aliqua dolor eiusmod sed lorem eiusmod do elit dolore sit incididunt labore minim dolore tempor eiusmod sed et adipiscing dolore ad tempor ipsum incididunt ut magna minim eiusmod et lorem sit eiusmod enim labore labore veniam quis ad dolore sit sed amet adipiscing lorem minim minim minim quis labore magna lorem dolor amet lorem quis magna minim ipsum lorem enim dolor ad labore et quis incididunt tempor ut lorem adipiscing consectetur lorem amet aliqua dolor adipiscing adipiscing elit lorem et eiusmod elit aliqua amet ut incididunt dolore minim et adipiscing incididunt sed dolore ad do elit consectetur tempor lorem ut sed tempor sit veniam tempor tempor consectetur quis ut dolor dolor elit elit ad ad quis ipsum adipiscing sed amet eiusmod enim adipiscing minim minim ipsum et adipiscing enim veniam sit elit lorem dolore magna veniam sed lorem dolor et ipsum dolore incididunt sit ad ut incididunt incididunt aliqua tempor et tempor minim eiusmod labore quis ad sed dolore amet magna consectetur sit ut amet quis veniam dolor adipiscing veniam incididunt ut veniam aliqua do dolore minim labore ipsum eiusmod consectetur aliqua adipiscing aliqua eiusmod incididunt sed labore enim amet magna incididunt elit amet quis veniam quis eiusmod aliqua veniam adipiscing elit incididunt veniam lorem et veniam minim do amet quis minim aliqua adipiscing eiusmod eiusmod sed quis ipsum lorem dolore dolor elit incididunt aliqua quis consectetur dolore aliqua magna ipsum dolore adipiscing ad elit elit dolore eiusmod ut eiusmod ipsum veniam ad lorem magna ad enim labore eiusmod quis eiusmod consectetur quis elit et lorem incididunt ipsum do magna ut lorem do amet quis amet ad incididunt lorem ut et enim veniam amet adipiscing veniam sit eiusmod ad et ut dolor eiusmod tempor adipiscing dolor adipiscing enim ut sit aliqua ut magna amet dolore incididunt ut ut labore et enim elit enim ut aliqua elit minim dolor lorem eiusmod tempor elit sed ad ut eiusmod minim consectetur tempor elit adipiscing aliqua veniam amet et consectetur consectetur amet minim elit magna ipsum incididunt tempor eiusmod et magna tempor tempor quis quis elit et ut labore elit dolore sit minim amet adipiscing amet enim aliqua elit amet eiusmod ut labore ad labore labore sit minim adipiscing lorem labore incididunt et adipiscing aliqua enim labore ad dolor aliqua consectetur labore do ipsum incididunt enim veniam do consectetur consectetur labore do minim do lorem et et do tempor magna et eiusmod dolor dolor et magna consectetur labore minim ut adipiscing quis minim eiusmod sed quis veniam amet sed incididunt eiusmod lorem enim dolor labore magna quis veniam quis labore tempor quis et do incididunt do sit lorem veniam ipsum lorem amet lorem sed consectetur ipsum et veniam veniam ut aliqua ut eiusmod incididunt dolore adipiscing amet enim aliqua minim lorem ut ad eiusmod dolore incididunt minim ad elit elit adipiscing amet et adipiscing ad aliqua eiusmod ipsum consectetur dolor consectetur minim lorem ut do veniam ipsum minim ut enim adipiscing adipiscing et minim lorem dolore enim dolore aliqua veniam do minim ad enim elit sit eiusmod lorem et ut adipiscing elit labore consectetur consectetur amet magna consectetur labore tempor sit elit minim do enim ut ut labore minim lorem incididunt amet ut dolore sed incididunt enim tempor do quis ut et quis enim aliqua ad sit eiusmod sit consectetur amet do veniam minim dolore dolore veniam veniam consectetur dolor labore et ut enim sit dolore magna ad et incididunt magna incididunt tempor do do labore et quis labore do sed et elit eiusmod adipiscing minim dolore magna magna ut do sit magna ipsum eiusmod incididunt minim et sit dolor sit labore eiusmod do incididunt dolor tempor adipiscing et dolor dolor labore tempor eiusmod minim do sit incididunt ipsum adipiscing minim labore quis et sit do tempor elit sit et amet ut ipsum ipsum lorem ipsum sit sed veniam enim sit adipiscing sit sed aliqua lorem enim dolore magna dolore dolor do enim consectetur quis veniam labore amet consectetur quis quis enim ad minim adipiscing adipiscing incididunt aliqua labore ad enim minim dolor dolor minim amet et tempor sed ipsum sed adipiscing enim dolore consectetur dolore magna dolore consectetur labore consectetur magna ut veniam ut amet aliqua elit lorem do aliqua minim ipsum enim ipsum consectetur incididunt labore amet minim dolore enim adipiscing quis magna labore adipiscing sit amet magna sit minim sed do lorem magna minim aliqua tempor eiusmod minim quis do dolore dolore ipsum eiusmod lorem lorem incididunt labore consectetur minim enim enim incididunt adipiscing aliqua lorem quis adipiscing quis minim amet et ad ipsum tempor incididunt ut consectetur et veniam consectetur consectetur dolore enim consectetur labore veniam sit enim magna ad adipiscing lorem aliqua sed lorem tempor ad dolor eiusmod ut tempor aliqua et ut enim sed dolor ad d \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/solid_lorem_b.txt b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/solid_lorem_b.txt new file mode 100644 index 00000000..4679c421 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/ppmd/solid_lorem_b.txt @@ -0,0 +1 @@ +ipsum dolor dolor tempor consectetur quis minim do sed enim adipiscing enim ipsum aliqua minim consectetur ut ad incididunt quis dolore tempor magna labore dolore sed ipsum lorem tempor labore eiusmod incididunt ut dolore consectetur magna consectetur elit elit lorem consectetur eiusmod consectetur amet dolore dolore tempor dolore minim magna consectetur labore ut quis dolore tempor aliqua tempor tempor labore consectetur incididunt veniam quis labore ad dolore elit et sed et dolore dolore tempor minim labore labore tempor aliqua quis magna quis labore et minim elit eiusmod veniam consectetur enim sed et do do veniam dolore magna dolore dolore ad enim aliqua ut do quis adipiscing et dolore tempor minim enim dolor eiusmod quis lorem adipiscing quis sit ipsum aliqua ad ipsum sed aliqua elit minim sit dolore amet sed elit adipiscing ipsum ut veniam ipsum ipsum tempor tempor consectetur elit minim lorem dolor sit dolor lorem ipsum quis lorem tempor sed amet consectetur quis consectetur dolore veniam lorem incididunt aliqua ipsum elit amet ipsum lorem tempor enim ad quis quis sit do eiusmod et lorem do labore magna enim quis ipsum sed incididunt enim veniam amet et elit dolor minim minim eiusmod sit lorem labore amet dolore aliqua incididunt et dolore eiusmod amet eiusmod sed sed enim ut ad lorem veniam magna amet minim ipsum sed ipsum amet consectetur consectetur sit labore ad elit dolore veniam ipsum elit elit veniam labore dolor sed dolor aliqua elit enim enim veniam tempor sed minim ut sed dolore lorem amet ipsum incididunt ut consectetur sit dolore quis dolor elit sit sit lorem consectetur elit sit adipiscing lorem dolore minim labore labore do magna ad incididunt adipiscing minim adipiscing quis ut ut dolore lorem aliqua aliqua ipsum ut dolore aliqua consectetur sit minim et tempor lorem dolore sit enim tempor do veniam tempor do lorem minim ut sit sit do adipiscing minim lorem labore ipsum ut ad et labore adipiscing aliqua enim dolor lorem do lorem tempor do quis dolor elit et adipiscing sit aliqua tempor incididunt veniam labore amet tempor incididunt sit sed sit sit dolor enim eiusmod ad incididunt adipiscing veniam sit lorem enim minim et ipsum quis veniam et do tempor labore amet tempor sed et dolore et quis quis ut et minim do incididunt elit consectetur et enim sed magna ut veniam minim veniam dolor aliqua quis aliqua sit dolor tempor consectetur magna amet ut dolor dolor minim ad ipsum amet do incididunt elit veniam minim minim eiusmod labore consectetur dolore do sit amet magna ut sit eiusmod dolore elit veniam dolore sed consectetur consectetur labore veniam elit incididunt tempor aliqua quis amet labore labore quis lorem enim incididunt quis consectetur incididunt dolore ipsum et sed incididunt sed veniam quis ut veniam ad et tempor magna eiusmod veniam quis minim dolor quis elit magna enim adipiscing incididunt minim incididunt ad lorem eiusmod labore dolore veniam labore ad consectetur sit lorem incididunt adipiscing quis aliqua enim incididunt adipiscing sit incididunt magna adipiscing sed quis aliqua aliqua adipiscing et enim amet lorem enim minim ut et sed dolore aliqua consectetur labore veniam adipiscing dolor tempor lorem et magna minim minim dolor aliqua et minim eiusmod labore sed dolore labore lorem dolor enim tempor consectetur incididunt sed minim ad quis amet ipsum consectetur et incididunt labore minim do amet lorem do magna labore lorem tempor ipsum magna incididunt aliqua labore adipiscing minim do et ad amet et veniam magna veniam do dolor sed eiusmod do eiusmod ad do ad ad incididunt dolore dolor dolore ad adipiscing incididunt enim dolore amet dolore ad dolor do ipsum elit labore magna elit dolore sed ipsum sit sit minim incididunt tempor adipiscing eiusmod tempor dolor eiusmod labore tempor consectetur et labore do labore amet veniam labore ad adipiscing sed eiusmod consectetur sit elit et adipiscing minim tempor consectetur tempor amet amet elit sed magna ad incididunt incididunt quis eiusmod sed quis enim dolore aliqua veniam quis eiusmod quis incididunt veniam veniam ad veniam do magna enim ad minim dolor tempor do incididunt et consectetur sed tempor labore et dolor consectetur eiusmod incididunt amet lorem sit tempor consectetur tempor dolor quis ad ut lorem magna eiusmod elit enim incididunt magna do et ad amet tempor eiusmod adipiscing et sit amet adipiscing eiusmod sed amet ut tempor sed dolor eiusmod adipiscing elit veniam elit quis enim ipsum eiusmod tempor ad enim ipsum amet consectetur dolor ut labore sed amet eiusmod dolore aliqua sit eiusmod ad veniam enim incididunt elit ipsum incididunt et et enim eiusmod magna enim enim dolor aliqua dolore magna minim et incididunt veniam labore consectetur ut incididunt dolore labore ipsum sit labore aliqua amet sit minim dolore consectetur dolor incididunt do labore veniam lorem sed sit minim tempor elit consectetur lorem amet ut minim dolor eiusmod ad labore ipsum et elit dolor et amet magna lorem amet veniam dolore magna ipsum ipsum adipiscing magna lorem dolore eiusmod minim dolore elit amet tempor et lorem amet magna sit elit sit labore adipiscing ipsum enim adipiscing ad incididunt eiusmod enim ad incididunt veniam dolore dolore minim consectetur dolore sit amet ad adipiscing consectetur incididunt adipiscing do eiusmod ut amet ut amet incididunt eiusmod do sit magna sit et sed do dolore et sed elit ut veniam amet veniam magna minim sit lorem enim magna adipiscing adipiscing adipiscing incididunt aliqua ipsum ad amet ad lorem quis sed veniam quis et magna ipsum quis elit amet enim eiusmod ipsum veniam adipiscing sit amet ad veniam magna consectetur dolor minim labore ad do adipiscing consectetur eiusmod veniam sed dolore aliqua dolor ut ut minim quis ipsum labore do minim sit ad veniam quis sed lorem adipiscing ut eiusmod sed magna quis incididunt aliqua dolore quis adipiscing ut amet veniam consectetur labore labore tempor incididunt et enim sed enim adipiscing aliqua et labore adipiscing et aliqua eiusmod do dolor consectetur tempor enim ad et elit enim ad minim aliqua amet minim do adipiscing magna do sit lorem lorem adipiscing eiusmod ipsum eiusmod magna sed quis minim eiusmod labore dolor ut et veniam lorem do aliqua aliqua amet adipiscing amet consectetur enim incididunt quis dolor ad aliqua labore sed ad dolor et et elit amet aliqua do elit adipiscing enim veniam eiusmod aliqua enim veniam incididunt dolore ut elit ad adipiscing magna ipsum sed minim elit amet enim quis incididunt ut sit labore incididunt incididunt et incididunt do adipiscing elit elit ipsum magna dolore dolor enim magna minim lorem ipsum incididunt veniam ut incididunt elit dolore sed sit tempor dolore tempor dolore et aliqua dolor veniam labore quis veniam elit sed lorem lorem et ipsum amet ad amet adipiscing eiusmod elit magna ipsum enim amet ad do sit ad magna magna dolor minim minim amet ut veniam amet ipsum do dolore minim sed et ipsum magna tempor eiusmod minim sit enim tempor sit enim tempor tempor ad sed et do dolore enim amet lorem ipsum eiusmod ut ad lorem tempor minim magna veniam ipsum minim dolor veniam magna dolore enim ut ut ut elit consectetur consectetur enim ipsum lorem aliqua quis tempor minim consectetur do lorem ipsum elit aliqua elit incididunt dolor tempor sit enim dolor elit elit magna adipiscing sit lorem veniam incididunt dolor dolore sed aliqua ad elit ipsum dolore dolore dolore incididunt ut amet amet ut amet labore quis tempor ipsum aliqua consectetur dolore labore ut enim ad labore consectetur et enim amet tempor amet lorem sed veniam consectetur amet ad ut aliqua ad sed labore et labore adipiscing ut ut sed elit tempor ad ipsum incididunt enim lorem ut do lorem magna et aliqua sed minim sed elit labore veniam labore tempor dolore enim labore minim elit magna magna consectetur labore do tempor ut sit dolore minim elit quis ad minim incididunt sit ut enim labore enim dolore labore dolor quis incididunt labore enim veniam veniam quis tempor magna tempor consectetur amet elit minim ad consectetur ut labore et veniam quis consectetur ut sed eiusmod aliqua incididunt do ad veniam sed minim eiusmod minim lorem incididunt aliqua ipsum adipiscing labore sit sit lorem tempor eiusmod enim eiusmod quis incididunt consectetur quis eiusmod dolor dolore enim et incididunt enim elit labore sit enim lorem tempor lorem do et amet veniam veniam ipsum lorem eiusmod incididunt et ad enim lorem et ad aliqua adipiscing elit quis enim eiusmod consectetur eiusmod do incididunt aliqua enim quis et labore sed dolor dolore adipiscing aliqua tempor elit tempor tempor consectetur elit magna ad quis enim elit adipiscing aliqua labore elit incididunt sed aliqua adipiscing dolore consectetur lorem incididunt et quis tempor ad consectetur minim adipiscing quis veniam enim consectetur et enim lorem quis amet adipiscing veniam adipiscing lorem enim dolor labore ad adipiscing quis consectetur sed incididunt enim lorem lorem tempor sit do ipsum aliqua tempor et incididunt sit dolor labore consectetur amet labore ad dolor sed amet et magna dolor dolore veniam do do lorem magna magna adipiscing dolor ut amet consectetur aliqua do labore adipiscing ipsum veniam eiusmod labore ipsum amet elit enim tempor ad do veniam ad sit adipiscing amet elit lorem eiusmod sit sed sit tempor dolor dolore veniam enim amet tempor amet ut quis quis adipiscing do magna lorem lorem eiusmod ad amet quis dolor magna ipsum amet quis et ut minim tempor et minim do do dolor ad amet quis sit lorem dolor amet tempor veniam amet do aliqua minim magna amet dolor elit aliqua quis quis elit eiusmod dolor consectetur minim ut adipiscing ut ad minim ipsum eiusmod ut et ad magna tempor consectetur dolore sit minim ipsum amet minim incididunt amet ut quis elit ad sed enim magna veniam sit dolor dolore aliqua quis enim consectetur adipiscing minim enim minim veniam dolor magna eiusmod amet dolore elit elit minim labore minim incididunt labore sit eiusmod elit ipsum tempor sit incididunt lorem aliqua sit quis sed tempor magna elit eiusmod ut elit ipsum amet consectetur amet quis consectetur incididunt amet sit ad do ut labore lorem sit ut lorem veniam sit ipsum adipiscing ut lorem adipiscing incididunt eiusmod amet consectetur sed sed sed aliqua elit dolore do do aliqua ut et et incididunt do incididunt consectetur ad ut do aliqua labore dolore ut aliqua veniam eiusmod magna dolor aliqua lorem sed ut aliqua labore quis dolor dolore magna et dolore ad sed consectetur adipiscing tempor sit quis elit adipiscing ut amet dolore adipiscing dolore incididunt consectetur dolor do labore amet dolore dolor quis tempor minim consectetur elit magna ipsum ut magna ipsum incididunt magna do elit consectetur quis incididunt elit adipiscing aliqua elit dolore sed magna sed enim sed consectetur lorem ad enim incididunt dolore veniam ipsum ipsum et adipiscing magna eiusmod do quis ad ipsum labore eiusmod consectetur labore ipsum adipiscing tempor sed quis labore sed minim dolore sed ipsum quis eiusmod lorem ut ad elit consectetur incididunt veniam minim et amet ut veniam do dolor minim consectetur ipsum ut dolor magna aliqua amet lorem lorem tempor adipiscing minim ad aliqua magna et ut minim enim veniam aliqua eiusmod sit lorem dolor adipiscing lorem quis dolore dolore lorem amet lorem do sed sit lorem ad aliqua lorem tempor sed sit incididunt minim minim ad minim consectetur ut lorem elit adipiscing consectetur enim veniam ut magna ipsum aliqua sed sit elit aliqua elit ipsum adipiscing minim elit minim consectetur enim lorem consectetur sit aliqua adipiscing ad elit do eiusmod veniam dolor ad adipiscing ad amet aliqua elit sit tempor et veniam consectetur adipiscing adipiscing incididunt do lorem dolor elit aliqua do ad tempor do magna do minim ipsum do dolor et dolore minim amet enim aliqua tempor do enim ut enim ut ad sit magna consectetur ad enim elit enim ad amet dolor labore dolor enim amet et lorem amet incididunt adipiscing et tempor dolore ad amet ut adipiscing sit eiusmod eiusmod do ut amet elit aliqua sed et eiusmod lorem quis ipsum lorem sit adipiscing do ipsum labore dolore elit adipiscing elit dolor ipsum lorem sit incididunt sit ut aliqua aliqua ut ad dolore lorem quis lorem consectetur sit adipiscing elit dolor veniam sed dolore magna sed sit consectetur enim do adipiscing quis enim dolor consectetur minim lorem lorem minim sit ad labore labore lorem ad aliqua lorem elit veniam eiusmod minim dolor elit veniam sed sit consectetur adipiscing dolor ut sed et sit sed enim enim amet eiusmod incididunt elit aliqua elit adipiscing et labore sit et incididunt ad eiusmod veniam ipsum labore ipsum ut labore quis minim veniam veniam elit magna ut magna ad ipsum minim dolore incididunt sed sit labore do do quis ut elit minim consectetur veniam elit sit lorem sed magna veniam amet incididunt ut aliqua quis ipsum minim minim amet veniam ad ipsum ipsum adipiscing ipsum veniam sed magna consectetur consectetur et tempor minim dolor veniam ut magna enim consectetur sit ut do elit lorem ad adipiscing sed ipsum elit adipiscing minim et dolore quis et labore lorem ipsum do ut dolore enim ad dolore adipiscing ut sit labore labore dolore minim consectetur magna consectetur do enim amet veniam magna sit dolore consectetur lorem eiusmod quis ipsum enim quis enim sed ad incididunt eiusmod enim enim aliqua ad amet dolor aliqua elit amet aliqua eiusmod veniam eiusmod ipsum adipiscing enim ipsum adipiscing tempor ipsum lorem minim et incididunt enim amet veniam ipsum enim ad ut dolor eiusmod consectetur dolore dolore elit sit sed sed eiusmod et quis minim sed labore quis adipiscing eiusmod magna lorem enim lorem quis enim quis aliqua sit ut ut ut enim labore aliqua sed ut lorem sed lorem lorem consectetur enim ad tempor adipiscing adipiscing dolor magna tempor do eiusmod ipsum sit eiusmod minim quis do consectetur minim amet sed sed et dolore magna magna sed do aliqua eiusmod dolore aliqua magna labore ut incididunt dolor incididunt dolor dolore amet sit eiusmod incididunt et minim aliqua dolor quis ut incididunt labore enim sed et lorem magna et magna tempor ad ut quis adipiscing enim dolor lorem consectetur minim sed veniam lorem do ipsum minim ut do incididunt veniam elit minim ut dolore sed enim incididunt incididunt sed lorem sed consectetur do lorem adipiscing lorem adipiscing consectetur eiusmod sed aliqua adipiscing minim veniam eiusmod incididunt aliqua amet lorem ipsum sit elit ut minim sit incididunt do amet sed adipiscing incididunt ipsum minim ut ut eiusmod dolor adipiscing labore amet veniam magna ad labore incididunt et veniam sed enim elit ut labore elit ad et ad sed adipiscing minim et incididunt lorem sed dolore sed do eiusmod adipiscing aliqua ad ut ipsum dolor sit consectetur consectetur eiusmod do magna elit ad adipiscing ad dolor sit dolore sed dolore labore eiusmod dolore amet adipiscing elit sit do dolore et et labore ad veniam dolore eiusmod do amet labore labore lorem ut dolore incididunt consectetur elit enim eiusmod dolore ut do labore amet sed tempor ad ut minim minim minim ipsum consectetur tempor veniam minim dolore aliqua adipiscing aliqua aliqua ut eiusmod magna adipiscing sed minim amet minim do aliqua enim elit ad aliqua dolor lorem minim magna veniam quis et lorem magna adipiscing lorem dolor magna ad eiusmod consectetur incididunt consectetur enim aliqua do ut eiusmod amet aliqua ad lorem minim dolore aliqua labore lorem incididunt quis consectetur consectetur elit minim sit labore adipiscing eiusmod elit dolor do ipsum sit sed ut sit veniam sit adipiscing veniam incididunt adipiscing ut aliqua veniam elit et ad ad adipiscing dolor amet magna dolor tempor quis adipiscing adipiscing tempor veniam et tempor sed consectetur labore ad dolore sed magna aliqua dolor lorem dolore magna amet eiusmod magna elit elit elit elit ut veniam ut do amet minim do minim lorem ut dolore minim ad labore amet do eiusmod dolore aliqua sit amet dolore veniam minim incididunt enim tempor ipsum veniam ut et incididunt quis dolor dolor et labore et veniam ipsum veniam minim consectetur elit quis tempor incididunt eiusmod minim enim eiusmod enim sed aliqua minim quis veniam magna labore aliqua ut ut veniam quis amet dolore enim amet adipiscing quis adipiscing ad adipiscing sit ipsum ipsum ut dolor incididunt magna labore consectetur eiusmod incididunt amet lorem lorem incididunt ut enim amet consectetur magna do amet sit sit consectetur adipiscing adipiscing veniam adipiscing sit eiusmod ipsum elit eiusmod ipsum dolore quis minim magna amet eiusmod ipsum dolore veniam do elit amet magna elit quis elit veniam labore ipsum magna dolore incididunt do veniam et dolore ipsum consectetur aliqua elit dolor tempor lorem et enim lorem ut sit elit incididunt eiusmod enim veniam et sit elit labore dolor enim ad dolore magna elit amet lorem et labore eiusmod enim labore et labore do amet enim enim do aliqua et do dolor sit lorem ut quis sit eiusmod ipsum do lorem adipiscing aliqua amet tempor ad ad tempor ad amet dolor sed enim sit dolor amet aliqua elit lorem do aliqua consectetur veniam ipsum incididunt tempor dolor do et ad labore consectetur aliqua ipsum elit incididunt quis sed veniam eiusmod consectetur aliqua aliqua labore eiusmod dolor do do adipiscing dolor dolor eiusmod dolore et magna lorem minim amet ad tempor tempor do et dolore lorem amet minim aliqua tempor veniam magna aliqua consectetur sed sed sed amet minim lorem ut amet incididunt incididunt quis sit dolore ad tempor ut eiusmod sed eiusmod et ipsum do incididunt do dolor dolor ad quis et amet consectetur minim sit ut magna sed dolore incididunt lorem ad labore quis ipsum amet adipiscing lorem amet labore quis et lorem elit quis magna quis quis quis ad quis elit ad dolore ad dolor veniam do aliqua amet adipiscing magna labore adipiscing do et quis dolor amet et ipsum elit sed sed enim lorem ipsum aliqua eiusmod dolore sed incididunt labore dolore veniam ad lorem tempor enim minim tempor elit sed lorem ipsum tempor incididunt lorem enim dolore elit labore sit ad labore lorem veniam magna labore eiusmod consectetur labore eiusmod enim do veniam enim eiusmod sed amet veniam magna magna incididunt do ipsum sed ad eiusmod dolore veniam magna ut magna do minim consectetur dolor minim amet do aliqua incididunt tempor magna et dolor enim sed minim labore do sit minim magna aliqua veniam sed do sit enim magna quis enim sit enim ut quis ad ad ipsum lorem aliqua ut ad adipiscing eiusmod adipiscing labore magna do veniam labore eiusmod tempor incididunt eiusmod lorem magna dolore ad minim et elit et quis adipiscing ad magna labore minim enim enim quis enim adipiscing sit incididunt magna amet incididunt aliqua incididunt consectetur aliqua minim incididunt eiusmod lorem incididunt ut ad do ipsum do sit elit magna quis sed do lorem quis ipsum eiusmod lorem tempor do veniam veniam incididunt veniam ut elit amet aliqua ipsum ad dolore et aliqua labore consectetur adipiscing ut et magna magna ipsum sit dolor ut do ut eiusmod sed magna adipiscing ad magna quis ut dolor ad sit minim adipiscing lorem veniam labore dolor ut enim consectetur minim enim aliqua minim dolor veniam tempor ut magna et enim aliqua incididunt sed lorem quis adipiscing lorem et magna ipsum amet aliqua dolore incididunt et ad ut tempor consectetur quis adipiscing do incididunt ut ipsum ut ut adipiscing incididunt dolore tempor magna amet et magna dolore adipiscing veniam eiusmod enim dolor tempor aliqua ipsum sit amet minim aliqua consectetur labore sed labore sit sit aliqua eiusmod lorem et aliqua do quis sed dolore adipiscing adipiscing do labore consectetur adipiscing quis incididunt quis sed magna veniam aliqua veniam enim ipsum eiusmod lorem tempor do ut do ad labore minim tempor veniam sed adipiscing ad aliqua incididunt consectetur do labore ad tempor sit labore incididunt amet quis tempor quis ad magna et lorem adipiscing quis do aliqua ipsum ad lorem minim dolor enim amet dolor lorem lorem adipiscing quis consectetur dolore lorem tempor magna tempor dolor dolor et minim veniam minim elit ut quis sed elit enim ut ipsum consectetur veniam tempor elit do ut incididunt ut adipiscing consectetur incididunt et sed consectetur incididunt quis incididunt elit adipiscing sit magna ut et sit ipsum sit elit et consectetur ad elit amet tempor ad ipsum tempor lorem magna do elit adipiscing do consectetur lorem sed et enim dolore aliqua et minim consectetur ad ut quis consectetur dolor ipsum adipiscing incididunt labore dolor eiusmod amet dolor enim ad do incididunt sed amet magna dolore sit magna incididunt consectetur tempor amet dolore eiusmod incididunt aliqua dolor consectetur sit veniam sed labore aliqua elit elit incididunt sed consectetur adipiscing incididunt ad dolore ad eiusmod dolor do veniam et ad sed lorem ut do do sed minim lorem minim veniam sed sed aliqua et ipsum enim minim minim incididunt adipiscing veniam dolor enim ut eiusmod sit sed dolore tempor dolore sit eiusmod do elit amet dolor aliqua quis magna dolor ut amet consectetur amet ut et aliqua tempor magna dolor minim ut eiusmod dolor enim dolor labore consectetur ut elit veniam magna eiusmod ut ad dolore et ad ad elit tempor minim et enim lorem sed dolore adipiscing ipsum eiusmod veniam aliqua do magna elit tempor ipsum labore do dolor tempor ut ad veniam enim consectetur amet sit dolore adipiscing elit dolore veniam aliqua elit labore do minim magna quis magna tempor adipiscing aliqua ipsum consectetur quis eiusmod magna tempor eiusmod ad ut tempor enim et consectetur aliqua veniam adipiscing do ut tempor adipiscing quis aliqua sed enim adipiscing dolore labore magna labore minim elit et ad enim eiusmod enim do eiusmod magna aliqua tempor lorem labore minim ut eiusmod lorem sed adipiscing incididunt lorem dolore adipiscing ipsum et veniam consectetur tempor labore incididunt aliqua et quis magna lorem adipiscing elit eiusmod sit adipiscing consectetur elit veniam elit do veniam sit magna amet enim enim ipsum amet lorem quis labore aliqua ipsum ad aliqua elit tempor tempor enim aliqua ipsum minim sed enim adipiscing ad aliqua magna et elit consectetur eiusmod incididunt magna elit quis minim quis amet minim consectetur ad do elit labore sed sed elit ut tempor adipiscing ut quis dolor dolor sit consectetur tempor incididunt ut labore consectetur lorem labore amet et ut dolor ut magna consectetur do elit veniam veniam sit dolor dolore elit aliqua et sed elit labore aliqua tempor eiusmod ut ut adipiscing consectetur sit labore lorem eiusmod do incididunt dolor et labore amet amet do adipiscing labore adipiscing ut eiusmod adipiscing elit veniam adipiscing dolore sed aliqua ut et dolore veniam et minim elit amet ipsum elit ad sed ipsum aliqua incididunt et tempor quis eiusmod elit eiusmod quis labore enim aliqua minim magna dolore eiusmod eiusmod consectetur labore quis ad incididunt adipiscing dolor labore do eiusmod consectetur et sit dolore enim lorem sit tempor eiusmod ut eiusmod sed consectetur magna sit consectetur eiusmod enim elit do quis do magna incididunt quis enim aliqua eiusmod labore elit ad sit dolore dolore tempor eiusmod aliqua sit quis dolor adipiscing eiusmod magna sed quis dolor ad labore sed do dolor aliqua minim ad sit consectetur lorem enim sit ad labore elit labore quis ad minim sit tempor et enim dolore veniam sed dolor sed amet minim eiusmod ut incididunt adipiscing et do aliqua ut sit ut incididunt dolor lorem minim et do adipiscing ipsum lorem do consectetur magna dolore eiusmod amet et tempor adipiscing magna sed lorem veniam aliqua minim magna dolore tempor eiusmod minim ut incididunt ut do ut lorem elit dolor do minim enim lorem do ad et incididunt et ad et sit tempor tempor aliqua veniam dolore elit minim tempor sit amet ad incididunt enim tempor ad et ipsum magna do dolor consectetur do dolore quis consectetur ad amet ipsum consectetur minim minim dolor adipiscing lorem sed magna ad tempor ipsum do dolor sit sed ut eiusmod adipiscing ut sit consectetur sit ipsum et amet et aliqua do incididunt et veniam sit elit incididunt tempor ipsum quis veniam tempor do minim labore veniam lorem lorem et sit quis consectetur et labore magna ut lorem elit quis ipsum dolor ad veniam et ad quis lorem sed enim et sed elit ut sit elit minim quis elit quis magna enim enim sit adipiscing dolor ipsum dolore aliqua enim minim do sit magna sed quis enim magna aliqua lorem lorem sed enim dolor dolore elit ad incididunt ad sed adipiscing amet et ut ipsum minim quis ut enim dolor elit sed sit do veniam do amet consectetur consectetur ipsum sit amet labore enim et amet aliqua quis quis lorem adipiscing aliqua enim aliqua adipiscing veniam ut magna adipiscing amet elit eiusmod veniam eiusmod tempor do dolor incididunt quis sit minim amet sed sit eiusmod incididunt et labore do dolor tempor tempor dolor et sit tempor enim magna et ad incididunt enim ad incididunt dolor aliqua enim incididunt adipiscing minim dolor sed lorem adipiscing quis et dolore aliqua elit amet eiusmod tempor magna magna tempor aliqua lorem eiusmod consectetur amet dolor eiusmod quis adipiscing veniam lorem quis quis ut enim veniam dolor dolore sed aliqua ut ut do veniam enim tempor magna eiusmod do et ut ut labore dolor sit eiusmod dolor ad dolore ipsum magna ad ipsum labore consectetur amet tempor adipiscing dolor dolor lorem sit adipiscing amet et ipsum dolore amet dolore dolore magna magna ut labore dolore sit incididunt magna adipiscing labore aliqua enim minim et dolor tempor magna ipsum elit labore ut dolor lorem magna elit labore aliqua dolor tempor magna labore minim sed quis do do ad dolor amet ipsum consectetur dolor ut adipiscing lorem sed labore ad sed lorem veniam ad magna consectetur consectetur sit ut sit lorem lorem labore do sit eiusmod sit sed labore tempor dolore consectetur sed amet minim do lorem elit veniam do ut lorem veniam et sit dolor elit ipsum ut ut labore sed elit dolor sit do minim magna et do quis incididunt quis labore et adipiscing dolor quis dolor consectetur incididunt tempor veniam lorem adipiscing dolore ut quis aliqua enim labore et lorem adipiscing dolor magna veniam enim amet eiusmod aliqua tempor labore consectetur incididunt et et tempor tempor aliqua ad eiusmod consectetur et lorem veniam do minim dolore lorem minim ad veniam sed adipiscing ipsum ad lorem ad do incididunt quis sit aliqua elit elit do ipsum dolor incididunt amet amet magna sit do magna sed minim quis ipsum aliqua minim elit labore lorem quis et veniam dolore labore consectetur minim magna amet lorem incididunt consectetur do eiusmod amet et tempor ad sed magna sed sit enim consectetur veniam quis eiusmod dolore tempor ad ad incididunt dolor quis ad tempor sed lorem elit dolor elit lorem ad adipiscing sit sed incididunt sed dolor adipiscing sit amet amet elit elit do lorem enim aliqua dolore dolore veniam minim enim ut eiusmod magna dolore amet do sed et incididunt ut labore aliqua incididunt sit labore tempor ut veniam dolor et magna ipsum consectetur aliqua elit ad dolore enim dolore sed magna ipsum amet dolore sed ipsum minim dolor labore do ut labore sed magna dolor do amet ut incididunt et quis minim elit magna consectetur veniam enim aliqua veniam lorem elit ut ut veniam sed do lorem do veniam dolore enim et elit amet amet quis adipiscing aliqua do veniam minim sed ad magna eiusmod tempor dolore eiusmod dolor labore eiusmod incididunt aliqua sed labore dolor dolor enim amet elit minim dolor et amet consectetur tempor adipiscing eiusmod quis enim ipsum eiusmod sit adipiscing amet et enim dolore incididunt dolore ipsum incididunt labore magna do veniam tempor elit veniam do ut lorem veniam ipsum incididunt sit quis veniam quis quis dolore ipsum sit minim ad veniam lorem consectetur quis ad tempor ut amet consectetur enim adipiscing consectetur lorem amet dolore dolore tempor labore dolore tempor amet magna dolore tempor do amet tempor veniam lorem aliqua veniam sit amet dolor veniam elit aliqua sed enim eiusmod incididunt labore ipsum dolor enim labore tempor et dolor aliqua et et sed enim ut magna minim consectetur consectetur amet ad quis ad consectetur ad sed veniam dolor ad sed eiusmod eiusmod adipiscing labore sed sed labore adipiscing quis quis incididunt sed quis ipsum aliqua consectetur sit et aliqua veniam quis dolor incididunt eiusmod quis et dolor enim adipiscing do magna et dolore quis dolor ipsum sed adipiscing enim aliqua sit ipsum lorem ipsum minim et incididunt ad labore aliqua dolor do ut eiusmod ipsum incididunt ad sed labore veniam magna elit sit ad sit ipsum quis ad et veniam elit quis aliqua enim sed incididunt enim tempor labore quis do sed incididunt enim dolor labore incididunt adipiscing ad labore magna ad tempor lorem ut elit quis magna dolor dolor labore aliqua tempor ipsum sit veniam adipiscing sit lorem ad minim magna et lorem et consectetur ut veniam sit sed enim sed magna veniam adipiscing labore sed enim tempor elit veniam ut eiusmod tempor minim veniam sit enim adipiscing minim lorem et elit quis elit sed incididunt lorem ut labore ipsum ut elit aliqua et ad eiusmod enim ipsum magna minim dolore adipiscing sit ipsum ipsum et adipiscing sit eiusmod ad tempor consectetur ut do elit ipsum adipiscing labore lorem aliqua ad labore ipsum labore veniam veniam ad magna ut tempor labore sed veniam dolor aliqua do dolor adipiscing tempor elit eiusmod sit lorem adipiscing dolore ut elit sit minim quis et do lorem quis magna veniam quis aliqua tempor eiusmod minim consectetur ipsum consectetur magna quis ad tempor consectetur dolor quis lorem consectetur et eiusmod do veniam dolor do sit do dolor lorem quis quis incididunt adipiscing aliqua labore consectetur amet elit labore quis dolore sit tempor dolor veniam sit adipiscing aliqua do labore dolore do quis dolor veniam et enim ad dolore ad et ad ut amet do ipsum lorem consectetur magna enim lorem adipiscing incididunt labore tempor amet incididunt magna do sit amet eiusmod eiusmod ipsum ipsum consectetur do do quis elit minim adipiscing ut ipsum do incididunt veniam dolor amet elit sed veniam sed ut veniam amet lorem minim tempor sit lorem aliqua sed adipiscing quis incididunt minim minim enim minim et adipiscing ad quis sed veniam elit incididunt eiusmod ipsum aliqua labore et veniam magna labore eiusmod minim et veniam quis incididunt do sit incididunt lorem dolor ipsum ipsum dolor veniam tempor tempor lorem aliqua et lorem dolor do et sit incididunt et lorem veniam ipsum dolor tempor lorem elit minim consectetur tempor dolor consectetur sit eiusmod magna eiusmod dolor incididunt minim amet do amet tempor veniam ipsum magna minim quis tempor aliqua minim magna labore ipsum quis ipsum quis amet aliqua eiusmod amet minim ad amet magna veniam ipsum eiusmod minim dolore quis ut minim sed adipiscing ad dolore minim et elit do dolor minim et labore veniam sed minim incididunt amet magna labore consectetur et sed tempor minim adipiscing veniam eiusmod tempor et veniam quis ut ad consectetur elit ad ipsum do adipiscing sit minim ut labore labore lorem consectetur quis dolor dolor lorem et do consectetur tempor adipiscing dolore do do sed do eiusmod quis sit lorem labore magna eiusmod et incididunt ad sit do et adipiscing minim sit lorem ut elit do et minim dolor ut ipsum sit dolore veniam dolor quis eiusmod tempor consectetur quis enim tempor sed tempor do adipiscing amet lorem elit ipsum sit magna ut et veniam tempor enim sed elit labore aliqua sit dolor veniam minim ipsum do veniam consectetur veniam labore ipsum incididunt veniam et sit tempor eiusmod lorem ut labore quis ut adipiscing incididunt dolor incididunt amet eiusmod elit enim ut minim consectetur quis aliqua et amet et aliqua ut et incididunt labore labore quis ad do et lorem do ad amet amet consectetur amet ad eiusmod do et elit lorem sit do sed et lorem incididunt veniam ipsum do enim veniam incididunt minim minim magna elit sed dolore enim amet incididunt do enim ut sit quis minim magna quis quis dolore incididunt veniam aliqua sit tempor sit dolore veniam dolore dolore sit magna quis ut sed dolor dolor quis ut sit eiusmod incididunt eiusmod et minim et lorem enim lorem dolor et elit et ut adipiscing magna ad aliqua magna ut enim sit incididunt eiusmod tempor minim eiusmod enim et aliqua ad elit quis veniam sed ipsum amet magna et et magna tempor elit quis dolor sed lorem sit labore adipiscing labore consectetur elit ut lorem ipsum sed eiusmod do lorem tempor ut sit ad incididunt dolore adipiscing sed veniam dolore lorem magna tempor adipiscing sit amet dolor amet elit eiusmod incididunt veniam adipiscing amet veniam lorem dolore sed sed tempor dolore incididunt incididunt et aliqua consectetur amet sit do tempor adipiscing elit minim sed elit et tempor sit dolore sit enim ad dolore ad tempor adipiscing do minim sed veniam sit quis adipiscing quis quis amet quis sed sit adipiscing enim labore dolore minim magna magna aliqua lorem amet dolor eiusmod dolor et tempor dolor amet tempor amet ad adipiscing minim et quis magna adipiscing labore veniam enim ut ad do magna eiusmod amet ut ipsum elit ad dolor ipsum sed enim quis do ipsum minim do ipsum et et et ad sed do sed sed quis ad aliqua ut incididunt enim labore quis enim adipiscing enim tempor minim ipsum amet ipsum et ut aliqua amet sed dolore consectetur elit incididunt quis eiusmod amet dolor veniam enim sit elit et consectetur enim eiusmod elit quis magna sit veniam sed elit aliqua minim adipiscing magna ipsum et ut ut incididunt et amet dolor sit minim ad sed sit sit ut incididunt amet incididunt ipsum tempor ad quis labore sed eiusmod dolor adipiscing elit enim dolor ipsum elit amet minim incididunt tempor ad consectetur dolor elit ut quis minim enim amet dolore elit veniam ad quis lorem ipsum consectetur minim veniam do incididunt et lorem consectetur do incididunt do do sed amet ut do amet consectetur tempor magna adipiscing tempor incididunt ut dolore enim aliqua enim magna lorem enim et elit ipsum dolore ad veniam elit eiusmod quis ipsum elit enim incididunt ut et adipiscing do lorem labore magna ut dolor sed aliqua sit ut elit eiusmod ipsum consectetur do quis incididunt amet incididunt incididunt ipsum veniam ad incididunt dolor sit consectetur minim eiusmod ut incididunt magna quis lorem sed do veniam tempor consectetur sed tempor enim do amet sed minim veniam lorem eiusmod enim tempor incididunt consectetur sed enim incididunt tempor dolore sit aliqua do quis quis do ut ad quis sed adipiscing quis sit enim magna sed eiusmod quis adipiscing lorem sed minim ipsum ipsum lorem veniam consectetur et tempor consectetur amet amet enim magna enim adipiscing dolor dolore incididunt magna incididunt enim lorem et ipsum aliqua consectetur labore adipiscing incididunt quis lorem ad ad elit ipsum incididunt lorem sed eiusmod ad tempor incididunt ut amet labore amet ipsum adipiscing veniam incididunt amet dolore adipiscing lorem dolore quis enim sit sed ut enim lorem labore amet consectetur dolor eiusmod sed ipsum aliqua tempor magna magna elit sit veniam quis do minim ipsum adipiscing sit aliqua ipsum enim do ad enim eiusmod ad magna enim ut magna quis dolor amet adipiscing veniam labore lorem ut enim sed sed eiusmod enim elit dolore ut consectetur quis labore elit elit dolor elit et tempor do veniam labore ut elit ipsum lorem adipiscing amet enim ad quis elit ad do minim sit enim et dolore ut do eiusmod aliqua eiusmod amet consectetur et eiusmod tempor quis consectetur dolor ut magna do dolore sed minim veniam ut et veniam elit magna do incididunt lorem sit dolore aliqua dolore et dolor minim sed ipsum consectetur aliqua minim magna do sit veniam adipiscing adipiscing veniam adipiscing et et do tempor veniam elit adipiscing aliqua magna dolor elit do dolor veniam elit dolor sed sed enim amet ut dolore et dolor veniam sit et tempor magna et adipiscing consectetur labore tempor labore veniam incididunt minim quis do veniam labore quis enim quis labore aliqua ad consectetur quis labore quis quis enim dolore enim minim magna veniam labore sed et labore veniam ut minim et lorem labore ipsum lorem do ad ipsum quis eiusmod enim et amet magna ad adipiscing elit eiusmod amet enim sed amet labore ipsum tempor dolore dolore dolore dolor ad elit dolore aliqua aliqua quis sit et ut labore tempor dolore dolore tempor dolor adipiscing do amet sit tempor dolore labore dolore do aliqua ut adipiscing aliqua tempor enim magna labore elit et sit et dolor tempor tempor adipiscing quis sed magna tempor ipsum dolor sed amet quis do sed quis elit elit incididunt quis veniam sit ad labore consectetur et et ipsum ipsum lorem ad sed quis quis adipiscing adipiscing lorem do minim elit tempor lorem elit quis et adipiscing ipsum do ipsum ipsum eiusmod ut aliqua tempor enim aliqua amet magna quis eiusmod incididunt tempor dolor enim sit eiusmod ad sed do consectetur consectetur dolore ut incididunt eiusmod dolor enim enim magna do veniam amet incididunt sit dolore quis incididunt lorem veniam aliqua et consectetur ut et magna ipsum ad veniam et do sed et et lorem ad minim ad lorem enim lorem ut magna amet incididunt sit quis dolore adipiscing minim lorem sit enim ipsum dolore ut magna lorem do sed quis eiusmod magna amet ut magna dolor amet quis tempor ipsum veniam eiusmod magna adipiscing sit quis magna adipiscing minim sit sit labore quis enim quis do do sed elit dolor eiusmod enim ad consectetur minim elit veniam minim et incididunt sed labore amet consectetur sed do eiusmod tempor sed dolore do ut enim tempor aliqua tempor eiusmod do consectetur dolor veniam adipiscing ad tempor dolor sit minim labore aliqua dolor ipsum ad incididunt tempor labore et quis elit enim labore ipsum quis eiusmod sit consectetur do do sed lorem sed adipiscing amet dolor labore dolore do sed incididunt tempor aliqua sed ad dolor ad eiusmod sit ut amet minim sed eiusmod aliqua sed dolor amet et tempor lorem incididunt ipsum et enim et ut ad ut dolor ut quis magna do incididunt consectetur tempor do elit amet quis minim aliqua magna ad veniam ut enim et dolore ad sed sed sit elit sed lorem sed ad ut et elit magna et aliqua amet eiusmod labore sit incididunt ad aliqua enim lorem sed ipsum sit veniam magna sed ipsum et quis dolore sit incididunt dolore ut enim ad ipsum veniam dolor labore et labore ipsum incididunt do et adipiscing tempor et labore quis elit minim do dolor tempor ipsum dolor quis enim ad amet do adipiscing dolore ipsum ut dolore dolor dolor do dolore consectetur eiusmod dolor et veniam dolore et lorem ad sit enim sit adipiscing quis elit dolore magna eiusmod veniam ipsum consectetur elit minim ipsum do lorem sit enim tempor ut ad amet lorem incididunt consectetur sed ipsum adipiscing enim incididunt veniam dolore enim labore consectetur consectetur ad do dolore incididunt sed aliqua ut tempor lorem enim veniam ipsum amet ipsum veniam enim enim sed ad tempor amet amet amet magna ipsum ad eiusmod amet magna amet et ad et et adipiscing incididunt amet veniam adipiscing lorem tempor consectetur do sit ad adipiscing do minim et labore incididunt adipiscing do quis do lorem amet do eiusmod sit ipsum quis lorem quis lorem do lorem incididunt eiusmod ipsum aliqua dolor dolore adipiscing elit lorem minim ut sit tempor amet lorem labore elit veniam quis do incididunt et ipsum amet ad ut quis ad et ad minim do incididunt incididunt tempor minim veniam dolor et ipsum eiusmod ipsum adipiscing veniam consectetur eiusmod sit tempor et do amet ipsum veniam dolor minim veniam dolor sed adipiscing sed lorem labore ad elit sit magna adipiscing tempor enim minim lorem sed veniam veniam sit magna ipsum consectetur incididunt amet amet veniam veniam magna sit elit tempor sit adipiscing labore minim elit aliqua labore aliqua do enim incididunt incididunt do incididunt enim ut tempor ad elit sit ut magna elit minim minim elit incididunt eiusmod aliqua quis enim amet magna sit incididunt magna ad eiusmod lorem amet dolore aliqua aliqua tempor et magna aliqua amet ipsum dolor eiusmod adipiscing sed veniam et quis elit eiusmod lorem tempor tempor labore do incididunt elit lorem dolor amet elit magna elit dolor adipiscing magna tempor magna labore tempor ipsum amet dolor sit incididunt ad adipiscing incididunt ipsum lorem ad ut incididunt sit adipiscing et magna minim ad eiusmod sit lorem sit labore minim lorem minim ad sed et ad adipiscing quis quis enim veniam dolore eiusmod consectetur incididunt consectetur dolor sit dolor lorem sit ut incididunt labore tempor dolore dolor aliqua consectetur et ipsum sed magna quis ut amet ad do dolore quis ut minim do ipsum quis ad ipsum ipsum minim eiusmod ipsum eiusmod ut incididunt tempor minim do adipiscing sit sit consectetur dolore magna dolor aliqua sit minim sed elit ut do lorem consectetur incididunt magna amet lorem ad enim tempor sit aliqua lorem amet amet do tempor dolore quis dolore adipiscing veniam quis incididunt sit lorem ipsum labore tempor amet enim do eiusmod dolore do magna enim ad veniam adipiscing magna minim do dolore consectetur do magna dolore dolor adipiscing eiusmod ut enim amet aliqua adipiscing eiusmod ipsum dolor lorem elit quis incididunt aliqua do consectetur sed adipiscing sed quis ipsum ipsum incididunt dolor dolore dolor elit tempor veniam minim incididunt minim ipsum tempor tempor eiusmod eiusmod ipsum do aliqua consectetur aliqua aliqua ad ipsum sed ad tempor magna ipsum minim ad aliqua eiusmod ipsum et amet enim adipiscing minim incididunt sed ad elit enim amet dolore veniam do dolor adipiscing do magna dolore labore elit tempor veniam eiusmod magna veniam consectetur dolor lorem quis amet ipsum ad enim quis labore ut ipsum veniam magna lorem enim enim tempor tempor dolore ad dolor dolor lorem dolore magna ut enim ipsum et lorem labore elit do magna quis ipsum minim sit sed amet aliqua sed consectetur consectetur et ad amet adipiscing elit consectetur incididunt ad enim magna adipiscing minim minim ut dolor adipiscing veniam incididunt enim et do amet ipsum ipsum ad ipsum ipsum adipiscing lorem ut magna incididunt ut et magna dolor sit eiusmod ipsum labore magna sed et adipiscing veniam do eiusmod veniam aliqua dolore incididunt labore adipiscing consectetur eiusmod minim eiusmod veniam minim et veniam veniam sit enim sit ipsum eiusmod dolore sit ut ipsum ipsum consectetur incididunt et enim ipsum consectetur lorem elit do adipiscing consectetur labore ut incididunt et labore do ipsum labore ipsum consectetur do labore amet ad minim magna ipsum veniam magna consectetur et sed minim tempor dolor quis et quis veniam ut labore labore dolore aliqua ut sed sit labore quis dolor labore incididunt eiusmod incididunt eiusmod quis aliqua do amet tempor enim elit ad tempor ad ad sed labore ut sed magna enim elit dolor adipiscing dolore sed ut aliqua tempor do lorem dolore et sit do amet lorem eiusmod ad minim quis dolore adipiscing minim amet incididunt ad sit quis enim ad minim consectetur ut magna lorem quis do magna eiusmod consectetur ad tempor ad labore ut dolor veniam tempor enim do veniam dolore sed eiusmod labore dolore veniam minim consectetur sed consectetur labore consectetur magna ut minim enim elit quis sit sit magna labore et veniam ad consectetur adipiscing aliqua sed do amet consectetur adipiscing sed sed consectetur minim adipiscing ut dolore elit minim ut amet tempor aliqua eiusmod tempor ad ipsum ut lorem aliqua elit adipiscing aliqua incididunt do lorem do aliqua labore dolore enim adipiscing lorem eiusmod tempor dolore do consectetur veniam eiusmod consectetur veniam ad dolore labore ipsum minim adipiscing tempor et consectetur sed enim minim aliqua adipiscing dolore sit dolore do adipiscing veniam incididunt dolor incididunt sit ad consectetur magna quis incididunt magna labore incididunt do aliqua adipiscing incididunt dolor ad adipiscing consectetur ut do sed ipsum labore ad eiusmod magna eiusmod enim sit adipiscing adipiscing adipiscing quis ut ipsum dolor sit veniam do labore do dolor enim sit eiusmod labore magna dolor adipiscing lorem ad sit do veniam ut minim et incididunt do labore quis et aliqua adipiscing incididunt lorem consectetur labore enim elit ipsum sit ut eiusmod amet dolore dolor magna veniam incididunt et ut aliqua ut eiusmod magna lorem lorem ad dolore ad incididunt adipiscing minim incididunt consectetur elit magna labore ipsum aliqua ipsum labore consectetur tempor lorem et ut labore consectetur et amet incididunt elit veniam amet quis elit enim aliqua ad adipiscing do veniam aliqua incididunt enim ad ut labore sit labore magna quis consectetur ut quis lorem ut dolor ad ad dolor do aliqua minim ut quis consectetur sed veniam minim ipsum dolore eiusmod magna adipiscing aliqua et sed do veniam minim dolor dolore lorem consectetur elit ad amet elit sed magna sit tempor elit sit enim do quis enim lorem do et et dolor magna do elit et veniam dolor minim do labore lorem consectetur lorem tempor ad sed ut incididunt do veniam ad lorem magna aliqua incididunt sed dolore quis dolore minim labore quis incididunt dolor consectetur do minim lorem consectetur dolore tempor ut et aliqua lorem ipsum aliqua do ad do veniam ipsum amet ad dolore labore labore dolore sit do dolor tempor amet aliqua aliqua enim ipsum amet lorem sed magna adipiscing et quis sit ipsum tempor quis ipsum eiusmod sit ad adipiscing minim tempor do amet dolor veniam ad lorem veniam ut consectetur lorem tempor eiusmod veniam quis ipsum incididunt adipiscing adipiscing et aliqua tempor ipsum dolor et ad ipsum labore adipiscing dolor adipiscing sit ad lorem quis minim dolor do consectetur quis ut ut enim minim dolore elit elit veniam et eiusmod minim adipiscing ad et sit quis quis minim do do sed consectetur adipiscing veniam minim do adipiscing ipsum tempor ut sed veniam magna adipiscing lorem dolor dolore eiusmod elit ad veniam sit ut labore lorem adipiscing incididunt tempor aliqua et ad consectetur minim enim magna ad magna magna dolore sed sed do labore dolor quis sed incididunt elit ut sed quis enim adipiscing amet labore adipiscing incididunt sit dolore ipsum veniam sed tempor labore lorem adipiscing adipiscing ut minim sit tempor incididunt eiusmod enim ipsum consectetur ut adipiscing sed labore quis do aliqua sit elit lorem lorem quis sit eiusmod elit sit magna incididunt ipsum quis elit sed et dolore tempor ipsum elit minim aliqua magna elit magna labore consectetur labore tempor lorem veniam dolor ad sit minim veniam ipsum incididunt adipiscing tempor dolor et minim minim sed sit ipsum elit sed labore enim minim labore ut ut consectetur eiusmod labore lorem enim dolor eiusmod labore minim aliqua magna veniam et dolore ad tempor lorem ut amet quis magna ipsum minim amet dolore minim do dolore lorem ad veniam lorem amet dolor consectetur elit dolor labore ipsum do do do ipsum sit dolor consectetur enim quis dolor minim lorem eiusmod minim minim amet labore sed aliqua adipiscing et sed labore dolor dolore aliqua adipiscing sed lorem quis adipiscing magna sit enim adipiscing veniam eiusmod veniam enim enim sed ipsum tempor lorem minim amet et ad dolore magna quis labore magna do ut adipiscing sit incididunt tempor et do amet dolor sed aliqua et labore lorem veniam do minim et ipsum et tempor ad adipiscing et sed et et dolor veniam dolor et lorem adipiscing sit dolor ut sed minim magna eiusmod ad ad do tempor lorem aliqua quis tempor sit tempor enim elit ut ad ipsum enim minim minim veniam dolor eiusmod lorem elit lorem ut sed adipiscing dolore sit consectetur aliqua consectetur adipiscing do enim amet dolor tempor ad do dolor dolore sed minim tempor do sit dolore ipsum sed amet magna minim dolor incididunt veniam consectetur aliqua dolor ad incididunt quis labore labore sit minim magna veniam quis lorem sit do quis consectetur adipiscing minim eiusmod labore aliqua et incididunt amet do magna amet enim consectetur ad adipiscing elit consectetur dolore sed incididunt lorem veniam aliqua quis ipsum sit sit dolor elit dolore labore ut minim dolor eiusmod veniam eiusmod minim ad enim ut adipiscing adipiscing ipsum eiusmod elit sed enim quis magna dolore labore ipsum minim ad lorem ad sit ut ad enim adipiscing dolore adipiscing amet labore minim veniam eiusmod enim minim incididunt ut enim dolor enim incididunt labore do do elit dolor incididunt lorem adipiscing tempor incididunt tempor labore consectetur amet ipsum enim magna incididunt labore dolore consectetur ipsum sed labore ad consectetur veniam aliqua dolor sed consectetur incididunt do et ut et eiusmod aliqua incididunt incididunt elit do ipsum sit sed magna minim veniam et sed amet magna aliqua amet labore dolor sed dolor veniam incididunt do minim incididunt incididunt incididunt amet dolor ut magna aliqua dolor ipsum ut ut tempor et ad labore minim ipsum ipsum dolore ad labore enim quis dolore minim aliqua sed sed elit quis sed labore sed veniam enim quis do et labore sit sed consectetur minim ad ad tempor veniam sed do quis amet dolor elit consectetur lorem amet labore veniam magna tempor ad incididunt aliqua consectetur eiusmod ad elit labore aliqua et ut aliqua veniam magna ad labore ipsum magna consectetur incididunt enim ut consectetur quis ad incididunt dolore minim et tempor dolore adipiscing minim enim do aliqua ipsum adipiscing consectetur dolor elit consectetur ad et eiusmod et incididunt incididunt ipsum lorem eiusmod ipsum ad amet tempor enim et eiusmod incididunt dolore ipsum minim et tempor tempor dolore quis sit minim dolor ut ut eiusmod veniam eiusmod dolor sit magna amet tempor do quis quis elit sit tempor do magna adipiscing veniam labore elit elit elit quis aliqua aliqua ad ad et tempor et enim ad dolore et ipsum adipiscing sit incididunt adipiscing elit enim et veniam dolor consectetur lorem magna amet labore ad aliqua consectetur amet lorem consectetur magna adipiscing do et sed et et veniam labore lorem veniam dolor minim elit ut ut adipiscing tempor dolore ut et elit magna veniam sed lorem elit sit eiusmod labore minim enim consectetur elit elit do incididunt et eiusmod incididunt enim quis labore ut lorem enim veniam sed enim quis do aliqua minim labore dolore lorem dolor elit eiusmod et eiusmod veniam sit ad eiusmod adipiscing lorem do adipiscing veniam veniam magna eiusmod ipsum et elit elit adipiscing amet ut consectetur elit incididunt ut quis amet ad dolore ad amet do enim quis eiusmod lorem labore ut et ipsum sed ut dolor aliqua sit elit do veniam tempor ad lorem ipsum eiusmod elit veniam adipiscing dolor ut quis veniam do lorem quis ad enim sed quis elit tempor adipiscing quis veniam elit veniam dolore ut incididunt sit aliqua aliqua sed incididunt minim veniam minim adipiscing incididunt sit labore labore dolore quis amet enim enim aliqua consectetur eiusmod magna sit lorem minim eiusmod ad dolor ad do tempor lorem quis quis quis minim sed dolore ad et do et minim sit eiusmod veniam lorem aliqua labore incididunt tempor sit ad aliqua ipsum incididunt sed incididunt consectetur do ad dolore amet et enim amet adipiscing veniam magna enim lorem consectetur enim amet amet quis dolor dolor consectetur elit quis amet adipiscing incididunt consectetur consectetur tempor minim lorem do consectetur do labore magna ipsum incididunt consectetur minim enim eiusmod elit lorem veniam amet lorem amet aliqua do ipsum adipiscing labore sit ad aliqua tempor sed dolor consectetur dolore veniam dolor enim do dolor tempor sed incididunt ipsum labore veniam amet quis consectetur lorem ipsum magna lorem adipiscing aliqua tempor ipsum dolor magna dolore eiusmod elit sed aliqua dolor eiusmod dolore tempor veniam aliqua ad sit tempor veniam amet dolor adipiscing consectetur aliqua et consectetur amet ad tempor incididunt lorem amet eiusmod lorem amet veniam tempor aliqua ut magna incididunt enim aliqua lorem eiusmod eiusmod enim amet dolore et sed sit lorem dolore sed do amet minim ut adipiscing dolor eiusmod labore do magna ad amet adipiscing sed tempor quis magna enim amet sed ut dolor quis quis ad adipiscing incididunt et incididunt amet lorem do ut ipsum ut elit eiusmod ut ipsum dolor eiusmod enim labore lorem tempor incididunt ipsum amet amet magna sit amet sit dolore dolor ipsum enim tempor amet minim et minim lorem dolore amet veniam labore eiusmod sit ut adipiscing minim dolore dolor minim aliqua aliqua sed sed tempor magna labore lorem et ipsum labore lorem tempor adipiscing eiusmod magna ipsum et amet dolor quis ut consectetur minim tempor ipsum lorem eiusmod amet sed incididunt ad labore tempor tempor sed veniam dolor aliqua quis enim et ad labore magna enim aliqua dolore sed et elit enim et adipiscing dolore adipiscing sit elit veniam ad dolore sit minim ut consectetur magna aliqua eiusmod elit elit veniam amet quis aliqua minim et enim minim amet aliqua veniam aliqua eiusmod labore dolore ad sed labore dolore tempor quis magna ipsum lorem magna eiusmod veniam ad ut enim incididunt sed adipiscing ut aliqua et dolor amet labore minim magna eiusmod lorem enim labore labore consectetur ut veniam tempor do adipiscing do dolor consectetur adipiscing ut ipsum do et dolore et sed dolore enim ipsum minim et dolor tempor incididunt quis aliqua ad sit lorem enim ut ipsum lorem tempor ipsum quis lorem veniam consectetur dolor dolore ad enim incididunt amet tempor labore tempor sed sed magna sit sit enim lorem do enim quis minim quis amet dolor aliqua ut veniam ad elit tempor dolore dolor incididunt incididunt enim do enim dolor adipiscing tempor ut tempor incididunt dolor sit et lorem amet consectetur ipsum magna dolor minim sit adipiscing adipiscing quis sit et tempor elit consectetur enim adipiscing tempor magna ut minim ipsum sit sed dolore consectetur enim ut elit enim sed ipsum minim lorem aliqua et sit ipsum magna ipsum sed ad labore aliqua aliqua tempor dolore sed sit et sit magna veniam adipiscing eiusmod dolore aliqua magna ut ad aliqua dolor sit magna dolor ipsum et magna sit enim sit tempor consectetur dolore elit et dolor eiusmod labore incididunt enim elit enim ipsum elit labore labore dolore consectetur veniam ut adipiscing ipsum aliqua et labore sit incididunt minim sit minim elit sed elit labore sed do sed et elit veniam magna do ut labore elit do ut aliqua magna incididunt labore ut aliqua veniam sed minim ad magna ad dolore ipsum ad magna sed quis ad magna ut adipiscing minim sed elit ad dolore ut aliqua et veniam ad aliqua ad lorem sed ut ipsum incididunt dolor labore aliqua tempor consectetur labore minim do incididunt sit ipsum incididunt magna magna veniam ipsum minim dolor amet elit sed elit consectetur dolore minim et ad enim labore amet aliqua incididunt veniam dolor adipiscing elit lorem enim sed ut do consectetur tempor incididunt enim et enim quis ipsum do adipiscing magna ad veniam adipiscing sit dolore do do do do sit aliqua enim veniam aliqua minim ad amet incididunt elit consectetur aliqua ad ipsum aliqua sed et adipiscing sit do sit enim sed ipsum veniam dolore amet et elit lorem ad tempor tempor eiusmod labore ut sed ut lorem sit dolor adipiscing enim veniam sit ut sed et et adipiscing labore ad ipsum eiusmod tempor sed adipiscing ad lorem sed ut magna quis do quis et ad consectetur sed dolor quis adipiscing ipsum amet tempor quis et aliqua consectetur dolore et enim amet adipiscing sed adipiscing labore ad amet magna tempor quis dolor incididunt aliqua ipsum minim eiusmod veniam et labore ipsum incididunt sed magna ad ipsum enim enim tempor elit labore et adipiscing consectetur dolore ut adipiscing do lorem sed do enim amet enim do labore aliqua lorem sit elit lorem adipiscing enim consectetur ipsum ut elit eiusmod labore elit enim sed ipsum lorem et tempor dolore lorem enim ut do lorem eiusmod et lorem lorem dolore minim sit tempor sit amet dolore et labore enim veniam tempor labore tempor minim minim ipsum incididunt lorem ipsum ut labore amet dolor enim lorem aliqua incididunt adipiscing amet minim amet aliqua enim adipiscing amet do magna quis ipsum sit labore elit incididunt minim sed amet aliqua adipiscing dolore et eiusmod aliqua ut adipiscing consectetur lorem ut eiusmod incididunt labore incididunt ut et eiusmod ut elit magna aliqua enim amet labore incididunt quis et incididunt amet aliqua quis enim magna labore lorem dolore ut ad ut enim eiusmod amet ad incididunt et sit dolore eiusmod elit quis magna amet enim sed ad lorem incididunt adipiscing sit eiusmod enim amet ipsum adipiscing elit labore elit adipiscing sed do ipsum lorem elit adipiscing et incididunt ut labore elit ad dolore minim minim ipsum enim amet quis ipsum eiusmod quis dolore consectetur sed enim veniam amet tempor labore eiusmod dolore eiusmod ad dolore labore tempor consectetur tempor do tempor aliqua incididunt quis sit lorem enim adipiscing ipsum magna ipsum consectetur incididunt dolore adipiscing dolore ut sed tempor et incididunt adipiscing incididunt incididunt do aliqua lorem tempor amet elit ad ut amet consectetur quis dolore amet enim magna et do ad lorem ad tempor adipiscing veniam et minim ad aliqua ipsum incididunt magna dolore dolore magna labore aliqua amet do elit do magna do ipsum adipiscing ipsum adipiscing sit ipsum enim ad dolor lorem tempor elit quis sed ut minim do dolor labore quis quis dolore adipiscing amet dolor quis minim enim adipiscing magna sit consectetur sed amet incididunt labore eiusmod veniam enim incididunt dolore eiusmod quis labore amet veniam dolore ipsum magna eiusmod adipiscing lorem amet ad tempor ipsum veniam ipsum do ipsum ipsum quis et magna enim ut consectetur sit aliqua ipsum et do ipsum dolore enim tempor lorem consectetur amet quis lorem ad veniam labore do tempor ad dolore consectetur dolor ad ad minim lorem ipsum elit sit tempor incididunt tempor incididunt tempor dolor dolor ad aliqua incididunt consectetur amet amet aliqua minim do minim amet ipsum amet sed ad tempor ut eiusmod tempor ut dolore incididunt ut quis quis incididunt tempor enim sit elit dolor dolore lorem elit magna amet dolor veniam consectetur lorem ipsum magna eiusmod sed minim minim adipiscing dolor sit labore sit adipiscing quis tempor tempor adipiscing ipsum eiusmod elit aliqua eiusmod dolore veniam ad tempor ad lorem dolore dolore eiusmod enim lorem aliqua dolore dolore sit lorem ad adipiscing dolore ipsum veniam elit quis ut sit adipiscing magna aliqua quis consectetur dolore veniam ipsum tempor sit magna aliqua et amet magna do magna ipsum quis elit lorem do dolore adipiscing consectetur eiusmod ut sed incididunt enim consectetur amet enim minim sit ipsum veniam lorem quis dolor sed tempor do amet quis ut adipiscing incididunt elit elit sed do ut tempor aliqua elit quis eiusmod dolore ad ad sed sit sed eiusmod do quis et consectetur sed sed labore tempor eiusmod enim amet tempor ut consectetur dolor consectetur dolor veniam elit eiusmod dolor consectetur do sit ut amet sed enim do ipsum dolor sed do incididunt incididunt sit sit labore magna consectetur magna lorem ipsum enim amet eiusmod veniam elit ipsum magna et dolor dolor dolor et magna aliqua enim magna magna incididunt minim adipiscing aliqua lorem eiusmod aliqua eiusmod aliqua incididunt elit eiusmod elit ipsum dolor ipsum enim sit do sit adipiscing eiusmod adipiscing ipsum incididunt eiusmod dolor magna dolore ut veniam sed sit sit eiusmod amet do minim eiusmod sed amet labore et eiusmod incididunt et tempor ad labore eiusmod ut minim labore veniam enim dolore magna quis eiusmod enim tempor do et quis tempor consectetur eiusmod eiusmod ut amet lorem sed eiusmod sit do magna incididunt do sit ad sit sit elit ut ad elit aliqua ipsum dolore enim lorem labore minim minim elit veniam ipsum amet magna et et amet eiusmod sed ut et tempor enim elit ut eiusmod ipsum dolor ad et sit do dolore amet amet enim do aliqua eiusmod ut sed amet enim aliqua labore magna sed quis incididunt sit ipsum aliqua adipiscing elit ad adipiscing quis ipsum enim tempor enim dolor eiusmod tempor dolore ad incididunt minim ut veniam dolor enim aliqua quis ipsum sed do lorem adipiscing ad incididunt do adipiscing tempor sit ut enim do dolor elit labore tempor magna dolore quis labore consectetur aliqua lorem aliqua tempor veniam et eiusmod labore et ut magna enim labore sit ipsum incididunt ut adipiscing ut sit et magna ut aliqua tempor elit consectetur tempor magna aliqua dolore dolore consectetur consectetur veniam amet ad aliqua labore consectetur eiusmod amet incididunt elit veniam minim adipiscing eiusmod quis labore sed sed lorem aliqua dolore ut et adipiscing eiusmod labore ut lorem minim dolor do sed sit ad quis do tempor et tempor sit labore ipsum eiusmod enim ut eiusmod tempor et labore lorem magna magna labore lorem dolore eiusmod veniam incididunt ad sit lorem ut labore lorem tempor incididunt veniam minim veniam minim eiusmod adipiscing consectetur sit sed ad et enim quis ipsum incididunt aliqua ipsum et enim ad magna minim consectetur dolor eiusmod ipsum ad dolore veniam aliqua veniam sit sed dolore dolore ipsum quis eiusmod dolore sit dolore amet adipiscing ut dolor eiusmod quis magna dolor et magna do quis amet ad incididunt ipsum sit eiusmod eiusmod ad labore consectetur quis enim minim ipsum quis elit veniam minim sit amet incididunt elit ipsum dolore ad minim lorem tempor ad do eiusmod eiusmod incididunt quis elit aliqua dolor incididunt sed sed et ut sit quis eiusmod aliqua do magna consectetur enim ad minim labore minim dolore ad do adipiscing eiusmod sit adipiscing labore elit sed amet dolore ipsum eiusmod ipsum sed amet quis dolore adipiscing enim ut tempor amet quis veniam lorem et do consectetur tempor sit eiusmod elit ad eiusmod quis incididunt ad elit incididunt ipsum ipsum do sed aliqua adipiscing magna et veniam ad labore ut lorem lorem ut amet sed aliqua quis consectetur tempor ut enim incididunt lorem quis ad labore consectetur do ad aliqua ad lorem minim elit quis tempor incididunt tempor sed do eiusmod dolor ut minim labore quis ut et sit sed eiusmod sit labore eiusmod ut dolor consectetur tempor adipiscing amet incididunt enim magna incididunt et dolor elit eiusmod dolor magna magna minim lorem ad ipsum labore enim sit adipiscing dolore magna aliqua dolor et et tempor ut tempor dolore dolore sit dolor ad labore ad dolore sed tempor aliqua labore quis sed elit labore tempor ut elit do sit lorem enim aliqua enim consectetur amet labore ad magna sed dolor consectetur sed et sit labore consectetur et amet incididunt quis dolor ut minim minim elit et quis magna veniam magna ut amet consectetur ad ut aliqua dolore lorem minim dolore quis minim magna et ipsum eiusmod lorem dolore magna magna quis lorem labore adipiscing et ut sed elit ipsum veniam dolore elit dolore ut quis et adipiscing enim amet sit magna do eiusmod amet ipsum magna quis sed consectetur dolor labore do et ut amet eiusmod consectetur dolore consectetur incididunt sed sit aliqua consectetur tempor enim enim tempor sit do aliqua et eiusmod enim enim ad eiusmod ad dolor do lorem amet sit magna minim elit sed sit veniam sed quis sit amet ad quis et quis adipiscing ad sit ad dolor sit adipiscing sit eiusmod incididunt eiusmod dolor incididunt sed adipiscing veniam sit sit elit minim veniam veniam aliqua ipsum lorem adipiscing sit tempor labore et quis magna ipsum ipsum minim do do ipsum ut dolor consectetur adipiscing magna tempor lorem quis enim dolore amet magna amet adipiscing aliqua ad consectetur amet sit enim veniam consectetur veniam elit aliqua do magna amet eiusmod do consectetur consectetur tempor minim tempor enim lorem eiusmod tempor sed quis sed elit tempor consectetur amet dolor do ut minim lorem incididunt lorem enim tempor dolor ipsum dolore magna ut et adipiscing lorem ut sit labore enim labore tempor minim veniam do amet et labore minim aliqua elit ut quis dolore consectetur elit et eiusmod dolor magna eiusmod ad amet enim labore incididunt tempor labore quis ipsum lorem ad elit tempor dolor et quis et elit dolor dolore amet amet magna labore dolore adipiscing ut consectetur sed consectetur adipiscing incididunt lorem veniam eiusmod magna incididunt veniam ad ipsum sit do dolore do do ut do ipsum lorem lorem dolor ut amet dolor magna adipiscing labore ipsum incididunt enim lorem lorem ad veniam incididunt sit consectetur ut tempor incididunt do adipiscing amet et minim elit ipsum et amet elit consectetur adipiscing enim et aliqua et et amet dolor dolor incididunt sed incididunt elit magna lorem ut enim ad adipiscing ut labore do consectetur do incididunt dolor minim veniam ut ad dolor incididunt ad minim consectetur tempor tempor incididunt sed tempor labore sit veniam consectetur ad ut minim sed quis eiusmod lorem adipiscing minim dolor do veniam labore tempor sit labore tempor quis ut quis ut adipiscing dolore minim enim enim ad magna ad aliqua quis consectetur aliqua sed tempor aliqua consectetur sed amet ipsum dolor ad minim adipiscing sit magna ipsum et ut enim elit dolore eiusmod aliqua magna dolore sit incididunt ad dolore sed consectetur elit lorem consectetur veniam enim incididunt et sit dolor ipsum elit tempor sit ad sit eiusmod ut ipsum minim do incididunt ad labore consectetur dolore ad incididunt do magna veniam adipiscing sed ipsum sit aliqua ad minim lorem elit sed dolor ut labore sit tempor elit ad tempor sed dolor incididunt veniam lorem veniam consectetur sit sed quis ut adipiscing labore lorem incididunt amet magna magna aliqua sit dolor lorem magna minim veniam do dolore ipsum enim dolor do dolor labore quis sit tempor incididunt enim lorem aliqua labore amet minim magna adipiscing magna et sit enim eiusmod ipsum do adipiscing lorem sed labore minim amet et magna dolor eiusmod dolor quis ad elit sit tempor elit sit eiusmod veniam sit lorem sit dolore enim aliqua dolore amet eiusmod magna amet dolore dolor ipsum adipiscing et adipiscing amet minim elit et ut ad eiusmod consectetur elit lorem consectetur labore do veniam incididunt et eiusmod adipiscing ut dolor veniam do ut tempor labore sed adipiscing ut adipiscing ipsum amet lorem ad sed do incididunt et consectetur quis minim consectetur quis consectetur sit sed ipsum amet veniam do labore minim eiusmod dolor do magna eiusmod ad incididunt magna ipsum ipsum incididunt ut do veniam ad elit eiusmod labore ipsum aliqua minim ipsum labore do quis sed ad ipsum minim veniam aliqua dolore enim ut quis incididunt incididunt adipiscing elit sed ut et eiusmod dolore tempor aliqua minim ipsum elit elit et quis magna quis dolore sed ipsum sed et tempor et ipsum sit amet dolore quis adipiscing sed adipiscing tempor labore sed ut amet dolor incididunt adipiscing dolor ut ipsum enim elit magna enim incididunt eiusmod aliqua dolore aliqua veniam sed veniam elit ipsum elit lorem veniam quis eiusmod eiusmod enim consectetur ad minim aliqua tempor minim et ut quis consectetur quis consectetur minim magna minim minim do consectetur et minim amet tempor minim labore amet quis dolore quis ut et ipsum dolore ad ut ipsum quis minim ipsum eiusmod elit ad minim adipiscing ipsum aliqua incididunt enim aliqua sed tempor tempor ad consectetur elit quis dolore amet dolore dolor do amet lorem do sit enim sed sed lorem lorem magna ipsum amet consectetur eiusmod veniam adipiscing amet dolor dolore eiusmod sed et sed labore dolore dolore ut dolor lorem adipiscing sed enim eiusmod et eiusmod et aliqua ut ad aliqua consectetur minim magna et elit dolore elit sed dolore quis enim magna elit adipiscing incididunt adipiscing incididunt labore sed eiusmod incididunt lorem adipiscing ipsum dolor sit do tempor dolore ut dolore et do lorem enim magna ad tempor tempor et ipsum sit ipsum incididunt quis sed ut labore ipsum magna lorem sed ut enim tempor dolore ad ut ipsum amet dolor sit et enim ad sed quis do eiusmod incididunt sed lorem lorem dolore et et lorem sit tempor sed labore veniam elit et consectetur eiusmod et tempor do sed sed incididunt eiusmod sed adipiscing amet elit labore magna et consectetur veniam et ut incididunt ipsum aliqua consectetur adipiscing incididunt dolor ipsum sed dolor quis labore tempor elit dolore dolor eiusmod veniam eiusmod do incididunt do consectetur elit sit aliqua veniam et ipsum amet tempor veniam incididunt dolore amet eiusmod minim do minim magna ut quis amet lorem magna enim enim lorem sit minim amet et dolor lorem ut ipsum eiusmod labore elit ipsum magna ut labore lorem ut labore amet adipiscing quis dolore do tempor elit enim dolor minim adipiscing et incididunt enim incididunt et sed tempor consectetur et sit aliqua minim sed quis labore et et lorem eiusmod ipsum consectetur tempor labore aliqua dolor ad amet do magna enim enim adipiscing lorem ad ipsum adipiscing sed aliqua quis dolor adipiscing ad minim adipiscing dolor veniam veniam dolor sed elit ipsum lorem dolor enim labore dolore dolore amet et magna magna ipsum enim sit veniam amet sit quis magna do eiusmod elit veniam minim do enim enim incididunt consectetur enim tempor dolor do ut dolore sed sit sit amet adipiscing ut enim lorem lorem enim ut dolor ad sit magna enim sed et ut amet sed adipiscing quis consectetur adipiscing eiusmod eiusmod aliqua labore magna incididunt ad incididunt incididunt tempor labore dolore sed dolore sed dolor adipiscing sed ipsum tempor magna elit labore labore lorem ut dolore consectetur labore tempor lorem ipsum veniam consectetur tempor ipsum ipsum eiusmod eiusmod quis veniam eiusmod magna ipsum dolore et sed sed amet magna elit elit tempor dolore lorem dolore et ut aliqua incididunt minim labore quis quis ad do ipsum ad ad labore aliqua veniam eiusmod sit dolore lorem ut quis ipsum veniam do eiusmod ad sed consectetur magna ipsum veniam consectetur dolor eiusmod et labore elit lorem veniam consectetur quis et incididunt dolor elit dolor do do quis minim ad sed consectetur aliqua labore adipiscing consectetur adipiscing quis consectetur aliqua ad minim adipiscing do sed magna enim labore magna ut quis enim dolore dolore minim quis enim consectetur dolor adipiscing consectetur magna quis ad amet sit quis minim enim elit tempor sit tempor quis et adipiscing adipiscing consectetur ut dolor ut minim dolore sed adipiscing consectetur sit elit adipiscing consectetur aliqua minim amet ipsum adipiscing ut lorem incididunt consectetur dolor ut do lorem ad consectetur lorem do sed dolor sed minim consectetur magna veniam eiusmod do ut labore dolor ipsum ipsum aliqua minim labore veniam ad adipiscing labore ut consectetur ut sit ipsum sed incididunt veniam lorem quis do adipiscing ad ut minim minim dolor veniam sit incididunt labore eiusmod eiusmod sed eiusmod adipiscing veniam elit dolor veniam adipiscing ad aliqua dolore adipiscing elit eiusmod amet adipiscing adipiscing do dolor consectetur eiusmod sit ipsum do sed quis veniam sit tempor sed sit adipiscing sit magna enim tempor aliqua enim elit lorem sit lorem enim eiusmod lorem veniam sed adipiscing enim elit et magna consectetur adipiscing lorem eiusmod incididunt veniam ut eiusmod enim aliqua veniam aliqua consectetur ut ipsum veniam tempor magna sed sed elit do sit consectetur elit consectetur et sed quis veniam aliqua sit adipiscing labore adipiscing do ad dolore do adipiscing dolore et eiusmod minim lorem quis dolor ipsum tempor labore dolor ipsum amet ad veniam enim adipiscing aliqua enim tempor lorem enim sed incididunt sed labore elit adipiscing tempor quis dolor magna magna dolor sit quis amet dolor do elit eiusmod amet dolor quis sit quis lorem labore aliqua amet consectetur do dolore et ut quis adipiscing enim aliqua elit elit elit tempor tempor eiusmod lorem dolore ipsum quis veniam elit dolor lorem labore minim incididunt lorem ad quis magna magna minim veniam sed minim ut ad labore dolor dolor ut dolor sit lorem eiusmod et ad tempor elit quis dolor quis et dolore elit enim minim magna aliqua dolore sit sed sit et amet amet dolore veniam labore aliqua minim sit enim ipsum ipsum lorem do magna quis amet labore et ipsum do lorem consectetur veniam tempor minim enim ipsum dolor do ad do veniam do magna ipsum ad ipsum ipsum veniam eiusmod do elit et ut sed enim enim labore aliqua ad veniam sed dolor ipsum dolore incididunt sit et dolor eiusmod dolore incididunt amet ut veniam incididunt ut incididunt et elit dolore labore amet amet eiusmod et dolore ut sit dolor magna labore quis labore adipiscing magna amet veniam dolore do adipiscing ut elit labore ipsum enim labore tempor do enim lorem aliqua ad ipsum incididunt quis aliqua magna et eiusmod tempor dolore elit lorem elit enim incididunt consectetur do enim ipsum ad et quis ipsum eiusmod do labore aliqua aliqua minim dolor do magna lorem enim lorem amet eiusmod amet magna sed adipiscing elit sed minim tempor eiusmod eiusmod lorem do enim ut dolor labore incididunt sit ut dolore labore veniam dolore adipiscing enim ipsum dolor eiusmod aliqua do sed amet lorem veniam consectetur enim do magna adipiscing dolor dolore aliqua incididunt dolore enim consectetur consectetur magna ut quis enim incididunt minim adipiscing quis magna incididunt ipsum sed consectetur ad magna ipsum ut enim dolor magna veniam et enim quis ipsum tempor tempor ad ad adipiscing incididunt tempor elit elit minim enim lorem lorem enim et quis ipsum magna lorem veniam do et lorem lorem ut eiusmod dolore sit dolor enim aliqua lorem do minim ad aliqua amet ut lorem quis veniam eiusmod eiusmod sed eiusmod eiusmod amet do enim minim consectetur tempor minim dolore incididunt et et amet eiusmod quis eiusmod eiusmod ipsum sit incididunt adipiscing ut aliqua dolore ipsum labore lorem do enim enim tempor minim adipiscing et ad ipsum dolore eiusmod amet sed adipiscing quis amet incididunt adipiscing magna amet sed do ipsum veniam labore dolore incididunt tempor elit et ut et sed ad dolor ut aliqua adipiscing eiusmod minim ut dolor ut eiusmod minim eiusmod ut ipsum minim amet adipiscing sit do tempor ad ipsum dolore do consectetur ipsum consectetur eiusmod incididunt ut magna dolore veniam veniam adipiscing sed dolore aliqua ut do quis enim consectetur sed incididunt veniam amet dolore labore aliqua eiusmod lorem lorem consectetur incididunt ad do consectetur ad dolor labore sit ad aliqua quis et ut elit ad ad aliqua do dolore sed eiusmod labore ad ut ad lorem ut veniam incididunt sit aliqua enim adipiscing labore adipiscing minim veniam sed consectetur quis amet magna veniam do eiusmod minim lorem sit veniam dolor amet adipiscing incididunt lorem minim tempor dolore amet veniam veniam et tempor eiusmod amet et enim sit quis lorem ut dolore lorem tempor lorem ipsum labore elit elit minim veniam sit eiusmod lorem minim labore ipsum et labore sit veniam veniam lorem et dolor elit magna incididunt eiusmod veniam aliqua lorem et lorem dolore incididunt tempor aliqua magna magna ad dolor do lorem do sit et do elit sed do ut dolore eiusmod aliqua consectetur do elit incididunt amet aliqua enim ad ad do magna consectetur ut et quis dolor veniam dolor elit consectetur eiusmod adipiscing incididunt amet dolore amet incididunt ad veniam elit eiusmod amet minim enim elit tempor elit sed ad veniam incididunt eiusmod elit dolore incididunt magna amet lorem veniam et adipiscing eiusmod aliqua ad adipiscing consectetur veniam minim minim ut do et sit dolor ut adipiscing elit minim ut adipiscing consectetur magna lorem et lorem eiusmod sit tempor magna eiusmod et adipiscing aliqua et aliqua quis magna sit incididunt eiusmod labore sed sit elit dolor ipsum amet labore ipsum elit adipiscing sed eiusmod sed consectetur minim sed adipiscing ad labore magna elit incididunt aliqua dolor eiusmod ipsum lorem adipiscing quis magna ad sit sed labore tempor do ipsum magna minim veniam eiusmod labore sed enim sed et sed incididunt adipiscing consectetur ipsum minim sit ut enim elit eiusmod minim eiusmod dolore dolore veniam incididunt incididunt magna incididunt ad do elit aliqua sed veniam adipiscing sit veniam elit dolore do et ad consectetur consectetur quis tempor dolore sit aliqua aliqua ut elit amet et quis et sit ut incididunt incididunt sit enim amet dolor incididunt sed ad consectetur ipsum ut eiusmod dolor adipiscing adipiscing ipsum magna adipiscing minim magna elit quis lorem enim elit sit dolore elit eiusmod magna quis magna sed amet elit eiusmod magna ut enim dolor eiusmod aliqua ad ipsum dolor elit eiusmod magna sed tempor enim incididunt ad sed aliqua lorem labore enim eiusmod et consectetur incididunt sed et ut aliqua adipiscing adipiscing elit eiusmod eiusmod elit adipiscing lorem et aliqua elit veniam sit dolore amet sed adipiscing aliqua ut sed dolor aliqua amet sed do veniam sit dolor incididunt amet sit amet incididunt dolore veniam tempor consectetur magna dolore tempor amet consectetur sit ad et minim adipiscing dolor adipiscing adipiscing et quis ut consectetur do quis eiusmod sed ipsum labore dolor lorem ipsum ad elit consectetur enim dolore veniam ut tempor elit ut sit ut enim magna veniam amet labore eiusmod incididunt dolor eiusmod incididunt veniam eiusmod veniam veniam labore magna elit minim et tempor dolore veniam et do consectetur lorem do ad aliqua amet adipiscing lorem ut ad amet ad aliqua ut enim adipiscing elit sed aliqua eiusmod labore enim sit do lorem eiusmod lorem ad enim eiusmod dolor minim labore sed ipsum incididunt consectetur quis elit amet adipiscing veniam et dolor lorem incididunt dolore sit elit sit sit ad ut dolor veniam dolor do sit magna sed sed aliqua ad adipiscing lorem magna consectetur ad magna dolor quis aliqua elit do tempor eiusmod aliqua dolore elit magna dolor dolor sit sed et minim ad dolore consectetur dolor consectetur et magna consectetur ipsum dolor elit ad et adipiscing incididunt dolor incididunt lorem sed aliqua elit dolor amet dolore et labore elit elit sit incididunt et quis tempor dolore ut sit consectetur enim quis incididunt minim veniam dolore minim tempor amet ad tempor ipsum quis labore enim aliqua eiusmod minim ut adipiscing dolore adipiscing lorem do incididunt labore dolor consectetur ut lorem dolore dolor elit amet labore veniam veniam do dolor minim amet minim dolore do magna elit minim ut tempor et dolore amet incididunt dolore dolor adipiscing enim sed aliqua minim veniam lorem sed enim ut veniam enim ad do sed sit magna amet incididunt ipsum do eiusmod dolore dolor do incididunt amet labore enim et ipsum ad ipsum magna enim ipsum elit ut quis consectetur do ad ad et dolore ut tempor dolor et elit incididunt ipsum lorem ad ut minim aliqua consectetur elit lorem aliqua do minim dolore aliqua eiusmod dolore aliqua minim amet sit labore enim sed labore sit amet quis dolor et quis quis aliqua quis do sit quis veniam labore magna sed adipiscing dolor eiusmod ad enim adipiscing consectetur elit incididunt veniam labore incididunt ut sed aliqua eiusmod incididunt lorem do ad labore do minim tempor aliqua labore minim minim eiusmod veniam veniam adipiscing sit incididunt eiusmod incididunt amet sit consectetur amet lorem elit veniam aliqua lorem lorem enim adipiscing tempor eiusmod elit eiusmod ad quis lorem ut dolore amet minim ad incididunt tempor tempor quis aliqua ut dolor ipsum dolore sit minim amet enim quis incididunt lorem tempor dolore aliqua magna sit et consectetur ut et elit dolore minim ut ad ut consectetur dolore ad enim ut ad magna amet dolor elit ut ut quis enim adipiscing lorem ad sed amet sit sed dolor lorem minim dolore enim veniam sed quis consectetur minim et magna dolore enim minim do veniam et quis tempor labore dolor consectetur sit amet dolore lorem lorem tempor minim incididunt eiusmod sed incididunt et quis tempor ipsum ipsum eiusmod enim incididunt elit ut sit do ut dolore ad eiusmod quis et enim adipiscing tempor dolor magna dolore incididunt dolor labore ut eiusmod et sed eiusmod amet aliqua dolore do sit labore dolore incididunt lorem et ipsum ut incididunt lorem ad aliqua enim incididunt ut adipiscing aliqua ipsum labore minim sit sed minim eiusmod aliqua tempor ad magna eiusmod magna amet enim veniam minim lorem enim lorem minim labore aliqua magna aliqua sed do lorem labore et aliqua consectetur dolore incididunt sit ad adipiscing adipiscing minim dolore eiusmod minim do incididunt magna sit minim enim dolore adipiscing amet ut quis enim minim ad magna enim ipsum incididunt adipiscing lorem minim magna magna quis veniam veniam elit ut do aliqua consectetur dolor enim quis lorem consectetur magna ad amet aliqua quis sed et incididunt elit labore aliqua ipsum ipsum labore aliqua consectetur sit ipsum sit veniam lorem consectetur ad aliqua ipsum quis tempor ipsum veniam adipiscing sit ad sed elit dolore aliqua elit veniam ut sed aliqua veniam eiusmod elit lorem labore aliqua tempor do adipiscing ut sed dolor ipsum ad enim ut minim sed consectetur incididunt ad tempor amet labore enim magna labore sit quis magna dolor magna sit eiusmod eiusmod adipiscing magna adipiscing amet sed eiusmod et amet tempor ipsum lorem ipsum dolore consectetur veniam consectetur dolor ipsum ut amet magna ut labore veniam consectetur eiusmod labore veniam incididunt magna do aliqua ad sed ipsum ut sit et incididunt dolor aliqua sed do amet eiusmod veniam quis dolore aliqua tempor do amet labore aliqua elit et quis tempor labore sit labore elit adipiscing minim ut amet elit sed ad elit dolor ad ad incididunt labore veniam do ut labore amet amet tempor adipiscing ad ad enim amet enim sit enim enim et lorem quis incididunt amet enim magna dolor lorem minim dolor dolor veniam elit tempor quis minim dolor magna lorem minim consectetur lorem elit aliqua ipsum amet dolor magna ipsum veniam quis aliqua dolore ipsum aliqua sed adipiscing sed sed tempor adipiscing amet lorem sed labore lorem lorem et sed eiusmod dolore dolor adipiscing eiusmod lorem ipsum sed magna quis ad incididunt et do sed ipsum enim lorem veniam lorem ipsum ad magna eiusmod quis amet veniam veniam minim ad sit sed incididunt enim ipsum sit dolore do incididunt amet et consectetur magna quis dolore magna elit et ipsum labore sed do amet quis dolor labore ad sit amet consectetur dolore adipiscing sed dolore ut ipsum aliqua elit veniam labore elit tempor ipsum aliqua elit elit ad dolor quis ad incididunt consectetur dolore ipsum incididunt ut elit minim elit tempor ad incididunt amet dolor elit dolore ad tempor minim magna labore amet incididunt quis labore minim do dolor enim ipsum dolore tempor sit quis dolore labore ut amet do amet sit sit eiusmod eiusmod enim adipiscing dolore aliqua dolore elit incididunt magna consectetur ipsum ipsum elit sit ut et sed ipsum magna et enim ad et lorem magna dolor et ad dolore sed veniam minim labore dolore dolor consectetur enim et enim lorem adipiscing consectetur minim enim dolor elit ipsum magna lorem dolor ut lorem ipsum tempor consectetur amet sed aliqua elit do enim sit minim aliqua sed veniam do aliqua consectetur veniam magna lorem do do veniam quis elit quis ad ipsum enim sit minim adipiscing aliqua minim adipiscing amet magna magna sit ut tempor eiusmod adipiscing eiusmod dolore do lorem dolore elit dolor quis eiusmod incididunt adipiscing incididunt et do tempor eiusmod labore magna lorem adipiscing lorem incididunt ad ut magna lorem dolor incididunt eiusmod dolor dolor sit ad eiusmod incididunt amet dolor quis veniam lorem ut amet lorem incididunt eiusmod adipiscing elit dolore dolor dolor adipiscing dolor quis incididunt tempor amet tempor adipiscing ut et ut magna dolor tempor labore et ut incididunt dolor sed amet ad dolore et et dolor sit adipiscing ut quis quis ad ut enim dolore do quis veniam ad et veniam dolore tempor ad dolore consectetur ut sit enim minim sed ipsum sit quis amet minim consectetur magna ad veniam minim sed aliqua sit sit sed consectetur adipiscing quis adipiscing et sit quis quis quis veniam enim adipiscing do dolore ad sit ipsum sed do quis eiusmod ut dolore ut enim magna amet eiusmod dolor magna veniam elit amet dolore sit do tempor magna enim ut aliqua minim sed amet ipsum dolor enim consectetur sit magna minim adipiscing et do ipsum sed adipiscing adipiscing adipiscing consectetur incididunt tempor incididunt tempor et dolor ipsum magna quis ut dolore do quis labore labore amet lorem consectetur quis lorem sit tempor ut veniam ut do adipiscing et et sit labore eiusmod incididunt magna sed lorem aliqua eiusmod elit do labore lorem eiusmod labore amet et amet quis enim do quis eiusmod veniam aliqua dolor consectetur dolore magna et quis incididunt minim incididunt aliqua eiusmod minim enim sed dolore elit ipsum et ipsum et ut ipsum quis magna ut lorem enim ad veniam consectetur sit dolore lorem ad tempor adipiscing dolore eiusmod ipsum consectetur et enim enim lorem ad sed quis dolore do sit sit magna consectetur dolore labore elit ad magna minim veniam ipsum sed quis tempor elit do do minim adipiscing ad do magna elit labore dolor tempor minim et ipsum elit sed adipiscing ut dolore do tempor ut incididunt enim elit minim aliqua ad labore labore et quis do lorem consectetur dolore sit veniam eiusmod et elit enim et sit labore et magna ad ut quis dolore amet do dolor do et et sed labore do minim consectetur do consectetur labore dolor sit labore eiusmod minim ut magna tempor minim amet aliqua quis sed veniam aliqua enim labore consectetur do eiusmod sit do quis minim enim dolore amet sed sit tempor incididunt labore enim minim magna magna veniam ipsum ad tempor veniam aliqua consectetur ut amet quis labore tempor elit eiusmod et dolor enim labore labore adipiscing magna labore tempor amet amet labore magna tempor aliqua quis dolor elit ipsum aliqua amet consectetur veniam dolore sed minim minim quis ut elit dolore ad sed amet eiusmod incididunt sed tempor sit tempor ad elit minim eiusmod veniam tempor sed lorem tempor magna ad ad adipiscing ad et ad sit consectetur incididunt quis magna consectetur lorem ad lorem tempor magna lorem labore labore eiusmod dolor ad sed do incididunt et veniam enim ad magna ipsum adipiscing incididunt ipsum elit veniam aliqua minim ipsum dolor dolore adipiscing aliqua ipsum sed et eiusmod dolor tempor lorem minim quis eiusmod enim sed lorem minim amet labore consectetur dolore tempor dolore tempor do enim incididunt aliqua ipsum aliqua eiusmod aliqua amet labore adipiscing labore sed et elit incididunt enim sit lorem dolor minim adipiscing lorem dolore ad magna labore dolor aliqua adipiscing labore dolor labore lorem ipsum ipsum quis sed quis eiusmod sit labore adipiscing sed labore aliqua elit veniam adipiscing elit lorem ad dolore et magna do ut et do adipiscing consectetur enim lorem elit enim enim ut sit veniam elit elit aliqua amet incididunt incididunt sed lorem sed aliqua magna adipiscing dolor dolore et ad aliqua consectetur sed et elit magna consectetur eiusmod sit aliqua ad amet ut elit enim ad ipsum quis consectetur tempor sed tempor consectetur ut minim dolore adipiscing aliqua magna tempor adipiscing ut eiusmod dolor et labore dolore ad do elit consectetur ad eiusmod do magna veniam sit dolore tempor adipiscing et aliqua amet minim enim lorem sit incididunt ad amet ut dolor labore incididunt ad lorem sed amet ipsum elit aliqua labore ut magna minim incididunt ipsum dolor dolore dolor magna elit incididunt lorem labore sed do et do eiusmod dolore dolore elit consectetur ad ad ipsum eiusmod adipiscing eiusmod et lorem aliqua enim veniam adipiscing incididunt eiusmod incididunt magna sed ad minim quis adipiscing eiusmod sit adipiscing do tempor consectetur adipiscing eiusmod ut sit elit labore ut magna magna incididunt enim tempor ut ipsum consectetur labore do dolor sit ipsum et sed enim et elit adipiscing ut ut do dolor ut eiusmod consectetur enim et magna amet labore sed dolor ipsum sed aliqua ad tempor elit veniam magna amet sed ad lorem lorem incididunt dolore magna eiusmod ut sed elit lorem magna ipsum amet dolor consectetur veniam consectetur adipiscing incididunt elit enim magna dolore enim magna do do veniam lorem aliqua eiusmod elit quis do lorem elit minim veniam enim adipiscing ipsum dolor minim incididunt aliqua eiusmod consectetur elit incididunt consectetur adipiscing tempor ut ipsum labore aliqua ut aliqua veniam minim adipiscing aliqua do ipsum sed ut labore elit et et labore amet sed labore do lorem do dolore sit ut tempor ipsum lorem sit elit ad consectetur tempor do dolore magna sit do sed magna dolore et labore do labore tempor veniam lorem ut magna incididunt labore lorem dolor dolore dolore eiusmod do labore et dolor adipiscing aliqua incididunt aliqua do quis consectetur veniam lorem adipiscing veniam minim amet dolor veniam consectetur aliqua dolor magna enim enim tempor dolor et incididunt aliqua dolore adipiscing amet magna elit tempor tempor aliqua adipiscing minim ipsum minim consectetur do dolor ipsum enim sit minim quis elit labore sed et eiusmod quis ipsum sit quis ipsum adipiscing ipsum labore eiusmod ipsum eiusmod labore enim enim ad ut dolore enim veniam consectetur minim minim ipsum adipiscing consectetur ad incididunt consectetur dolor quis do eiusmod minim quis ipsum minim ut consectetur ad ipsum adipiscing quis elit lorem dolore minim sit enim et enim veniam ad elit tempor dolor veniam incididunt et consectetur aliqua lorem minim et adipiscing dolor ut enim quis labore aliqua elit amet lorem eiusmod minim aliqua dolore consectetur minim do consectetur do ipsum eiusmod ad do incididunt ipsum enim adipiscing adipiscing amet lorem sit eiusmod dolor sed ut labore aliqua elit consectetur ipsum consectetur elit magna magna consectetur sed consectetur eiusmod veniam ad dolor ad ipsum dolore et ut dolor tempor eiusmod lorem lorem sit labore labore tempor dolor consectetur veniam minim ut lorem sit eiusmod veniam sit ad dolore enim magna sit quis amet do aliqua do elit aliqua do dolore dolor labore dolor labore magna consectetur labore lorem incididunt ut amet aliqua labore dolor aliqua aliqua dolore eiusmod tempor tempor ipsum quis magna tempor enim ipsum ad minim lorem lorem dolore incididunt labore labore ipsum et consectetur lorem tempor adipiscing et sit minim tempor minim amet dolore sit consectetur enim lorem eiusmod ad sed elit tempor do sed ut quis adipiscing enim ad labore veniam sed do adipiscing tempor tempor lorem veniam amet do quis do lorem incididunt magna dolore incididunt incididunt amet eiusmod dolor tempor lorem magna eiusmod do ipsum magna ad dolor sit veniam lorem ipsum et quis magna dolor veniam dolor dolore ad incididunt dolor do elit ad minim consectetur enim quis ut labore adipiscing quis do elit elit sed sit elit adipiscing ipsum amet sit eiusmod lorem ipsum quis amet dolore labore elit magna aliqua lorem dolore sed sed sed ut eiusmod labore sed veniam aliqua quis ipsum aliqua sed consectetur dolor consectetur dolor amet magna tempor ipsum incididunt ipsum et et quis quis enim adipiscing do enim elit lorem sit lorem dolor adipiscing lorem consectetur consectetur lorem dolor sed ipsum enim do sed elit sit et minim ipsum do eiusmod ad amet lorem do amet amet eiusmod ut tempor et lorem magna dolor aliqua aliqua et amet ad sit do eiusmod magna aliqua consectetur quis sit ipsum do eiusmod do veniam consectetur adipiscing dolor do incididunt sit incididunt magna veniam amet lorem quis amet lorem sed do et labore ut ad elit enim ad et do lorem enim dolor incididunt et minim consectetur eiusmod labore sed dolor amet quis ipsum sed eiusmod lorem ad do incididunt amet ipsum ad sed eiusmod sed ipsum quis ipsum eiusmod incididunt do et enim quis tempor consectetur et elit ut ad sed dolor sed aliqua magna adipiscing labore minim quis dolore lorem dolore ut do consectetur lorem amet ad lorem incididunt quis et labore amet ad sit enim tempor dolore sit eiusmod elit amet labore elit sit dolore minim quis sit dolore quis ut dolore ut minim ut dolore ipsum sed dolor labore amet labore quis et quis consectetur incididunt do tempor ut ad magna adipiscing adipiscing ipsum minim minim aliqua eiusmod consectetur adipiscing dolor aliqua tempor lorem aliqua tempor minim enim aliqua minim minim dolore magna dolore quis quis ut magna dolore aliqua enim incididunt consectetur elit veniam et amet amet amet et dolore lorem adipiscing sed magna tempor minim veniam ad do incididunt adipiscing incididunt ut sit elit sit enim minim dolore aliqua enim incididunt sed eiusmod incididunt sed dolore dolore ad sed elit amet ipsum do sit ut veniam aliqua adipiscing et do tempor amet sed aliqua incididunt dolor elit enim ipsum elit eiusmod ut do dolore quis veniam dolore enim ipsum veniam quis adipiscing sed consectetur enim ipsum ut sit labore consectetur magna consectetur quis sit dolor lorem magna dolor do enim sit amet aliqua ipsum et ad ipsum lorem tempor magna magna incididunt incididunt ipsum consectetur consectetur tempor ut enim ut enim et do sit adipiscing ut magna labore magna labore ipsum dolore dolor et et aliqua do aliqua ut lorem tempor elit quis ut dolor dolor ut enim ipsum ipsum ad ad minim dolor elit ipsum sed aliqua labore et consectetur ut do aliqua ipsum amet do ut dolor ut ad incididunt quis aliqua eiusmod ipsum veniam ut incididunt veniam aliqua et ipsum lorem magna veniam tempor eiusmod dolor enim incididunt dolore dolor quis enim do aliqua dolor amet sit incididunt quis enim incididunt sed labore incididunt tempor labore dolore do ut ipsum quis eiusmod dolore incididunt sit eiusmod dolor ad incididunt et sed labore sit consectetur aliqua eiusmod consectetur dolor elit adipiscing ad incididunt ipsum et sit elit dolor lorem ipsum dolore adipiscing enim minim minim ad adipiscing minim tempor ipsum magna tempor et amet dolore dolore enim ipsum dolor incididunt ipsum eiusmod quis veniam sed ipsum dolor ipsum do adipiscing ut ipsum adipiscing aliqua eiusmod consectetur ipsum incididunt adipiscing tempor sed ad consectetur ipsum incididunt et enim aliqua do lorem minim sit tempor adipiscing magna quis ut enim adipiscing lorem veniam labore sed amet elit adipiscing sit labore ipsum enim labore do ad tempor magna minim et elit lorem veniam adipiscing aliqua quis lorem sit elit tempor ad ipsum sed ipsum magna minim ipsum aliqua ipsum lorem ut ad incididunt minim do quis adipiscing ut magna adipiscing magna tempor et magna ipsum adipiscing magna adipiscing minim eiusmod dolore lorem minim elit adipiscing consectetur lorem labore quis ad minim eiusmod aliqua quis consectetur veniam sed sed consectetur do et magna incididunt lorem minim adipiscing dolore dolor veniam lorem amet amet aliqua et minim amet dolore labore enim lorem ipsum do incididunt eiusmod adipiscing consectetur magna ad quis enim adipiscing ad consectetur incididunt veniam labore tempor quis lorem tempor consectetur magna ut lorem ad adipiscing quis adipiscing ut do lorem labore minim enim ut dolor et ad aliqua sed sed quis sed ut sit lorem ad quis minim dolore dolore adipiscing enim sit elit ut incididunt enim labore labore magna quis enim eiusmod incididunt adipiscing ut dolore do adipiscing sed lorem incididunt magna elit dolore consectetur labore labore eiusmod do elit dolor ad aliqua eiusmod sit enim do incididunt sit adipiscing dolor adipiscing ad veniam magna dolor sed ad eiusmod aliqua quis minim sit do sed sit et minim tempor consectetur adipiscing veniam enim tempor et veniam veniam sit tempor labore do sed ut aliqua dolore tempor ipsum amet sit incididunt sed quis lorem quis do do ipsum aliqua ipsum tempor eiusmod minim aliqua dolor ipsum et aliqua labore dolor labore et do consectetur ut incididunt adipiscing veniam amet do consectetur quis ut sed dolore ipsum consectetur veniam sit et et sit et adipiscing tempor et veniam elit quis eiusmod et et quis ad sed minim eiusmod sit magna dolor consectetur adipiscing adipiscing labore aliqua sed adipiscing et veniam adipiscing et adipiscing ut dolore amet minim veniam minim enim ipsum aliqua minim aliqua veniam ad et labore et magna eiusmod sed consectetur et ipsum enim eiusmod aliqua dolor et dolor dolore et veniam ut et minim incididunt amet tempor incididunt tempor dolore enim dolor do aliqua adipiscing sed aliqua quis sed ad ad labore elit ad labore eiusmod dolore adipiscing et do magna magna veniam elit lorem sed veniam adipiscing et dolor do amet incididunt ipsum elit lorem tempor aliqua minim dolore veniam minim quis incididunt ad eiusmod dolor ad magna minim dolor dolor quis ut tempor sit ut labore enim veniam aliqua quis enim ipsum sit elit ipsum consectetur aliqua tempor veniam do enim sit elit ut minim enim consectetur minim et ad ipsum amet incididunt ipsum aliqua ad dolor quis dolor tempor sed ad tempor et minim ut eiusmod et sit veniam quis magna minim et enim enim adipiscing et minim ipsum elit et consectetur veniam dolor do dolore eiusmod sit enim dolor ut ipsum do sit consectetur amet sit aliqua minim labore et amet consectetur ipsum quis ad adipiscing do ad amet ad tempor magna adipiscing ut ipsum amet elit enim quis adipiscing do ut tempor amet dolore ad dolore minim eiusmod labore ipsum sed do sed et labore labore sit quis quis ipsum labore elit magna minim magna elit minim ipsum amet adipiscing amet et amet enim lorem magna dolor lorem eiusmod veniam dolore minim veniam et ut incididunt magna dolore ad et dolor consectetur elit sed dolor adipiscing elit consectetur tempor sed do sed ipsum incididunt dolore adipiscing magna dolore ut elit tempor et consectetur tempor ipsum enim incididunt consectetur incididunt ad elit magna magna eiusmod sed sit labore dolor tempor magna consectetur elit ad ut aliqua ad lorem sed ut sed et ad ipsum ad aliqua dolore consectetur do minim quis enim minim ipsum lorem labore amet veniam eiusmod do sed lorem veniam amet ipsum dolor sed magna adipiscing incididunt ut dolor dolor ipsum sit veniam tempor aliqua labore amet ad labore enim minim quis aliqua ad sed labore enim veniam sit sed adipiscing adipiscing ut incididunt dolor labore tempor incididunt lorem elit adipiscing ut aliqua veniam ipsum ut sed do ipsum adipiscing magna aliqua ut ipsum labore sed quis ut labore enim incididunt tempor tempor ipsum sit ut eiusmod ut eiusmod magna sit do adipiscing incididunt ipsum quis dolore amet ad do ut ad consectetur elit eiusmod magna minim ad veniam ut adipiscing sed elit sed dolor adipiscing sed quis elit eiusmod enim ipsum consectetur ad magna ut do minim dolor adipiscing adipiscing lorem ad amet magna labore consectetur sed enim dolor ut ut ad do elit s \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/README.md new file mode 100644 index 00000000..36ce0d09 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/README.md @@ -0,0 +1,17 @@ +# RAR 1.54 Unpack15 Fixtures + +Copied from the spec repository's `fixtures/1.54/` directory. +`expected/README.md` is the byte-for-byte payload expected from the +single-file README fixtures. + +| Fixture | Purpose | +|---|---| +| `readme_154_normal.rar` | Single-file compressed `README.md`, method `0x33`, `UNP_VER = 15`. | +| `readme_154_password.rar` | Same payload patched to set `FHD_PASSWORD` and encrypt the packed stream with CRYPT_RAR15, password `password`; validated by RAR 3.93. | +| `readme_154_store_solid.rar` | Single-file solid-archive flag variant; same payload. | +| `doc_154_best.rar` | Multi-file compressed text corpus, 17 entries. | +| `audio_win_names_unpack15.rar` | Audio-shaped WAV payload with Windows long names. | +| `audio_dos_names_unpack15.rar` | Same audio-shaped payload with DOS 8.3 names. | +| `random.rar` + `.r00` + `.r01` | Old-numbered multi-volume archive. | + +Expected `README.md` CRC32 is `0x509e5e3c`. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/audio_dos_names_unpack15.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/audio_dos_names_unpack15.rar new file mode 100644 index 00000000..4c07c78b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/audio_dos_names_unpack15.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/audio_win_names_unpack15.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/audio_win_names_unpack15.rar new file mode 100644 index 00000000..4070dd53 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/audio_win_names_unpack15.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/doc_154_best.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/doc_154_best.rar new file mode 100644 index 00000000..6dee0c87 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/doc_154_best.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/expected/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/expected/README.md new file mode 100644 index 00000000..3013952c --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/expected/README.md @@ -0,0 +1,65 @@ +# RAR Format & Implementation Docs + +Clean-room specifications and implementation notes for building a complete +RAR reader + writer across every format generation. Written for the `rars` +(RAR in Rust) project. + +Scope: RAR 1.3/1.4 (undocumented legacy), RAR 1.5–4.x (technote.txt era), +RAR 5.0/7.0 (current). "RAR 6" was a software release; "RAR 7" is RAR 5.0 +with a larger-dictionary algorithm revision. + +## Reading order + +Skim **§ Format specs** for the version you care about, then dive into +**§ Algorithms** and **§ Write-side** for the pieces referenced by the spec. +**§ Implementation** is the cross-cutting reader/writer companion material +that doesn't belong inside any single version spec. + +## Format specs (wire format, read + write) + +| Doc | Covers | +|-----|--------| +| [`RAR13_FORMAT_SPECIFICATION.md`](RAR13_FORMAT_SPECIFICATION.md) | RAR 1.3/1.4. `RE~^` signature, adaptive Huffman, full encoder + decoder. | +| [`RAR15_40_FORMAT_SPECIFICATION.md`](RAR15_40_FORMAT_SPECIFICATION.md) | RAR 1.5, 2.0, 2.9/3.x, 4.x. `Rar!` 7-byte signature, block/subblock formats, Unpack15/20/29 compression, RARVM filters, AV, recovery records. | +| [`RAR5_FORMAT_SPECIFICATION.md`](RAR5_FORMAT_SPECIFICATION.md) | RAR 5.0/7.0. 8-byte signature, vint encoding, extra-area records, service headers, Unpack50/70, hardcoded filter enum. | + +## Algorithms (version-agnostic primitives) + +| Doc | Covers | +|-----|--------| +| [`HUFFMAN_CONSTRUCTION.md`](HUFFMAN_CONSTRUCTION.md) | Length-limited canonical Huffman build; per-version wire packing. | +| [`LZ_MATCH_FINDING.md`](LZ_MATCH_FINDING.md) | HC4/BT4 match finders, greedy/lazy/optimal parsing, cost functions. | +| [`PPMD_ALGORITHM_SPECIFICATION.md`](PPMD_ALGORITHM_SPECIFICATION.md) | PPMd variant H: model, SEE, range coder, 7-Zip vs RAR variants. Used by RAR 3.x/4.x. | +| [`FILTER_TRANSFORMS.md`](FILTER_TRANSFORMS.md) | E8/E8E9/DELTA/ARM/ITANIUM/RGB/AUDIO forward transforms; RAR 3.x VM vs RAR 5.0 hardcoded dispatch. | +| [`RARVM_SPECIFICATION.md`](RARVM_SPECIFICATION.md) | Generic RAR 3.x/4.x VM bytecode, operands, instruction semantics, invocation globals, and safety limits for non-standard filters. | +| [`CRC32_SPECIFICATION.md`](CRC32_SPECIFICATION.md) | IEEE 802.3 CRC32 — parameters, table, test vectors. | + +## Write-side (encoder-specific decisions) + +The format specs cover the wire layout the decoder must accept. These docs +cover the choices an **encoder** must make that the format alone doesn't pin +down — emit order, flag selection, salt/IV policy, padding, streaming. + +| Doc | Covers | +|-----|--------| +| [`ARCHIVE_LEVEL_WRITE_SIDE.md`](ARCHIVE_LEVEL_WRITE_SIDE.md) | Block emit order, cross-reference graph, solid mode, multi-volume split, service headers, Quick Open cache, filesystem metadata, SFX stub. | +| [`ENCRYPTION_WRITE_SIDE.md`](ENCRYPTION_WRITE_SIDE.md) | All five encryption variants (RAR 1.3 through 5.0), key derivation, IV/salt rules, HMAC hashing, header encryption. | +| [`INTEGRITY_WRITE_SIDE.md`](INTEGRITY_WRITE_SIDE.md) | BLAKE2sp, 8-bit and 16-bit Reed–Solomon (`RSCoder`/`RSCoder16`), recovery-record layout, `.rev` files, per-block CRC ranges. | + +## Implementation (reader-side + cross-cutting) + +*To be written — see `IMPLEMENTATION_GAPS.md` for status.* + +| Doc | Covers | +|-----|--------| +| [`READ_SIDE_OVERVIEW.md`](READ_SIDE_OVERVIEW.md) | End-to-end parser walkthrough: type detection, SFX scan, version dispatch, block iteration, streaming, error handling. | +| [`PATH_SANITIZATION.md`](PATH_SANITIZATION.md) | Extraction-side security: traversal guards, drive letters, reserved names, symlink target validation. | +| [`TEST_VECTORS.md`](TEST_VECTORS.md) | Consolidated known-good samples for every primitive and each version's micro-archives. | +| [`IMPLEMENTATION_GAPS.md`](IMPLEMENTATION_GAPS.md) | Open items, including the few that genuinely need ghidra (filter heuristics, `-m0..-m5` tables, IV policy). | + +## Source references + +All source-level citations point into `_refs/` (gitignored): + +- `_refs/7zip/` — 7-Zip 25.01. Independent PPMd/Huffman/LzFind encoders and decoders. +- `_refs/XADMaster/` — The Unarchiver. Obj-C handlers for RAR 1.3–4.x, valuable for the undocumented 1.3 era. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.r00 new file mode 100644 index 00000000..836ec046 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.r01 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.r01 new file mode 100644 index 00000000..7b898d05 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.r01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.rar new file mode 100644 index 00000000..e04c4488 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/random.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_normal.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_normal.rar new file mode 100644 index 00000000..f1db48b0 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_normal.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_password.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_password.rar new file mode 100644 index 00000000..51f2a213 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_password.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_store_solid.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_store_solid.rar new file mode 100644 index 00000000..29e73da3 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar154/readme_154_store_solid.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/README.md new file mode 100644 index 00000000..b69928dc --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/README.md @@ -0,0 +1,28 @@ +# RAR 2.02 Fixtures + +Copied from the spec repository's `fixtures/2.02/` directory. + +These archives pin three RAR 2.x behaviors: + +- old-format main-header comments: the embedded comment subblock is included in + `HEAD_SIZE` but not in the main-header `HEAD_CRC`; +- old-format file comments: the same comment subblock, embedded in a file + header, with `HEAD_CRC` again covering only the fields the reader parses; +- RAR 2.0 `CRYPT_RAR20` encrypted compressed members (`comment_psw.rar`, + password `password`). + +unrar 7.20 counts one silent error per comment-bearing header in these files. +Both archives hold three such headers, the main-header comment and one per +file, and both report `Total errors: 3` and exit 3 while every member still +tests OK. So unrar checks the header CRC a second time somewhere past the +point that skips the comment bytes, and that second check has no matching +exemption. + +These are WinRAR 2.02's own archives, so a non-zero unrar exit code on a +comment-bearing RAR 2.x file is expected and not a sign our writer has +broken. + +Payloads: + +- `FILE1.TXT` = `file1\r\n`, CRC32 `0x7a197dba` +- `FILE2.TXT` = `file2\r\n`, CRC32 `0x785fc3e3` diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/comment_nopsw.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/comment_nopsw.rar new file mode 100644 index 00000000..329dc72a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/comment_nopsw.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/comment_psw.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/comment_psw.rar new file mode 100644 index 00000000..60fb14f4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar202/comment_psw.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/AUDIO.RAR b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/AUDIO.RAR new file mode 100644 index 00000000..5142bc82 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/AUDIO.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/AUTOREJ.RAR b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/AUTOREJ.RAR new file mode 100644 index 00000000..89058b91 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/AUTOREJ.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/BIGLZ.RAR b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/BIGLZ.RAR new file mode 100644 index 00000000..383f5095 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/BIGLZ.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/README.md new file mode 100644 index 00000000..c45d9205 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/README.md @@ -0,0 +1,31 @@ +# RAR 2.50 Unpack20 Fixtures + +Copied from the spec repository's `fixtures/2.50/` directory plus selected +external corpus samples promoted as regression fixtures. + +| Fixture | Purpose | +|---|---| +| `AUDIO.RAR` | RAR 2.50 `-mm` multimedia-switch fixture. Despite the name, the first table-read peek is `0x0040`, so bit 15 is clear and the member decodes through normal Unpack20 LZ. | +| `AUTOREJ.RAR` | Same `-mm` path on text input where the encoder rejects audio mode and emits normal Unpack20 LZ. | +| `BIGLZ.RAR` | Longer Unpack20 LZ history/table-stress stream. | +| `SOLID.RAR` | Solid Unpack20 state carry-over across two members. | +| `unpack20_audio_text.rar` | External `junrar` corpus archive with a WAV-named payload and stored text member. The first compressed member's table-read peek is `0x2221`, so it also exercises normal Unpack20 LZ rather than audio mode. | +| `unpack20_keep_tables.rar` | Unpack20 keep-table coverage. | +| `unpack20_multiblock.rar` | Explicit multiblock Unpack20 stream from the local corpus. | + +Known remaining gap: no vintage-encoder fixture currently proves a true +Unpack20 audio block. The test suite has synthetic one-channel audio coverage +at codec level and synthetic in-memory RAR 2.0 archive coverage for channel +counts 1, 2, 3, and 4, but RAR 2.50 `-mm[f]` probes with +mono/3-channel/4-channel WAV-shaped data and the historical `AUDIO.RAR` +fixture all selected normal LZ blocks. A useful future fixture must have bit +15 set in the first table-read peek word and should pin the selected channel +count. + +The spec repo's `scripts/find-rar20-audio-candidates.py` scans the local +external corpus, spec fixtures, promoted crate fixtures, and old numbered +volumes. Current result: 538 archive/volume files scanned, 37 raw bit-15 +candidates, 0 clean candidates. The candidates are stored, encrypted, +split-continuation, or solid-continuation false positives. `SOLID.RAR` member +2 intentionally pins one trap: raw data-start peek `0xdfbe`, but `LHD_SOLID` +means it is not a fresh table-read boundary. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/SOLID.RAR b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/SOLID.RAR new file mode 100644 index 00000000..22f1c423 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/SOLID.RAR differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_audio_text.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_audio_text.rar new file mode 100644 index 00000000..03bd2d4b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_audio_text.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_keep_tables.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_keep_tables.rar new file mode 100644 index 00000000..9c87f519 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_keep_tables.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_multiblock.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_multiblock.rar new file mode 100644 index 00000000..6336c82a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250/unpack20_multiblock.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250_protect_head_rr1.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250_protect_head_rr1.rar new file mode 100644 index 00000000..ecc3b161 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250_protect_head_rr1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250_protect_head_rr5.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250_protect_head_rr5.rar new file mode 100644 index 00000000..e806d62a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar250_protect_head_rr5.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/README.md new file mode 100644 index 00000000..20bf1ce7 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/README.md @@ -0,0 +1,22 @@ +# RAR 3.00 Container Fixtures + +RAR 3.00 fixtures for the RAR 1.5-4.x container. Most files are copied from the +spec repository's `fixtures/1.5-4.x/rar300/` and `fixtures/rarvm/archives-rar300/` +sets; several compact multivolume and solid fixtures are local regression +cases. + +| Fixture group | Purpose | +|---|---| +| `with_comment_rar300.rar` | Stored `hello.txt` plus RAR 3.x archive comment subblock. | +| `header_encrypted_multivol_rar300*`, `header_encrypted_newnaming_rar300*` | Header-encrypted old/new-numbered multi-volume extraction. | +| `compressed_text_rar300.rar` | Basic Unpack29 LZ compressed text member. | +| `solid_rar300.rar`, `solid_simple_rar300.rar` | Solid Unpack29 state and table reuse. | +| `multivol_*_rar300*` | Old/new RAR 3 volume naming flags and split-file extraction. | +| `stored_multivol_rar300*`, `compressed_multivol_prng_rar300*`, `encrypted_multivol_rar300*`, `encrypted_newnaming_rar300*` | Streaming stored, compressed, and AES-encrypted split-volume extraction. | +| `rev_oldstyle.*`, `rev_newstyle.*` | RAR 3.00 old-style and RAR 4.20 new-style `.rev` recovery-volume repair. | +| `rarvm_*_rar300.rar` | Standard RARVM filters: E8, E8E9, DELTA, ITANIUM, RGB, AUDIO. | +| `with_compressed_recovery_rar300.rar` | Derived from `with_recovery_rar300.rar` by recompressing the `RR` NewSub recovery payload with the local RAR29 literal encoder, setting method `0x33`, and recomputing the header CRC. Its packed stream ends with the specified new-file/new-table marker. | +| `with_compressed_recovery_header_synthetic.rar` | Derived from `with_recovery_rar300.rar` with the `RR` NewSub method byte changed from store (`0x30`) to compressed (`0x33`) and the header CRC recomputed, but with the stored payload left unchanged. It pins corrupt compressed-RR error handling. | + +Expected payloads and CRCs are asserted directly in +`rar15_40_fixtures.rs`. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/bigtext_64k.bin b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/bigtext_64k.bin new file mode 100644 index 00000000..2416fd64 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/bigtext_64k.bin @@ -0,0 +1 @@ +amet dolor sed sit et labore et incididunt adipiscing sit et lorem incididunt ut lorem labore sed elit sit eiusmod lorem lorem lorem magna lorem incididunt adipiscing ut lorem dolore elit labore et magna elit tempor elit elit labore do lorem ut magna sit consectetur do sit eiusmod dolore ut dolore adipiscing do do et dolore incididunt ipsum et elit incididunt ut consectetur tempor magna tempor dolor labore dolore sit consectetur dolore incididunt tempor et lorem et ipsum do incididunt consectetur consectetur dolore elit lorem adipiscing magna magna elit incididunt dolore tempor tempor labore sed magna lorem incididunt dolore amet dolore magna adipiscing ut ipsum et tempor magna adipiscing dolore ut et tempor ut tempor lorem magna magna eiusmod labore lorem elit consectetur magna consectetur dolor magna sed ipsum dolor dolor lorem labore lorem sed elit sed sit consectetur tempor do dolor consectetur consectetur sed dolore consectetur sed do labore eiusmod et et sit lorem do incididunt eiusmod ut adipiscing sed sit sed dolore adipiscing ut lorem elit lorem incididunt amet ipsum consectetur labore dolore ut magna elit dolore labore elit dolore lorem incididunt eiusmod ut ipsum do amet adipiscing ipsum do dolor dolor do do consectetur ut sed amet lorem magna ipsum adipiscing labore consectetur dolore ipsum incididunt adipiscing tempor sit adipiscing ut adipiscing et sit incididunt do dolore et lorem eiusmod incididunt do lorem consectetur adipiscing eiusmod amet eiusmod ut adipiscing sed sit incididunt magna tempor magna et magna elit dolor ipsum dolor amet consectetur consectetur magna adipiscing sed eiusmod dolore sed tempor eiusmod eiusmod sit do elit et amet magna sit eiusmod ipsum ut dolor incididunt amet amet eiusmod sit incididunt dolor magna elit dolor sed tempor do magna sit labore sed sit ipsum do lorem lorem dolor ut sit ipsum adipiscing elit ut consectetur sit labore consectetur elit consectetur sit ut incididunt magna do magna sed et eiusmod sit adipiscing eiusmod ipsum lorem lorem do eiusmod labore incididunt eiusmod incididunt dolor dolor eiusmod labore sit sed adipiscing magna et tempor sed consectetur magna adipiscing do adipiscing elit tempor dolor sed dolor labore dolor eiusmod elit incididunt do ipsum eiusmod consectetur eiusmod do elit eiusmod sit magna dolor elit elit lorem elit incididunt dolor sed magna dolor dolor lorem lorem do tempor et et amet sit dolore eiusmod dolor dolore consectetur consectetur amet amet eiusmod do sit dolore do amet adipiscing amet magna ipsum eiusmod magna adipiscing consectetur do ut magna consectetur ipsum elit sed dolor labore ut magna sed magna labore magna labore lorem incididunt eiusmod consectetur sed et lorem ut lorem ipsum tempor amet amet amet sed sed incididunt incididunt consectetur dolor elit et lorem consectetur dolore eiusmod dolore labore elit elit eiusmod et et elit ut eiusmod magna sed elit ipsum dolor dolore tempor consectetur dolore adipiscing do do do magna tempor consectetur labore dolor sit dolore incididunt consectetur amet sed ut adipiscing ipsum et incididunt tempor incididunt dolore consectetur magna ipsum dolore dolor sed sit sed dolor amet dolor labore elit incididunt ut incididunt consectetur eiusmod labore amet et adipiscing sit ut magna ut sit do sed elit incididunt magna lorem adipiscing dolore labore lorem lorem elit sed adipiscing consectetur do amet magna adipiscing sed do sed labore consectetur magna tempor et ut sit adipiscing incididunt adipiscing do sit lorem sit lorem magna do amet dolor dolore tempor do ut dolore tempor dolore eiusmod lorem sit labore labore tempor do magna incididunt eiusmod et sit incididunt incididunt adipiscing magna lorem sed dolore adipiscing labore dolore ut do consectetur labore dolore adipiscing tempor dolore lorem incididunt ut incididunt eiusmod dolor et elit do lorem ut amet incididunt sed consectetur dolor lorem tempor sed ut magna do amet labore sed et consectetur labore dolore ipsum sed dolore sit ut dolor tempor dolor labore lorem consectetur dolore consectetur dolor incididunt sed do adipiscing dolore adipiscing elit eiusmod sed dolor dolor dolore tempor labore dolore magna ipsum consectetur do magna sed tempor elit incididunt magna incididunt consectetur et sed eiusmod elit sed elit lorem incididunt eiusmod ut elit sed adipiscing dolor consectetur labore amet sed labore dolore consectetur amet amet labore tempor do incididunt elit sit adipiscing do dolor sit elit incididunt eiusmod et sit consectetur ipsum ipsum lorem adipiscing ipsum et dolore labore eiusmod sed sit consectetur sit elit incididunt elit et labore incididunt consectetur elit elit do labore magna incididunt adipiscing labore sed eiusmod et sit adipiscing dolor ipsum lorem lorem et eiusmod incididunt do adipiscing incididunt consectetur amet lorem lorem incididunt amet magna ipsum incididunt sed amet dolor labore do lorem ipsum magna ipsum dolore amet ipsum sed sit ut dolor adipiscing lorem et amet sed adipiscing labore incididunt eiusmod sed sed elit elit ipsum consectetur tempor ut magna dolore ipsum tempor magna ut magna adipiscing magna ut dolor sed dolor sed consectetur sit amet ipsum adipiscing ut ipsum ipsum dolor dolore et dolore tempor sit eiusmod ipsum amet magna ipsum labore amet incididunt labore lorem dolore sed dolor sed eiusmod dolor do ipsum incididunt ipsum sed eiusmod amet sed incididunt sit do sit ut elit dolore magna adipiscing eiusmod eiusmod dolore incididunt et sit amet labore dolore magna dolore magna lorem do consectetur adipiscing tempor incididunt dolore eiusmod sit ut tempor amet dolor ipsum do magna eiusmod ut do eiusmod tempor sed eiusmod dolore dolore lorem dolore sit amet eiusmod eiusmod eiusmod dolor labore sed et labore tempor incididunt dolor ipsum amet ipsum dolore et sed elit eiusmod tempor tempor incididunt do labore eiusmod magna dolore consectetur lorem amet sed elit amet sit consectetur ut ipsum sit magna sed sit adipiscing sed dolor dolore dolor dolor adipiscing consectetur dolore ut lorem tempor et do elit adipiscing et elit ut labore tempor magna adipiscing et dolor sed ut adipiscing lorem magna incididunt dolore et dolor incididunt dolore ut ipsum tempor labore lorem adipiscing do lorem magna sit do dolore eiusmod magna magna do dolore ut magna dolore ut do labore do amet dolore labore amet magna consectetur sed lorem ut ipsum tempor ut incididunt do lorem dolor dolor lorem incididunt sed labore sed tempor et eiusmod incididunt labore sit et tempor amet ut amet lorem consectetur sed tempor amet do ut sed dolore do ut sed ut eiusmod et adipiscing et incididunt ut dolor dolor amet adipiscing amet elit lorem sit sed amet et sit incididunt consectetur lorem dolor ut ipsum magna adipiscing magna ut tempor ipsum sit magna ut sit sed sed consectetur et ipsum adipiscing dolor incididunt sit labore do dolore et incididunt sit et sit amet incididunt adipiscing consectetur dolore sed ut magna do et magna adipiscing eiusmod et sit lorem tempor sed ipsum magna labore do sit elit dolore sed sed elit ut amet amet sed adipiscing ut magna ipsum magna dolore amet ut sed sed et do sed et adipiscing et tempor et elit eiusmod consectetur consectetur labore magna amet ipsum dolore eiusmod dolore amet adipiscing eiusmod et et eiusmod sit amet amet sed elit dolor magna ipsum consectetur sit elit adipiscing dolore do ut eiusmod lorem lorem do elit dolor elit sed eiusmod sed dolore incididunt lorem sit eiusmod tempor amet sit sed amet ipsum tempor dolor dolor sit do eiusmod elit sed dolore ipsum tempor lorem dolor amet incididunt tempor elit sit eiusmod sed lorem dolore eiusmod sit tempor amet sed incididunt dolor dolore et ut magna incididunt do elit do magna amet ipsum dolore sit consectetur elit adipiscing ut sed magna lorem sed magna sed dolore sed et amet incididunt sit tempor dolor magna tempor magna magna dolore lorem do labore amet amet dolor amet adipiscing et eiusmod tempor do consectetur amet incididunt labore incididunt sit amet sed do lorem magna lorem amet incididunt magna sit labore lorem ut ut sed tempor ut incididunt labore ipsum sit et ipsum lorem ipsum sit amet dolore dolore tempor magna sed tempor et elit elit sit magna tempor consectetur sit ipsum eiusmod ut tempor sed ipsum ut ut incididunt tempor do eiusmod labore elit dolore amet ipsum eiusmod sit dolore consectetur magna et eiusmod sit lorem et adipiscing incididunt consectetur incididunt elit sit elit eiusmod eiusmod elit labore et tempor et adipiscing ut labore incididunt magna sit et sed amet amet lorem incididunt ut sit lorem dolor consectetur labore incididunt dolore do amet amet dolore sit sed lorem labore incididunt elit magna incididunt lorem magna elit ut consectetur consectetur eiusmod elit dolor magna magna consectetur consectetur incididunt lorem dolore adipiscing ut elit ipsum dolore adipiscing dolore magna dolor elit incididunt labore sit ipsum incididunt dolor magna sit et ipsum dolore elit lorem lorem do labore sed ut consectetur amet magna eiusmod magna labore dolore ut magna consectetur incididunt incididunt adipiscing et sed tempor amet sed sed consectetur dolor tempor eiusmod amet sed sed sed tempor incididunt sed labore lorem amet amet sed elit adipiscing dolor magna adipiscing magna ut elit amet magna labore incididunt adipiscing dolor dolor amet ipsum lorem incididunt incididunt ut amet amet magna magna dolor elit incididunt amet do adipiscing incididunt tempor consectetur elit do amet tempor et magna do dolor dolore do adipiscing labore lorem do sit tempor labore sed ipsum ipsum eiusmod consectetur amet sit sit ut elit adipiscing dolore dolore incididunt sit adipiscing incididunt dolore amet sed lorem sit adipiscing incididunt et magna elit sed ipsum consectetur magna dolore elit ut sed ut incididunt sed et sit amet consectetur magna lorem labore ipsum et adipiscing incididunt magna eiusmod elit sit dolor ipsum ut labore adipiscing consectetur dolore adipiscing dolore incididunt dolore tempor adipiscing elit tempor dolor eiusmod ipsum labore ipsum consectetur amet do et ipsum dolore dolor incididunt dolor incididunt dolore do incididunt sed tempor et ipsum magna et lorem ut do eiusmod amet magna sed dolor tempor ut incididunt dolore lorem sit ipsum dolore lorem sit eiusmod eiusmod tempor magna ipsum tempor dolor et dolor magna labore eiusmod dolore magna lorem consectetur eiusmod tempor adipiscing amet amet sit incididunt eiusmod dolore ut tempor eiusmod sed tempor ipsum dolor elit sed incididunt magna do dolor dolor consectetur sed ut dolor amet do magna sed elit adipiscing sit sed et ipsum dolore do adipiscing magna dolor magna eiusmod eiusmod do dolore amet ipsum labore tempor ipsum lorem eiusmod ut consectetur magna ipsum dolore ut consectetur adipiscing elit sit amet dolore sit sed labore adipiscing ipsum tempor labore eiusmod tempor elit lorem lorem et ipsum consectetur sed magna ipsum lorem elit dolor dolore consectetur ipsum dolore adipiscing adipiscing labore do elit et dolore tempor eiusmod incididunt dolor adipiscing consectetur adipiscing do ut et tempor lorem et lorem sit ut eiusmod eiusmod dolor ut adipiscing dolore et magna dolore et labore et consectetur sed dolore do incididunt magna sed sed do lorem ipsum labore labore tempor elit dolore labore adipiscing et eiusmod amet incididunt ut ipsum sit tempor lorem sed magna ipsum do incididunt lorem eiusmod eiusmod do ipsum adipiscing dolor eiusmod sit dolor amet do ut eiusmod elit lorem consectetur dolore tempor do do incididunt ut dolore labore dolor adipiscing ut elit ipsum elit elit elit incididunt incididunt adipiscing amet do tempor lorem do labore et consectetur amet lorem tempor ut magna eiusmod dolore et eiusmod sit do magna sed ut lorem do dolor et sit dolore elit sed ut tempor elit ipsum sit dolore dolore dolore consectetur amet do ipsum dolor adipiscing lorem ipsum ut lorem dolor ipsum lorem ipsum magna eiusmod eiusmod lorem lorem magna adipiscing et adipiscing sed do magna dolore sed elit consectetur adipiscing incididunt ipsum elit magna labore ipsum eiusmod eiusmod ut sit lorem consectetur dolore dolor consectetur adipiscing elit consectetur do sit ipsum eiusmod amet dolor labore amet elit ipsum do tempor ipsum dolor labore adipiscing elit consectetur sit ipsum adipiscing ipsum sit dolor elit do sed dolore ut elit ipsum sed adipiscing eiusmod tempor tempor labore incididunt incididunt dolor ut elit et eiusmod consectetur sit elit dolor ut sed magna do eiusmod tempor ut labore tempor tempor eiusmod incididunt et dolore lorem tempor amet do consectetur do amet magna amet consectetur labore amet amet consectetur dolor sed elit tempor eiusmod consectetur sed et do dolor ut amet magna tempor labore sit amet eiusmod dolor consectetur et magna ipsum ipsum adipiscing tempor tempor dolore tempor dolore tempor eiusmod sit consectetur incididunt ipsum sed adipiscing ipsum elit do eiusmod incididunt elit tempor ipsum elit do lorem adipiscing sit amet elit tempor dolore sed amet consectetur elit dolor do dolore dolore magna magna ut labore dolore et consectetur dolore tempor adipiscing ut dolor sed adipiscing elit amet amet adipiscing lorem consectetur et tempor consectetur ipsum tempor dolor elit adipiscing dolor labore adipiscing eiusmod consectetur lorem adipiscing eiusmod et magna ipsum ipsum tempor et magna tempor amet et dolor dolore eiusmod do eiusmod dolor et eiusmod ut dolor sed dolor eiusmod lorem consectetur eiusmod elit eiusmod sed sed do et ut lorem do consectetur do ipsum sit ut ut adipiscing sed tempor et do sed consectetur eiusmod amet tempor sit incididunt tempor dolore adipiscing incididunt labore amet et elit ipsum elit dolor dolor ipsum dolore dolore et et eiusmod dolore consectetur et incididunt lorem incididunt magna magna labore consectetur tempor ipsum tempor tempor labore elit magna do dolor labore tempor adipiscing consectetur amet labore ipsum tempor eiusmod consectetur et et lorem elit ipsum labore consectetur dolore adipiscing incididunt labore sit eiusmod sed amet consectetur eiusmod amet consectetur dolore do elit magna ut labore labore dolore magna do consectetur dolore dolore do adipiscing do amet lorem eiusmod sit ut incididunt dolore consectetur labore labore magna labore tempor adipiscing ipsum dolor sit sit magna incididunt amet labore incididunt consectetur et labore dolore ipsum adipiscing labore et incididunt do tempor consectetur sed consectetur lorem magna ipsum dolor magna elit labore eiusmod labore eiusmod sit incididunt ipsum labore sed ut labore eiusmod dolore sit consectetur incididunt magna ut et dolore amet eiusmod amet tempor amet adipiscing elit adipiscing labore amet sit sit ut ipsum labore amet tempor magna eiusmod sed incididunt lorem incididunt et labore do do incididunt eiusmod do consectetur sit et consectetur labore amet labore sit magna sit magna eiusmod eiusmod et magna eiusmod eiusmod magna labore eiusmod et incididunt magna adipiscing consectetur elit magna adipiscing elit ipsum eiusmod ipsum eiusmod ut lorem tempor tempor tempor ut adipiscing do elit eiusmod incididunt incididunt consectetur lorem incididunt tempor elit elit dolor eiusmod incididunt adipiscing do sit ut lorem tempor dolor ut amet sit magna consectetur eiusmod amet incididunt ut eiusmod magna dolore sed adipiscing adipiscing consectetur consectetur magna consectetur amet sit labore dolore amet ut amet eiusmod eiusmod amet lorem tempor consectetur elit elit et et ipsum dolor amet magna et amet adipiscing tempor amet sed tempor dolor incididunt et lorem dolore labore adipiscing elit adipiscing lorem do ipsum sed dolore adipiscing dolor sit sit incididunt eiusmod sit labore dolore et sed amet ut tempor tempor incididunt ut ut tempor magna adipiscing adipiscing dolor amet elit elit lorem elit incididunt labore labore sit ipsum consectetur dolore lorem ipsum ut sed ut amet elit tempor ut eiusmod ipsum dolore labore amet dolore tempor ipsum tempor sit elit sit ut amet lorem tempor amet amet do lorem et lorem et dolor ut dolor et magna dolore sit amet magna incididunt magna ut elit dolore incididunt et eiusmod labore sit dolor adipiscing tempor sit sit tempor sit adipiscing sit dolor lorem dolore ut elit dolor do et ipsum ut magna do incididunt ipsum lorem sed et labore elit sed eiusmod et labore magna ipsum sed dolore consectetur labore labore do consectetur eiusmod dolore incididunt ut magna incididunt et elit do lorem dolor amet et sit tempor sed do magna do amet sit dolore amet labore ipsum labore et eiusmod magna tempor amet lorem magna adipiscing sed dolor labore do lorem sed dolore lorem incididunt sit sit eiusmod labore dolor et dolore eiusmod ipsum adipiscing consectetur ipsum sit ipsum sit magna dolore do adipiscing consectetur magna amet elit adipiscing dolor dolore tempor ut sed amet do elit dolor sed ipsum lorem ut do et ut ut dolor consectetur adipiscing ipsum ut ut tempor tempor dolore amet consectetur elit elit ipsum tempor dolor labore eiusmod adipiscing elit sed amet dolore incididunt sit et lorem et do sed do adipiscing amet incididunt ipsum incididunt labore magna lorem amet elit et sit do ut adipiscing dolore eiusmod sit elit elit et sit consectetur et tempor ut incididunt magna ut lorem incididunt amet ut amet ipsum do incididunt ut sit adipiscing sed et ut sed dolore sit eiusmod amet magna magna sed lorem magna sit tempor labore sed sit do amet dolor ut incididunt lorem et amet magna incididunt et elit dolore lorem incididunt ipsum ut dolor elit ipsum labore dolor do ipsum tempor ipsum dolor dolor ipsum do tempor do dolor magna et tempor eiusmod consectetur tempor dolore elit eiusmod elit elit adipiscing do do magna eiusmod do lorem et sed elit amet elit consectetur dolor sed incididunt adipiscing amet consectetur magna dolor eiusmod incididunt adipiscing consectetur ipsum labore adipiscing incididunt sit do elit do dolore labore eiusmod dolor dolor dolor elit sit dolore labore magna labore lorem consectetur labore ut magna sit adipiscing lorem elit do adipiscing dolore do do sed tempor sed do ipsum lorem lorem labore ipsum adipiscing dolor eiusmod labore do sit elit sit adipiscing lorem adipiscing amet lorem labore lorem magna elit et consectetur magna lorem elit amet dolor lorem amet eiusmod dolor dolore magna sed adipiscing incididunt lorem magna sed tempor sed magna incididunt incididunt dolore dolore magna labore sed dolor consectetur et incididunt amet adipiscing dolore lorem dolore ipsum eiusmod amet elit eiusmod incididunt ipsum ut et dolore dolor ipsum amet magna ut magna incididunt magna sed ipsum adipiscing adipiscing do incididunt do dolore lorem sed adipiscing magna dolore magna consectetur elit dolor adipiscing et consectetur ipsum incididunt do lorem amet sit ipsum ut et consectetur adipiscing labore sit incididunt elit dolor amet eiusmod dolore et et dolore tempor ut elit labore sed incididunt tempor incididunt elit incididunt sit consectetur tempor dolor lorem ut et ipsum labore sit elit labore tempor dolore dolor eiusmod ipsum sed dolore eiusmod amet consectetur ut do labore elit et incididunt lorem dolore sed sit do sed lorem dolor eiusmod dolore consectetur elit do dolor consectetur labore tempor incididunt labore et sit et dolor ipsum ipsum lorem sed ipsum sed do consectetur magna et eiusmod lorem labore eiusmod elit elit tempor ipsum lorem labore dolore adipiscing incididunt amet consectetur elit dolor incididunt ipsum consectetur eiusmod lorem labore magna dolore consectetur ipsum ut elit sed dolore labore adipiscing ipsum incididunt ut incididunt dolore ut sed labore eiusmod lorem dolor et ut consectetur ut consectetur magna dolore dolore dolore consectetur sed ut et do tempor labore incididunt magna incididunt do elit tempor magna magna dolore elit sed lorem dolor sed incididunt consectetur sed sed et lorem consectetur et sit elit amet sit incididunt ipsum consectetur dolor sit labore magna labore lorem ipsum sed ipsum dolore et adipiscing tempor labore sit eiusmod eiusmod incididunt incididunt do dolor elit labore magna tempor ut ut ut sed consectetur amet ipsum eiusmod tempor incididunt dolor eiusmod consectetur amet sit magna adipiscing et elit tempor dolore consectetur adipiscing do consectetur amet incididunt ut et tempor ipsum magna dolor lorem tempor elit amet adipiscing incididunt labore dolore sed ut eiusmod et eiusmod dolor ipsum amet magna et consectetur dolor lorem dolor lorem consectetur sed adipiscing labore incididunt magna dolore sed sed magna incididunt sit incididunt labore elit dolor eiusmod amet lorem incididunt ipsum do tempor lorem labore eiusmod lorem magna eiusmod incididunt ipsum labore sit ut incididunt sit lorem lorem magna ut tempor consectetur incididunt ipsum amet do dolore ut consectetur et do sed ipsum incididunt magna ut amet eiusmod consectetur labore incididunt magna amet dolore dolor incididunt sed incididunt et ipsum do consectetur sed incididunt sed sit sed lorem sit sit labore amet labore elit elit ipsum elit dolor sit sit ipsum sit ipsum sed ut amet tempor sit ipsum incididunt elit consectetur magna et consectetur tempor incididunt dolore consectetur eiusmod dolore dolor ipsum lorem do sit labore dolor lorem ipsum sed magna do sed labore incididunt sit elit do amet dolore dolore lorem tempor labore sit ut amet sed sit tempor sed adipiscing eiusmod amet magna elit lorem elit et tempor amet ut eiusmod ut labore sit sed ipsum dolore do dolore eiusmod adipiscing adipiscing elit elit incididunt tempor sed lorem et dolore amet ut et dolor dolore sed sit elit sit ut incididunt amet sit labore dolore adipiscing consectetur adipiscing sed tempor eiusmod tempor sed amet lorem elit sed et magna lorem eiusmod lorem consectetur adipiscing sed elit dolor ut tempor tempor adipiscing sit lorem incididunt eiusmod eiusmod ut eiusmod sed incididunt sed tempor dolor ut elit et tempor do lorem sit dolore ipsum consectetur elit magna labore do ut incididunt lorem dolor incididunt amet adipiscing et incididunt et sit ut consectetur et adipiscing do magna ipsum do do amet sed dolore do et amet ut eiusmod dolore eiusmod adipiscing sed ipsum do dolore do et do sed consectetur do sed eiusmod amet sed incididunt labore et consectetur incididunt ipsum dolor adipiscing eiusmod ipsum dolore do ipsum ut sit eiusmod amet lorem tempor elit tempor dolore ut elit dolore dolor ipsum eiusmod lorem labore lorem consectetur sed adipiscing ut do consectetur ipsum ipsum et incididunt magna sit incididunt do ut ipsum elit eiusmod ut et adipiscing dolore dolor eiusmod incididunt consectetur elit dolore et dolor ut incididunt adipiscing sed lorem do ipsum sed dolor consectetur sed labore ut do sit do ipsum et consectetur sed magna adipiscing amet ipsum incididunt magna lorem dolore do lorem incididunt eiusmod sit sed consectetur adipiscing dolor consectetur incididunt dolore lorem elit incididunt lorem lorem dolore ut consectetur ipsum incididunt ut adipiscing consectetur elit dolor labore magna magna eiusmod sed adipiscing dolore sed incididunt elit do sed amet sed tempor sed dolore elit adipiscing magna lorem sit adipiscing sed consectetur eiusmod elit consectetur ipsum elit incididunt sed sed adipiscing sed incididunt ipsum ipsum amet et ut do tempor incididunt tempor adipiscing do sed sed et amet tempor amet incididunt ipsum dolor sed dolor et adipiscing labore do ipsum sed eiusmod lorem et ut ut ut tempor et adipiscing ut incididunt do sit dolor consectetur eiusmod tempor ut incididunt sit incididunt ipsum ut adipiscing sit elit et incididunt consectetur amet elit sit tempor eiusmod dolore labore consectetur incididunt et consectetur ipsum dolore adipiscing elit amet sit sed magna lorem lorem tempor do adipiscing ipsum do amet amet dolor consectetur ut sed amet dolor adipiscing consectetur ut adipiscing incididunt magna et consectetur dolor et elit adipiscing dolor amet elit adipiscing amet magna sed dolor tempor dolor tempor dolore sed consectetur et ut magna magna elit amet magna sit ut ut tempor elit labore magna incididunt eiusmod consectetur ipsum ipsum tempor labore consectetur labore tempor tempor amet labore adipiscing magna et magna do adipiscing amet elit do sit dolor elit ut dolore adipiscing do et ipsum incididunt adipiscing ipsum do do adipiscing ut lorem labore eiusmod ut elit sit consectetur dolore ipsum incididunt consectetur lorem dolore et et tempor magna ut lorem ut incididunt elit dolore lorem magna ipsum adipiscing eiusmod ipsum amet labore consectetur labore amet amet sed dolore incididunt dolor dolore sit dolor incididunt et tempor adipiscing ipsum dolore ut elit et adipiscing consectetur elit adipiscing dolore eiusmod do et dolore elit adipiscing do sit lorem ipsum eiusmod dolor dolore consectetur labore magna dolor ut amet magna ipsum amet eiusmod tempor labore adipiscing incididunt labore dolor incididunt tempor lorem do adipiscing tempor tempor lorem sit labore incididunt do consectetur do elit eiusmod eiusmod adipiscing ipsum ipsum lorem consectetur labore eiusmod ipsum sed dolore sed dolor elit lorem amet ut tempor sed et ipsum incididunt sit do ut elit elit dolore ut sed lorem lorem amet et amet tempor dolor elit magna magna amet amet incididunt amet eiusmod adipiscing amet amet sit amet ipsum sed sed tempor lorem amet lorem dolor labore ut incididunt do amet incididunt ut tempor labore tempor do et consectetur sed lorem do elit ipsum et ipsum lorem dolor labore lorem elit amet incididunt incididunt elit sed consectetur adipiscing consectetur dolor eiusmod tempor dolor sit magna elit adipiscing eiusmod magna magna labore dolor ut tempor consectetur consectetur sit tempor consectetur et dolor labore ut adipiscing dolor dolor sed eiusmod incididunt tempor eiusmod ut dolore dolor adipiscing ut tempor dolore et tempor sit labore eiusmod lorem elit do ut amet adipiscing sed dolore ipsum consectetur do ipsum sit sed sit consectetur magna labore elit labore ut ipsum amet et tempor dolore do incididunt dolor ut amet dolore elit ut et dolor tempor magna dolore consectetur ipsum adipiscing adipiscing lorem tempor elit elit dolore dolore ut magna ut consectetur elit lorem elit dolore magna ipsum amet magna dolor lorem amet magna sed elit tempor eiusmod amet sit sed ut tempor ipsum magna ipsum labore ipsum eiusmod do do incididunt do incididunt et do sit lorem sit ut dolor adipiscing sit lorem elit et dolor adipiscing eiusmod adipiscing do do labore labore magna dolore adipiscing labore incididunt dolor lorem dolor do labore adipiscing do ut consectetur incididunt incididunt labore elit elit et lorem do sed et et tempor sit sit adipiscing labore incididunt adipiscing ut ipsum consectetur incididunt ut tempor dolore amet dolor dolore consectetur ipsum adipiscing dolore magna labore do do et amet lorem labore ut tempor ut tempor adipiscing sed adipiscing labore et sed ut do sed labore dolor sit eiusmod labore do dolore elit dolore eiusmod elit amet consectetur sed elit ut lorem ut incididunt elit amet dolor dolor consectetur labore incididunt elit do incididunt sed lorem do amet sit ut do do ut magna ipsum amet sit consectetur dolore et ut sit ipsum tempor eiusmod do ipsum do labore ipsum tempor do magna adipiscing sed sed consectetur do dolore eiusmod magna dolor ipsum amet amet incididunt eiusmod eiusmod et consectetur do lorem sed lorem magna lorem ut magna labore lorem dolore incididunt sit sit lorem incididunt dolor et adipiscing tempor ipsum ut et magna eiusmod adipiscing lorem amet et et sed ut dolore sit ut labore magna dolore do dolor ipsum ut amet tempor adipiscing dolor labore tempor sit eiusmod sit adipiscing eiusmod consectetur consectetur eiusmod dolor adipiscing do magna dolor et labore dolore labore incididunt tempor dolore dolore et consectetur amet lorem consectetur do consectetur amet adipiscing amet elit labore amet dolor et dolore magna incididunt incididunt ut magna dolore ut et sed et amet adipiscing incididunt ipsum sed amet labore adipiscing amet incididunt labore ipsum tempor elit amet do et eiusmod amet dolor incididunt dolor dolor lorem lorem dolor dolor amet magna sed ipsum adipiscing ut eiusmod sed tempor adipiscing consectetur ut dolor tempor sit ut labore eiusmod dolore sit lorem ipsum amet ut adipiscing adipiscing dolor consectetur labore dolore lorem eiusmod do do amet labore ipsum ipsum do consectetur lorem eiusmod lorem amet sed sit elit sed et et adipiscing dolor amet do lorem elit consectetur consectetur elit labore sit lorem adipiscing tempor consectetur sed sit dolor do elit incididunt do magna amet do amet do magna sit do dolore sit adipiscing labore incididunt sit lorem incididunt et lorem do et et tempor consectetur adipiscing et magna adipiscing dolore magna elit amet ut adipiscing tempor et elit ipsum elit sit tempor dolor ipsum adipiscing ut eiusmod ut labore labore labore labore tempor ipsum adipiscing sed amet dolore sit ut adipiscing eiusmod sit lorem elit adipiscing incididunt adipiscing do do tempor elit lorem elit sed do consectetur sit lorem tempor amet incididunt et labore sit elit tempor ipsum dolor elit consectetur adipiscing ut amet incididunt incididunt tempor dolor ipsum magna labore tempor do tempor eiusmod tempor lorem sit incididunt do sed ipsum dolore et sed ipsum et eiusmod ut labore magna dolore elit consectetur dolore ipsum incididunt sed adipiscing eiusmod adipiscing sit consectetur ut ut sed amet dolor sed elit sed consectetur labore ut lorem amet do dolore amet amet ut ipsum labore dolore et dolore ipsum incididunt sit do ut labore sit magna ut ut lorem sed ipsum do sed eiusmod dolore lorem amet magna ipsum adipiscing eiusmod sit consectetur do ut dolore amet dolor dolore et dolore dolor ut tempor amet labore incididunt eiusmod do et magna dolor dolore amet lorem dolor adipiscing tempor et dolore amet consectetur amet ut ipsum dolor tempor do sed tempor tempor do ut incididunt et labore tempor eiusmod ut amet amet et sed sed ut et ipsum do dolore et tempor et amet labore amet et amet elit eiusmod dolor magna tempor consectetur ut ut do sed elit lorem et tempor dolor sed et incididunt labore ipsum ut sed et dolore amet eiusmod amet adipiscing incididunt sit sit eiusmod amet et dolore labore dolore amet et amet ipsum adipiscing incididunt eiusmod sed et do ipsum ut dolor adipiscing lorem eiusmod incididunt do elit do ut tempor labore eiusmod sed sit tempor ipsum do adipiscing sit sed lorem sed eiusmod sit incididunt labore do magna sed tempor amet et ipsum sit incididunt labore eiusmod lorem labore elit amet eiusmod amet adipiscing et consectetur do tempor elit adipiscing ipsum labore sed labore labore et tempor ut ut dolor elit tempor ipsum eiusmod sit incididunt magna et eiusmod amet adipiscing dolor sed et et labore eiusmod labore magna dolor et sed magna sit amet ut dolor amet incididunt sit ut magna adipiscing amet lorem magna sit sit dolor eiusmod sit magna tempor ut incididunt dolor ut labore do ut labore incididunt magna amet labore et incididunt sit eiusmod sed amet sed ipsum sit consectetur ipsum lorem eiusmod eiusmod ipsum consectetur ut labore lorem ut magna incididunt sit incididunt ipsum lorem amet elit dolore et incididunt eiusmod amet do amet amet magna amet amet elit adipiscing lorem adipiscing et magna labore tempor et ut dolore lorem ut elit incididunt sed lorem dolore eiusmod sit dolore elit ut sed amet dolore labore magna tempor sed sit dolor tempor incididunt labore tempor ut dolore labore et incididunt magna lorem ipsum et eiusmod elit sit lorem tempor dolor consectetur et sit elit amet labore dolor consectetur sed et tempor consectetur elit ipsum do labore magna magna labore do elit lorem do et amet adipiscing adipiscing consectetur do dolor sed consectetur incididunt do consectetur labore incididunt dolor tempor tempor elit lorem magna consectetur labore amet do incididunt amet eiusmod eiusmod sed lorem amet adipiscing sed adipiscing lorem ipsum ipsum labore do elit dolore sit dolor amet adipiscing consectetur lorem ut consectetur consectetur labore eiusmod ipsum labore dolore tempor do amet magna dolore ipsum elit sit labore et elit dolore eiusmod sit do amet sed ut sed sit lorem lorem tempor dolore consectetur ipsum eiusmod consectetur ipsum ipsum lorem sed elit lorem do et et eiusmod dolor adipiscing consectetur tempor dolor tempor amet tempor adipiscing labore incididunt labore tempor dolor elit elit sit elit dolor do dolore tempor dolor sit tempor elit magna do do sit consectetur ut adipiscing labore amet adipiscing dolor ut dolor incididunt amet elit do sed dolore et incididunt sit sit tempor labore et incididunt adipiscing eiusmod amet sed eiusmod adipiscing ipsum incididunt sed eiusmod sit amet adipiscing tempor et eiusmod ipsum ipsum labore amet labore et dolore adipiscing sed sed amet do sit eiusmod elit eiusmod eiusmod adipiscing sit ut adipiscing ut amet dolore magna dolor incididunt lorem incididunt amet incididunt tempor dolore incididunt amet dolor dolore elit adipiscing et et incididunt eiusmod sed magna lorem dolore dolore sed lorem incididunt dolore incididunt amet consectetur sed sit ut sit et incididunt ipsum et dolor dolor elit eiusmod incididunt eiusmod tempor sed elit ut amet elit incididunt ipsum elit adipiscing amet eiusmod consectetur sit adipiscing ut ipsum dolore eiusmod magna tempor tempor ut elit tempor incididunt dolor elit labore amet tempor tempor dolor labore dolore ut incididunt sed dolor labore labore amet sed lorem dolore tempor ut elit tempor labore eiusmod labore dolore lorem amet elit sit elit sed tempor elit consectetur labore tempor tempor ipsum consectetur incididunt magna tempor tempor magna ut do dolore sed labore dolor ipsum sit magna sed do adipiscing ut magna adipiscing elit sed dolor ut incididunt dolore do do et dolor ut elit adipiscing et sit dolore tempor magna dolor ipsum dolor eiusmod elit labore dolor adipiscing ipsum labore amet dolor et dolor ipsum amet et amet et ut et do lorem sed magna sit dolor magna magna tempor ipsum elit magna magna amet lorem incididunt ipsum amet eiusmod incididunt magna dolor labore do eiusmod adipiscing ut et dolor incididunt sed lorem sit do elit incididunt lorem incididunt dolor dolor elit dolor ut dolore labore magna do dolore ipsum amet et et tempor amet magna sit adipiscing sed magna lorem et labore amet amet eiusmod incididunt labore sed labore tempor incididunt dolor lorem adipiscing magna elit elit tempor ut magna sit dolor lorem elit eiusmod elit adipiscing dolore consectetur magna dolor dolore consectetur ipsum do sed eiusmod tempor ut tempor adipiscing lorem et amet consectetur incididunt lorem consectetur lorem sed tempor adipiscing ut adipiscing elit ut adipiscing ut incididunt ipsum consectetur lorem adipiscing sit ipsum do amet magna incididunt tempor consectetur consectetur consectetur dolore amet et dolor labore consectetur do sed tempor labore elit ipsum elit adipiscing sed elit ut sed elit tempor et ut adipiscing labore dolore dolore eiusmod ipsum elit elit magna amet ut sit magna dolor ut ut dolore incididunt labore sit adipiscing ipsum incididunt adipiscing magna adipiscing ut dolore dolore dolore et tempor sit incididunt do dolore tempor eiusmod consectetur ut magna elit adipiscing dolor elit dolor sed ut elit et sed ipsum incididunt tempor incididunt labore ut adipiscing ipsum do elit amet amet eiusmod magna do labore ut do tempor eiusmod sed elit labore incididunt eiusmod ipsum adipiscing sit amet elit tempor amet dolore labore incididunt sed lorem ut eiusmod amet eiusmod dolor adipiscing incididunt adipiscing incididunt dolore magna dolor labore adipiscing sit amet incididunt sit incididunt dolore elit labore et magna dolor consectetur lorem tempor ipsum labore amet et incididunt adipiscing adipiscing ipsum eiusmod adipiscing et sed consectetur magna et dolore dolore ut ipsum amet adipiscing lorem do ut dolor consectetur lorem incididunt do dolore amet do amet consectetur do dolore elit do labore adipiscing dolore adipiscing ipsum adipiscing incididunt eiusmod tempor incididunt ipsum incididunt dolor consectetur et amet tempor do consectetur ipsum et labore tempor tempor tempor dolore magna adipiscing magna amet elit ut dolor elit sed do magna labore eiusmod labore lorem et et amet lorem ut incididunt consectetur do magna incididunt adipiscing incididunt lorem elit et eiusmod tempor labore do dolor ipsum dolore eiusmod consectetur amet ipsum tempor et elit ipsum ipsum lorem et labore labore ut elit incididunt consectetur sit elit do do elit lorem sit adipiscing ut do et sed do sit labore amet sed eiusmod do magna eiusmod ipsum do adipiscing sed labore dolore dolore elit lorem eiusmod labore sit dolor ut tempor sed ipsum amet dolore do tempor eiusmod incididunt consectetur ipsum labore eiusmod consectetur elit et consectetur ut magna incididunt sit do dolor labore consectetur ut et amet dolor lorem magna magna et magna amet elit ipsum magna do ut labore sed tempor magna eiusmod dolore tempor ipsum consectetur ut elit et tempor sed elit dolor sed do dolore ipsum do et et consectetur dolor dolore magna adipiscing ipsum dolore eiusmod do dolor magna elit consectetur amet ut eiusmod adipiscing labore consectetur lorem adipiscing ipsum lorem dolor incididunt tempor ut adipiscing consectetur lorem ut ipsum elit ipsum incididunt sit tempor tempor magna amet incididunt dolore sit eiusmod et lorem labore elit incididunt magna dolor ut incididunt lorem tempor dolore do elit incididunt et incididunt tempor ipsum ipsum ut ut consectetur amet eiusmod adipiscing amet et ut eiusmod lorem tempor tempor tempor dolor incididunt amet ipsum amet ipsum do sed eiusmod ipsum dolor amet amet elit eiusmod adipiscing consectetur labore adipiscing ut tempor amet do consectetur labore lorem consectetur sit ipsum amet ut do dolore adipiscing ipsum dolor dolore dolore magna ut sed et magna labore labore dolor eiusmod sit do elit ipsum elit sit adipiscing consectetur incididunt incididunt amet elit magna adipiscing et amet sit incididunt labore tempor lorem et et ipsum elit do consectetur dolore et dolor eiusmod dolor sed dolore do amet magna ipsum eiusmod dolor amet dolore amet consectetur lorem magna lorem ipsum dolor magna tempor et ipsum dolore magna dolor labore labore lorem sed tempor labore dolor tempor dolor consectetur consectetur lorem incididunt adipiscing elit amet et sit tempor dolore et sit magna consectetur dolore do sit do dolor sit eiusmod magna labore do labore do dolore labore magna labore amet do sed dolor do ipsum incididunt amet et eiusmod sed sit do ut elit elit incididunt magna dolor incididunt amet sed ipsum sed elit eiusmod dolor sed incididunt sit amet elit et eiusmod labore ut dolor dolore labore ipsum ut adipiscing incididunt sed amet amet incididunt ipsum lorem ipsum labore lorem tempor dolore labore eiusmod consectetur adipiscing labore adipiscing adipiscing incididunt eiusmod labore sed sed consectetur do dolor magna dolor dolore et magna elit sit et sit labore dolore magna consectetur sit eiusmod dolore incididunt elit dolore sit labore elit sit dolor et sed dolore do lorem eiusmod incididunt elit ut amet dolore labore dolor et sed ut magna magna dolor sit incididunt sit sed incididunt ipsum dolor magna tempor dolor incididunt et et consectetur incididunt ipsum sed incididunt incididunt incididunt labore lorem elit magna amet dolor et incididunt adipiscing sit labore adipiscing incididunt incididunt ut lorem sed lorem ipsum et ut amet consectetur adipiscing dolore incididunt do dolor magna ipsum ut amet labore consectetur do ipsum do do ipsum dolore eiusmod do ipsum et dolor consectetur incididunt et sed labore lorem ut ipsum do ipsum tempor adipiscing eiusmod consectetur elit amet dolor ipsum sit amet do eiusmod et et lorem incididunt magna incididunt elit dolore amet magna sit ut magna incididunt elit magna lorem magna eiusmod ut incididunt magna adipiscing amet ut consectetur sit sit do consectetur tempor et et dolore consectetur sed elit sed dolor dolore eiusmod elit magna elit sed consectetur amet ipsum labore tempor amet elit do ipsum ut labore et ipsum consectetur magna incididunt et eiusmod do sed consectetur tempor dolore tempor dolor ut elit tempor sit tempor elit dolor tempor magna do ipsum tempor dolore lorem incididunt sit lorem lorem consectetur labore sit adipiscing tempor dolor dolore amet incididunt consectetur labore sit dolore sed et consectetur tempor amet sit amet dolor incididunt consectetur consectetur elit do sit ipsum adipiscing lorem adipiscing lorem labore adipiscing incididunt do amet eiusmod tempor incididunt sit adipiscing eiusmod lorem ut labore adipiscing consectetur lorem dolore sed labore elit dolor dolor adipiscing amet elit elit magna tempor dolore dolore lorem dolore adipiscing labore ipsum sit consectetur ut amet et dolore elit elit ipsum tempor consectetur magna consectetur ut magna elit sed tempor magna adipiscing do elit sit magna ipsum eiusmod do magna consectetur amet ut eiusmod magna do lorem magna elit lorem eiusmod et ipsum adipiscing incididunt sit labore incididunt et magna magna adipiscing labore sit adipiscing consectetur sed sed lorem dolor incididunt lorem adipiscing dolore eiusmod sed adipiscing et magna amet labore ipsum incididunt dolore incididunt incididunt ipsum ipsum do tempor elit adipiscing labore magna et et ipsum magna adipiscing sit eiusmod do ut elit consectetur sed sed magna tempor sit tempor ut dolore ut labore et elit sit consectetur dolore labore labore ipsum sit dolor incididunt incididunt sed adipiscing et tempor incididunt magna dolore sit lorem elit et incididunt sed sed dolor consectetur incididunt sit ipsum adipiscing tempor consectetur consectetur do sit elit lorem ut do consectetur amet amet eiusmod magna adipiscing sit eiusmod et adipiscing do ipsum adipiscing consectetur eiusmod dolore adipiscing adipiscing ut sit eiusmod sed consectetur lorem dolore do incididunt magna incididunt incididunt dolor eiusmod eiusmod amet sed sed labore consectetur labore adipiscing incididunt et dolor labore sit magna consectetur sed do ut do do adipiscing tempor amet adipiscing magna sit ut adipiscing sed magna magna elit amet dolor eiusmod ipsum labore sed ipsum ipsum elit ipsum tempor dolor consectetur consectetur adipiscing adipiscing ipsum dolore dolore lorem ut et ipsum magna elit dolore elit sit eiusmod ut consectetur ipsum dolore incididunt do tempor lorem labore eiusmod incididunt labore et et tempor consectetur ut ipsum dolore eiusmod sed ipsum lorem incididunt amet et adipiscing incididunt magna labore consectetur elit dolore ipsum consectetur ut labore lorem adipiscing ipsum incididunt dolor dolore incididunt lorem amet elit incididunt eiusmod adipiscing sit incididunt eiusmod ipsum dolor adipiscing tempor tempor ipsum ut do ut do dolore consectetur consectetur dolore amet lorem lorem ut adipiscing elit labore sit lorem ut consectetur magna do dolore labore consectetur incididunt adipiscing do eiusmod sit sit magna sit et ipsum magna incididunt tempor magna ipsum sit labore sed ipsum eiusmod labore amet do labore ipsum amet do magna sit sit dolor ut ut dolore dolor ut tempor magna consectetur et et dolore ut do labore labore elit adipiscing magna adipiscing amet ipsum sed lorem labore labore consectetur adipiscing do magna do amet dolor consectetur dolore dolor consectetur consectetur ut consectetur adipiscing ut sed eiusmod amet magna elit ut eiusmod lorem elit consectetur do ipsum sit elit lorem ut lorem dolore ipsum consectetur lorem magna labore ut amet tempor incididunt elit et magna sed sit dolore do lorem do dolore ipsum do ipsum dolor tempor tempor sit dolore labore sed ipsum ipsum magna sed eiusmod tempor elit sed et sed dolor elit ut incididunt et consectetur ut labore ut elit labore dolore dolor dolore incididunt incididunt adipiscing adipiscing amet ipsum amet labore tempor tempor sit sed elit eiusmod do lorem sed ut labore sed eiusmod dolor lorem sed ut amet elit tempor incididunt lorem elit magna consectetur elit sit adipiscing sit adipiscing elit ipsum ut eiusmod amet lorem labore adipiscing ipsum ipsum sit ipsum adipiscing tempor dolor incididunt adipiscing eiusmod incididunt do consectetur do do sed tempor elit elit sit incididunt amet tempor do tempor dolore tempor ipsum dolore ipsum dolor ut eiusmod dolore tempor magna elit do dolore do incididunt dolore magna eiusmod consectetur amet magna sit amet elit do ut ut sed dolore tempor do labore labore ipsum labore magna ut lorem sit do dolore consectetur dolor sit elit tempor labore tempor do amet sed sit amet et eiusmod magna elit tempor ipsum ipsum sit consectetur et labore dolore dolor incididunt ipsum amet dolor do lorem labore elit elit adipiscing amet magna incididunt amet amet consectetur tempor ipsum tempor lorem et labore dolor ipsum labore amet incididunt sed ipsum ipsum dolore ipsum consectetur elit do dolore sit do consectetur ipsum sit et amet sed sit magna dolor eiusmod amet elit elit eiusmod labore tempor lorem dolor sed magna consectetur lorem magna tempor do magna consectetur lorem et magna consectetur tempor dolore tempor magna ut et et tempor ut sit dolor sit elit eiusmod adipiscing ipsum ipsum tempor elit ut incididunt ipsum ipsum ut sed tempor dolor sit eiusmod consectetur tempor incididunt labore consectetur consectetur ut et adipiscing lorem dolore consectetur adipiscing consectetur dolor lorem dolor adipiscing labore sed ipsum incididunt sed ut elit labore sit dolor adipiscing adipiscing adipiscing lorem consectetur ut et labore eiusmod consectetur adipiscing ut dolor incididunt tempor tempor dolor dolore labore dolore magna lorem adipiscing sed tempor tempor lorem adipiscing elit ut eiusmod tempor labore labore sit magna ipsum adipiscing eiusmod dolor ut sed elit ipsum do sit do amet ut tempor dolore dolor do dolor labore et consectetur elit lorem magna lorem adipiscing adipiscing eiusmod dolor incididunt ut dolore incididunt eiusmod incididunt labore do dolore tempor adipiscing amet magna do magna sit et tempor incididunt consectetur ipsum dolor et ipsum incididunt amet amet consectetur dolore dolor incididunt magna adipiscing dolore incididunt tempor ut consectetur ut sed elit tempor dolore lorem do do dolore sed amet adipiscing labore ipsum sed sit dolor ipsum dolor et amet et labore incididunt elit ipsum ut consectetur ipsum do labore ipsum amet dolore adipiscing adipiscing consectetur adipiscing amet lorem incididunt lorem incididunt tempor tempor eiusmod amet eiusmod eiusmod magna eiusmod ipsum eiusmod labore consectetur dolor lorem lorem et sit adipiscing incididunt dolore ut sit sed dolore ut ipsum labore adipiscing ipsum magna lorem do ut dolor eiusmod tempor incididunt ipsum sed amet adipiscing lorem consectetur labore ipsum ut sit magna ipsum dolore consectetur labore dolor lorem magna lorem amet elit lorem magna ipsum dolore elit ut sit et tempor eiusmod consectetur adipiscing elit dolor adipiscing consectetur do dolor dolor adipiscing amet dolor labore sed tempor elit amet consectetur elit ut dolor tempor dolore magna consectetur ipsum magna ipsum sed lorem magna et ut adipiscing sit tempor eiusmod dolore dolore sed consectetur do do et dolore eiusmod dolore ut sit ipsum dolor dolor do ut incididunt tempor eiusmod dolor amet magna labore dolore ut elit magna sed ut eiusmod do consectetur ipsum sit sit labore sit et tempor sed sit eiusmod incididunt sit consectetur dolore magna labore dolore dolor et eiusmod sit eiusmod sit sit magna elit et elit et eiusmod ipsum consectetur ipsum lorem ut eiusmod do magna elit tempor amet dolor adipiscing ipsum consectetur tempor ut sed adipiscing consectetur adipiscing incididunt dolore consectetur consectetur elit labore amet do eiusmod consectetur eiusmod magna do elit adipiscing consectetur do ipsum amet amet lorem ut ipsum tempor labore amet ut adipiscing consectetur magna tempor consectetur dolor sed tempor eiusmod ipsum dolore do amet et ipsum labore ut dolor lorem lorem ipsum lorem lorem adipiscing tempor adipiscing labore ipsum ipsum elit do adipiscing magna consectetur adipiscing adipiscing tempor magna adipiscing amet sit eiusmod consectetur magna labore et lorem magna et dolor lorem labore eiusmod tempor tempor ut amet sit sit labore eiusmod consectetur dolor incididunt lorem dolore labore lorem amet do sed do ipsum et et ipsum amet lorem et ut ipsum elit sit incididunt magna dolore lorem do lorem lorem elit dolore incididunt sed tempor lorem adipiscing ut adipiscing labore dolore ipsum amet magna sed ipsum sit incididunt sit ipsum et et amet sit et sed consectetur ipsum sit magna labore dolor consectetur eiusmod labore magna consectetur et tempor sed do et consectetur do dolore lorem consectetur elit sit incididunt dolor sit do sit sed elit consectetur tempor sit labore sit eiusmod dolore magna dolor labore ut dolor dolore sit labore et sed incididunt labore et incididunt incididunt sit incididunt eiusmod lorem dolor consectetur et ut lorem dolor incididunt dolore eiusmod ipsum lorem dolore lorem labore dolore consectetur labore do ut dolor dolore adipiscing ut eiusmod et ut incididunt lorem dolore magna sit consectetur incididunt magna amet sed adipiscing et amet eiusmod magna sit consectetur et lorem do ipsum magna sit sed et consectetur ut sed tempor do eiusmod sed consectetur dolore adipiscing magna sit consectetur sed sit adipiscing ipsum amet dolore labore magna sit magna magna elit consectetur ut ut consectetur ipsum eiusmod lorem eiusmod consectetur amet tempor elit ipsum ut consectetur eiusmod adipiscing elit et do sed dolore dolore dolore et elit consectetur dolor elit labore amet incididunt consectetur lorem sit ipsum ipsum magna ipsum et labore lorem et sed do sed incididunt et sit dolor do incididunt ut adipiscing elit dolore amet dolor do adipiscing sit tempor sed amet elit tempor tempor consectetur sit sit adipiscing do elit dolor ipsum eiusmod incididunt adipiscing tempor dolor labore labore eiusmod do eiusmod dolore lorem dolore dolor amet consectetur lorem lorem elit adipiscing ut do consectetur sit incididunt dolore dolore lorem adipiscing dolor adipiscing consectetur eiusmod adipiscing eiusmod elit sed tempor incididunt eiusmod ipsum labore adipiscing sit labore amet incididunt incididunt adipiscing eiusmod eiusmod elit et labore incididunt lorem labore tempor et eiusmod ipsum labore incididunt et et do ipsum dolore labore elit dolor do consectetur labore adipiscing sed consectetur magna do incididunt lorem adipiscing et lorem do sed magna sed dolor incididunt ipsum magna ipsum adipiscing et adipiscing incididunt eiusmod do et dolore sed sed consectetur incididunt incididunt et magna elit elit ut labore dolor incididunt ipsum lorem adipiscing tempor adipiscing sit ipsum et adipiscing ipsum consectetur eiusmod ipsum dolore sed adipiscing adipiscing ut dolore dolor consectetur sed ipsum amet lorem ipsum magna adipiscing consectetur eiusmod amet consectetur incididunt lorem lorem eiusmod dolore lorem adipiscing dolor et labore dolor consectetur sed dolore lorem ut do magna adipiscing consectetur eiusmod sit do do adipiscing dolore sit lorem sed adipiscing adipiscing elit magna adipiscing et eiusmod incididunt et sit tempor dolor labore sed et ipsum lorem eiusmod et lorem elit magna amet ipsum incididunt sit elit ipsum do tempor sit sit sed sed amet elit magna amet et ipsum ut labore lorem lorem eiusmod ut do et ipsum labore eiusmod ut lorem consectetur sed tempor elit dolor amet eiusmod incididunt magna magna do dolore eiusmod tempor et amet adipiscing ut adipiscing sit dolor consectetur lorem lorem dolor labore dolore dolor consectetur tempor dolore magna consectetur consectetur ut dolor labore sit lorem et tempor incididunt amet tempor amet elit incididunt eiusmod adipiscing dolore magna sit do labore adipiscing labore amet sed ipsum consectetur adipiscing et magna adipiscing ipsum incididunt tempor et dolore ut elit adipiscing dolor lorem amet et adipiscing et sed incididunt sit dolore dolor sit tempor consectetur adipiscing ut do ut sed dolore elit incididunt ut ipsum ipsum adipiscing tempor incididunt eiusmod ut sed labore incididunt consectetur amet ut tempor tempor tempor dolore ut ipsum tempor sit eiusmod ipsum adipiscing tempor consectetur magna do et magna consectetur ut tempor tempor lorem consectetur consectetur tempor et labore magna adipiscing sit eiusmod lorem ut elit labore do ut sed dolor magna adipiscing labore sit ipsum lorem magna elit elit ipsum et eiusmod adipiscing adipiscing sit lorem do sed sit amet tempor labore ut eiusmod ipsum dolore do dolor labore elit consectetur adipiscing dolore do et do sed sit magna sed consectetur consectetur dolore consectetur ut do incididunt incididunt elit consectetur ut consectetur magna magna eiusmod elit adipiscing amet do dolor dolor elit sit do lorem amet incididunt incididunt ipsum sit ipsum amet labore incididunt do elit consectetur et elit elit sed dolore ut labore do amet dolore amet ipsum amet magna amet lorem dolore sed magna magna consectetur ipsum tempor et magna consectetur elit eiusmod consectetur adipiscing consectetur et elit amet adipiscing ipsum dolore eiusmod elit incididunt labore sed consectetur sit ipsum dolor ipsum do et ipsum sit do tempor tempor elit adipiscing sed adipiscing consectetur eiusmod sed eiusmod tempor ipsum dolor adipiscing consectetur incididunt labore adipiscing lorem labore incididunt incididunt tempor tempor et lorem sed dolor ipsum incididunt ut tempor incididunt consectetur do adipiscing dolor do lorem ut ipsum labore lorem incididunt adipiscing magna et sed lorem adipiscing elit amet eiusmod elit sed eiusmod ipsum incididunt do lorem labore lorem ut lorem sit consectetur elit lorem incididunt labore amet dolor adipiscing adipiscing ut tempor tempor dolor elit adipiscing sit amet sed tempor eiusmod ut sed ut tempor elit dolor labore elit sed lorem dolore eiusmod adipiscing tempor do tempor adipiscing tempor eiusmod magna magna sed ut dolor incididunt adipiscing labore adipiscing adipiscing eiusmod et labore lorem ipsum do amet dolore incididunt dolor do magna amet eiusmod eiusmod incididunt et amet sed elit elit incididunt tempor magna ipsum do labore dolore tempor lorem amet et et lorem ipsum incididunt ut sit adipiscing ut tempor ut labore incididunt ut dolor et consectetur dolore adipiscing consectetur sit ut tempor elit consectetur incididunt tempor tempor amet magna consectetur elit sed magna elit dolor dolor adipiscing ut et ipsum eiusmod elit dolor ut dolor eiusmod consectetur et elit tempor adipiscing do consectetur labore sit lorem sit sed magna do sit do adipiscing labore elit lorem ut dolor eiusmod labore labore labore magna elit ut sed incididunt dolor incididunt incididunt sit sit tempor tempor tempor incididunt et et adipiscing adipiscing amet adipiscing sit lorem consectetur et eiusmod sed do dolor adipiscing magna amet sed elit sit do dolore ut eiusmod magna tempor amet dolore labore dolore elit eiusmod adipiscing magna eiusmod amet magna amet incididunt do ipsum lorem ipsum ut tempor ipsum dolore labore labore dolore do magna incididunt incididunt eiusmod sed et dolor lorem amet eiusmod dolor tempor ipsum ipsum incididunt eiusmod adipiscing amet ut elit do eiusmod lorem elit adipiscing sed incididunt do et incididunt et labore sed lorem labore labore labore dolore ut adipiscing tempor ut consectetur consectetur labore magna tempor dolore lorem adipiscing sit tempor ipsum labore dolor eiusmod sit dolor dolore tempor elit dolor eiusmod amet elit tempor elit sed elit dolore incididunt ipsum lorem sed sed et sit et ut sed sit ipsum tempor et tempor magna sed adipiscing ut incididunt consectetur tempor dolor magna labore do et tempor adipiscing eiusmod incididunt tempor consectetur labore tempor sed magna amet elit consectetur eiusmod lorem sit amet adipiscing amet magna dolore ipsum incididunt amet amet eiusmod et labore sit lorem incididunt amet incididunt dolor amet dolor dolore sed do amet eiusmod sit labore dolor dolore lorem adipiscing dolore tempor labore sit adipiscing lorem lorem magna sit do lorem ut magna consectetur magna sed consectetur ipsum lorem adipiscing adipiscing consectetur sit tempor ipsum do sit consectetur adipiscing lorem ipsum elit consectetur ut elit amet lorem dolore labore et ipsum amet et sed labore magna tempor eiusmod labore ut elit tempor labore et et lorem adipiscing eiusmod adipiscing sit eiusmod labore ipsum consectetur sed magna elit labore ut amet consectetur ipsum dolore eiusmod et do magna sed sed dolore dolore incididunt elit dolore elit eiusmod do sed eiusmod incididunt sed elit dolor et consectetur tempor magna consectetur magna dolor labore et tempor eiusmod dolor magna dolore adipiscing elit amet tempor eiusmod magna dolore dolore magna consectetur consectetur adipiscing adipiscing adipiscing magna ipsum ut adipiscing labore ut eiusmod lorem eiusmod consectetur amet elit ut magna et consectetur dolore sit sit sit lorem labore consectetur dolor magna dolor amet lorem ut sed adipiscing tempor labore et labore et labore sed sit eiusmod eiusmod dolor labore incididunt sed incididunt eiusmod dolor do amet sed lorem amet consectetur consectetur incididunt sit elit incididunt consectetur do tempor dolore amet incididunt sit incididunt tempor elit lorem sit adipiscing incididunt dolor lorem tempor tempor incididunt sit amet consectetur adipiscing adipiscing labore elit dolore consectetur labore sed et elit incididunt lorem lorem sit consectetur ipsum elit eiusmod eiusmod sit ut et amet ipsum sed tempor adipiscing magna eiusmod et amet do do lorem tempor et lorem elit elit ipsum magna magna do consectetur et do magna dolor adipiscing sit consectetur lorem eiusmod amet dolore amet ipsum adipiscing consectetur elit amet elit magna elit elit do dolore ut elit amet incididunt amet consectetur incididunt incididunt dolore amet sit dolore amet magna adipiscing consectetur et elit et dolor eiusmod ipsum et amet labore amet dolore magna incididunt incididunt sit lorem consectetur adipiscing elit sed incididunt sed ipsum ipsum et labore dolor ut amet ut adipiscing lorem lorem sed lorem elit amet adipiscing et sed consectetur dolore adipiscing sed tempor incididunt lorem incididunt dolor magna ut do lorem do sit adipiscing et sit tempor elit consectetur lorem incididunt incididunt ipsum consectetur magna amet sed et incididunt amet sit tempor labore elit sed consectetur consectetur labore labore adipiscing incididunt dolor adipiscing ut consectetur labore ipsum dolor ipsum amet amet ut adipiscing incididunt dolore sit eiusmod ut incididunt do adipiscing ut ipsum ipsum eiusmod sed incididunt et do dolor eiusmod amet ipsum dolor ipsum consectetur et eiusmod ut ipsum labore sit et sed adipiscing incididunt consectetur ut elit tempor incididunt sit et incididunt eiusmod elit ipsum dolor sed ipsum adipiscing ut adipiscing magna adipiscing ipsum dolor do dolor lorem dolore lorem amet labore consectetur ipsum ipsum elit elit adipiscing magna lorem labore elit ut elit amet tempor ipsum dolore magna tempor ut ipsum tempor lorem elit et incididunt adipiscing et sed consectetur tempor do sit lorem dolore magna incididunt magna incididunt sit amet consectetur sed tempor ipsum do adipiscing incididunt dolor labore ipsum amet ipsum amet amet labore consectetur amet consectetur amet eiusmod elit sit tempor sed sit ipsum consectetur sit ipsum elit ipsum do amet magna lorem consectetur labore ipsum ut lorem sit consectetur dolor ipsum ipsum eiusmod et labore sed et magna sit dolore magna elit elit do magna ut et lorem do et labore adipiscing incididunt amet consectetur consectetur dolor eiusmod adipiscing et lorem sit eiusmod lorem dolore eiusmod elit amet adipiscing dolore tempor adipiscing eiusmod consectetur ut labore consectetur incididunt tempor tempor do sed adipiscing tempor sit tempor consectetur amet sed consectetur dolore dolore sed elit elit labore adipiscing sed sed et eiusmod consectetur tempor do lorem lorem dolore incididunt ut do tempor amet amet sit dolor dolor sit consectetur elit tempor ipsum elit tempor ut ut magna sed labore incididunt lorem magna tempor labore incididunt dolor tempor sit tempor labore do sit ut do et ut labore amet adipiscing tempor adipiscing incididunt amet dolore eiusmod ipsum consectetur dolore sit dolor adipiscing lorem et eiusmod et ut consectetur eiusmod elit ipsum do dolor lorem ipsum lorem labore elit eiusmod eiusmod eiusmod sit amet do sed magna dolor incididunt eiusmod et eiusmod consectetur do ipsum do ut lorem magna magna adipiscing amet sit magna labore sed eiusmod sit sit tempor ut labore ut incididunt ipsum magna eiusmod lorem sit adipiscing incididunt elit do et ut tempor incididunt labore labore ut dolor do et do et dolor amet dolor sit dolore incididunt consectetur tempor consectetur dolor ut dolore lorem elit ut consectetur dolore dolor labore ipsum adipiscing tempor eiusmod sit amet elit lorem et consectetur et do adipiscing sed sed labore ipsum do eiusmod dolor et labore et ipsum tempor labore et tempor lorem tempor adipiscing amet dolor dolor do ut ut eiusmod sed dolore sed sit incididunt elit dolor et amet consectetur magna dolore magna tempor elit do dolor consectetur labore tempor labore adipiscing incididunt ipsum do sed lorem eiusmod consectetur do incididunt do adipiscing do tempor et do sed labore adipiscing adipiscing adipiscing do incididunt ipsum lorem magna sit tempor incididunt tempor elit dolor ipsum elit dolore magna tempor ut eiusmod eiusmod magna do ipsum sed sed labore incididunt incididunt ipsum magna sed ipsum dolore elit sed ipsum incididunt adipiscing eiusmod labore sed incididunt do sit magna dolore adipiscing consectetur dolore labore sed lorem ut magna ipsum eiusmod consectetur eiusmod ipsum adipiscing ut et elit et magna dolor tempor ut consectetur magna tempor sit eiusmod dolor adipiscing ipsum sit amet sit adipiscing magna eiusmod amet labore incididunt do lorem labore amet tempor et magna labore et labore incididunt dolor ipsum magna magna tempor amet eiusmod et ut lorem tempor sit ipsum tempor elit adipiscing ipsum incididunt do dolore incididunt ipsum amet ipsum adipiscing elit incididunt eiusmod tempor labore tempor ipsum ipsum sit dolor dolor amet ipsum lorem sit lorem ut eiusmod elit magna dolore elit adipiscing labore labore labore incididunt lorem ipsum do adipiscing do elit sed eiusmod sed tempor dolor sed magna adipiscing ut consectetur ut amet elit incididunt lorem eiusmod incididunt dolor ut sit dolore et elit et dolor consectetur do sed eiusmod tempor elit magna incididunt sit do elit sed incididunt magna labore sed magna adipiscing adipiscing consectetur magna adipiscing elit do dolor dolor sed elit adipiscing tempor adipiscing amet amet incididunt labore dolor dolore amet adipiscing adipiscing et labore amet eiusmod ipsum ipsum adipiscing dolore amet adipiscing amet labore ipsum eiusmod labore elit adipiscing dolor incididunt sed dolor lorem tempor dolore consectetur eiusmod do et et sed sit labore do tempor dolore adipiscing ut dolor do elit ut sed sit adipiscing adipiscing sed ut tempor lorem lorem et eiusmod sit tempor elit magna dolor sit sit ipsum ut do ipsum ipsum tempor consectetur eiusmod labore eiusmod lorem labore ipsum dolore dolore ipsum incididunt sit ut dolore sit tempor incididunt ut sit amet do incididunt elit incididunt labore incididunt magna et ut consectetur sit ipsum elit sit ipsum et tempor consectetur elit et ipsum ut labore tempor sed consectetur adipiscing et elit labore eiusmod adipiscing sit consectetur consectetur ipsum sit ut dolore sit adipiscing incididunt elit incididunt dolor ut tempor labore labore magna ut sit labore lorem magna ut ipsum ipsum incididunt ipsum labore ut eiusmod ipsum labore et sed ut et consectetur dolor amet eiusmod dolor incididunt elit ipsum elit eiusmod do adipiscing tempor labore adipiscing dolore adipiscing sit sit sed ut ut do elit amet sed sed elit lorem sit ipsum eiusmod dolore amet sit labore eiusmod sed dolor elit do ut do consectetur consectetur do elit consectetur amet elit labore et tempor lorem eiusmod incididunt ipsum tempor elit eiusmod et lorem et tempor dolore dolor dolor amet ipsum labore dolor tempor sed elit amet sed dolore tempor incididunt et consectetur et eiusmod magna ut adipiscing sed incididunt labore magna dolor lorem adipiscing tempor dolor ipsum consectetur et et incididunt elit ut et tempor labore elit lorem dolore consectetur consectetur labore consectetur elit sit ipsum incididunt dolore ipsum adipiscing eiusmod amet adipiscing amet incididunt dolor amet labore sed et tempor magna magna elit ipsum consectetur tempor sed magna amet dolore eiusmod dolor dolore et amet elit elit adipiscing tempor elit eiusmod sit eiusmod amet dolore sed amet labore tempor sed incididunt ipsum labore dolore sed ut eiusmod ipsum tempor do elit ipsum ipsum dolore adipiscing sed adipiscing sit tempor et sit magna lorem adipiscing adipiscing tempor incididunt sit adipiscing elit consectetur sit labore elit elit lorem tempor lorem tempor ipsum consectetur dolor et adipiscing magna do ipsum sed labore lorem elit consectetur dolore amet ut tempor consectetur et labore eiusmod dolor sit incididunt ipsum tempor sit adipiscing labore ut ut adipiscing ipsum incididunt amet elit tempor ipsum tempor eiusmod do adipiscing ipsum eiusmod labore adipiscing do lorem et ipsum ut ipsum tempor magna do ut do ut dolore sed ipsum incididunt adipiscing ipsum adipiscing sit sit ipsum eiusmod adipiscing do ut et \ No newline at end of file diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r00 new file mode 100644 index 00000000..f0046fe4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r01 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r01 new file mode 100644 index 00000000..8a2851cc Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r02 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r02 new file mode 100644 index 00000000..91f41494 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r02 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r03 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r03 new file mode 100644 index 00000000..7faa417d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.r03 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.rar new file mode 100644 index 00000000..1d51d4f7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_multivol_prng_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_text_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_text_rar300.rar new file mode 100644 index 00000000..e5c5f1cf Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/compressed_text_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_multivol_rar300.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_multivol_rar300.r00 new file mode 100644 index 00000000..481fc9b9 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_multivol_rar300.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_multivol_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_multivol_rar300.rar new file mode 100644 index 00000000..bee7978c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_multivol_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_newnaming_rar300.part01.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_newnaming_rar300.part01.rar new file mode 100644 index 00000000..3a4cedc8 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_newnaming_rar300.part01.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_newnaming_rar300.part02.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_newnaming_rar300.part02.rar new file mode 100644 index 00000000..8efddc7c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/encrypted_newnaming_rar300.part02.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_multivol_rar300.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_multivol_rar300.r00 new file mode 100644 index 00000000..24470ec7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_multivol_rar300.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_multivol_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_multivol_rar300.rar new file mode 100644 index 00000000..31982fb4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_multivol_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_newnaming_rar300.part01.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_newnaming_rar300.part01.rar new file mode 100644 index 00000000..e3ff386d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_newnaming_rar300.part01.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_newnaming_rar300.part02.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_newnaming_rar300.part02.rar new file mode 100644 index 00000000..4b5d0a07 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/header_encrypted_newnaming_rar300.part02.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_newnaming_rar300.part01.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_newnaming_rar300.part01.rar new file mode 100644 index 00000000..df890a15 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_newnaming_rar300.part01.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_newnaming_rar300.part02.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_newnaming_rar300.part02.rar new file mode 100644 index 00000000..5b128baf Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_newnaming_rar300.part02.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_oldnaming_rar300.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_oldnaming_rar300.r00 new file mode 100644 index 00000000..65d92431 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_oldnaming_rar300.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_oldnaming_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_oldnaming_rar300.rar new file mode 100644 index 00000000..6c1ddbc1 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/multivol_oldnaming_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_audio_stereo_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_audio_stereo_rar300.rar new file mode 100644 index 00000000..099a4713 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_audio_stereo_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_delta_4ch_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_delta_4ch_rar300.rar new file mode 100644 index 00000000..7556f28d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_delta_4ch_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_itanium_synthetic_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_itanium_synthetic_rar300.rar new file mode 100644 index 00000000..75ec9911 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_itanium_synthetic_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_rgb_gradient_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_rgb_gradient_rar300.rar new file mode 100644 index 00000000..1c36a39c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_rgb_gradient_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_x86_e8_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_x86_e8_rar300.rar new file mode 100644 index 00000000..2de3162d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_x86_e8_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_x86_e8e9_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_x86_e8e9_rar300.rar new file mode 100644 index 00000000..9dfe315a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rarvm_x86_e8e9_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part1.rar new file mode 100644 index 00000000..b897e5f2 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part1.rev b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part1.rev new file mode 100644 index 00000000..4bb09923 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part1.rev differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part2.rar new file mode 100644 index 00000000..2f08ebaf Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part2.rev b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part2.rev new file mode 100644 index 00000000..ccf34ae5 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part2.rev differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part3.rar new file mode 100644 index 00000000..e9bc79e3 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part4.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part4.rar new file mode 100644 index 00000000..d367c891 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_newstyle.part4.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part1.rar new file mode 100644 index 00000000..d6ec15b0 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part2.rar new file mode 100644 index 00000000..f9631bf8 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part3.rar new file mode 100644 index 00000000..6e9090da Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4.rar new file mode 100644 index 00000000..6bee4313 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4_2_1.rev b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4_2_1.rev new file mode 100644 index 00000000..8ed8be73 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4_2_1.rev differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4_2_2.rev b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4_2_2.rev new file mode 100644 index 00000000..ab333594 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/rev_oldstyle.part4_2_2.rev differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/solid_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/solid_rar300.rar new file mode 100644 index 00000000..4c0b7a40 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/solid_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/solid_simple_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/solid_simple_rar300.rar new file mode 100644 index 00000000..a033b7f9 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/solid_simple_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r00 new file mode 100644 index 00000000..0ac10b77 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r01 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r01 new file mode 100644 index 00000000..0ac10b77 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r02 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r02 new file mode 100644 index 00000000..fbb49c47 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.r02 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.rar new file mode 100644 index 00000000..bfc1307e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/stored_multivol_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_comment_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_comment_rar300.rar new file mode 100644 index 00000000..2daeda39 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_comment_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_compressed_recovery_header_synthetic.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_compressed_recovery_header_synthetic.rar new file mode 100644 index 00000000..5de43541 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_compressed_recovery_header_synthetic.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_compressed_recovery_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_compressed_recovery_rar300.rar new file mode 100644 index 00000000..f75c35ef Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_compressed_recovery_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_recovery_rar300.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_recovery_rar300.rar new file mode 100644 index 00000000..62794f77 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar300/with_recovery_rar300.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar420/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar420/README.md new file mode 100644 index 00000000..62d812d2 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar420/README.md @@ -0,0 +1,12 @@ +# RAR 4.20 Container Fixtures + +RAR 4.20 cross-version fixtures copied from the spec repository's +`fixtures/1.5-4.x/rar420/` set. + +| Fixture | Purpose | +|---|---| +| `ext_time_rar420.rar` | Extended-time (`LHD_EXTTIME`) nibble groups for mtime/ctime/atime. | + +The RAR 4.20 header-encrypted fixture is stored in the sibling `encrypted/` +directory as `header_rar420_password.rar`, because the tests group all +password-protected fixtures together. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar420/ext_time_rar420.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar420/ext_time_rar420.rar new file mode 100644 index 00000000..66f08710 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rar420/ext_time_rar420.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/README.md new file mode 100644 index 00000000..e3477ede --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/README.md @@ -0,0 +1,15 @@ +# rars-generated RAR 1.5 Writer Oracles + +Static copies of `rars`-generated RAR 1.5 writer fixtures from the spec repo. +They are validated with WinRAR/UnRAR 4.20 and used by +`rar15_40_fixtures.rs` to pin emitted writer bytes and decoded payloads against +public-reader behavior. The spec-repo originals live under +`fixtures/1.5-4.x/rars-generated/`, where `scripts/verify-fixtures.py` checks +their SHA-256 table. + +Password for encrypted fixtures: `pass`. + +`comments.rar` was regenerated for 0.7. Its file comment used to be a bare size +and text, which is the RAR 1.3 layout and wrong from RAR 1.5 on; it is now the +comment block WinRAR writes. The spec-repo copy and its SHA-256 entry need the +same replacement. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/comments.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/comments.rar new file mode 100644 index 00000000..49d900e1 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/comments.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/compressed.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/compressed.rar new file mode 100644 index 00000000..e3707d2b Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/compressed.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/encrypted.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/encrypted.rar new file mode 100644 index 00000000..c28f106a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/encrypted.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/solid.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/solid.rar new file mode 100644 index 00000000..9b30b38c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/solid.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.r00 new file mode 100644 index 00000000..985c0492 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.r01 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.r01 new file mode 100644 index 00000000..70e0c940 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.r01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.rar new file mode 100644 index 00000000..4df7e1de Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-encrypted.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.r00 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.r00 new file mode 100644 index 00000000..2cf6074f Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.r00 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.r01 b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.r01 new file mode 100644 index 00000000..c2e7a783 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.r01 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.rar new file mode 100644 index 00000000..061ce025 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/split-store.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/stored.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/stored.rar new file mode 100644 index 00000000..7ad7ef23 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rars_generated/stored.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/README.md new file mode 100644 index 00000000..2ba14492 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/README.md @@ -0,0 +1,29 @@ +# RARVM Regression Fixtures + +Archive-level fixtures for Unpack29 filters and generic RARVM bytecode. + +| Fixture | Purpose | +|---|---| +| `generic_delta_padding_mutation.rar` | Generic VM fallback path for a non-standard filter program. | +| `vm_encoded_u32_filter.rar` | VM filter control stream with 32-bit encoded integers. | +| `ppmd_embedded_vm_filter.rar` | RARVM filter record embedded in a PPMd stream. | +| `solid_e8_filter_member_offset.rar` | Solid E8 filter offset handling across members. | +| `filter_bsdcat_exe.rar` | Real executable filter archive; focused coverage for x86/E8-style filtered PE data. | + +The `solid_e8_filter_*.txt` / `.exe` files are expected payloads for the solid +filter regression tests. + +## delta_64_channels.rar + +One member, `delta64.bin`, 25,600 bytes: 400 rows of 64 interleaved channels +where byte `[row][channel]` is `(channel * 7 + row * 3) & 0xff`. It carries a +RAR 2.9 VM delta filter with `R[0] = 64`. + +RAR 5 caps delta channels at 32 because its filter record stores `channels - 1` +in five bits. The RAR 2.9 VM has no such field, taking the count from a +register, and the reference decoder accepts up to 1024. UnRAR 7.20 and RAR 7.12 +both extract this archive; rars refused it until the decode bound was raised to +match. + +Written by rars with the writer's own 32-channel limit temporarily lifted. That +limit is a choice about what to emit and stays in place. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/delta_64_channels.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/delta_64_channels.rar new file mode 100644 index 00000000..495de6bb Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/delta_64_channels.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/filter_bsdcat_exe.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/filter_bsdcat_exe.rar new file mode 100644 index 00000000..5318dbb3 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/filter_bsdcat_exe.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/generic_delta_padding_mutation.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/generic_delta_padding_mutation.rar new file mode 100644 index 00000000..6dfcc751 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/generic_delta_padding_mutation.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/ppmd_embedded_vm_filter.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/ppmd_embedded_vm_filter.rar new file mode 100644 index 00000000..185a8e2e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/ppmd_embedded_vm_filter.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_lead.txt b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_lead.txt new file mode 100644 index 00000000..4bd9f1f9 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_lead.txt @@ -0,0 +1,16 @@ +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero +prefix member to make solid stream offset nonzero diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_member_offset.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_member_offset.rar new file mode 100644 index 00000000..5a925b8a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_member_offset.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_payload.exe b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_payload.exe new file mode 100644 index 00000000..7c8e6ef4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/solid_e8_filter_payload.exe differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/vm_encoded_u32_filter.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/vm_encoded_u32_filter.rar new file mode 100644 index 00000000..5318dbb3 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/rarvm/vm_encoded_u32_filter.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/solid_flag_cleared_rar15.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/solid_flag_cleared_rar15.rar new file mode 100644 index 00000000..0f4c8ee7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/solid_flag_cleared_rar15.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/README.md b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/README.md new file mode 100644 index 00000000..43e2a162 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/README.md @@ -0,0 +1,14 @@ +# Out-of-window match fixtures (RAR 2.0 and RAR 2.9) + +Each archive holds one 240-byte member, `zerofill.bin`: eight `0x00` bytes +followed by `b"zero-fill regression payload "` repeated eight times. + +The first eight bytes are not stored as literals. They are a match of length 8 +at distance 4096, emitted at output position 0, where nothing has been written +yet and the distance reaches past the start of the stream. A reader that treats +that as an error cannot extract these files; a reader that fills the region with +zeroes reproduces the member exactly and its CRC32 matches. + +Both archives were produced by `rars` with the encoder temporarily patched to +splice that match over the leading literals, then verified against UnRAR 7.20 +and RAR 7.12, which extract them without complaint. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/rar20.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/rar20.rar new file mode 100644 index 00000000..bb797986 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/rar20.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/rar29.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/rar29.rar new file mode 100644 index 00000000..3ceb800f Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_fill/rar29.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_head_size.rar b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_head_size.rar new file mode 100644 index 00000000..997ae3d9 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar15_40/zero_head_size.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/README-crafted-rejections.md b/arca-rar/vendor/rars/tests/fixtures/rar50/README-crafted-rejections.md new file mode 100644 index 00000000..c0db053e --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar50/README-crafted-rejections.md @@ -0,0 +1,33 @@ +# Crafted RAR 5 headers a reader must handle + +Hand-built, since no encoder emits these. Each was checked against RAR 7.12 and +UnRAR 7.20 so the expected verdict is the reference's, not ours. All three are +patched copies of `-qo-` archives: an archive carrying a quick-open cache holds +a verbatim second copy of the header, and the reference reader silently +extracts from that instead, so an edit to the live header measures nothing. + +| Fixture | Edit | Reference verdict | +|---|---|---| +| `algorithm_version_2.rar` | CompInfo algorithm version 0 → 2 on a compressed member | `Unknown method in mid.txt` | +| `algorithm_version_2_stored.rar` | the same edit on a **stored** member | extracts cleanly | +| `first_block_without_tables.rar` | `table_present` cleared on the first block, block header XOR checksum fixed | member fails | + +The stored one is the interesting pair to the first: with nothing to +decompress, the algorithm version never comes up, so the refusal belongs on the +decompressor rather than at header parse time. + +Version 1 was measured too and is not kept as a fixture: it dispatches to +Unpack70 and then fails a checksum, because the stream is really v0. That the +version picks the decoder is what makes an unassigned value have none to pick. + +## Volume substitution + +`plaintext_stored_multivol.part2.rar` is a plaintext RAR 5 volume with the same +layout as `header_encrypted_stored_multivol.part2.rar`, for splicing into that +set. RAR 7.12 and UnRAR 7.20 abort with `ERROR: Bad archive` rather than +extract; rars refuses with "split entry encryption flag changed". + +Header encryption hides the file names, so an attacker who can replace one +volume of a set the user believes is protected throughout would otherwise +inject entries into the extraction. The set is the unit of trust, not the +volume. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/README-hash-precedence.md b/arca-rar/vendor/rars/tests/fixtures/rar50/README-hash-precedence.md new file mode 100644 index 00000000..6fbe604e --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar50/README-hash-precedence.md @@ -0,0 +1,15 @@ +# crc32_wrong_beside_blake2sp.rar + +`stored_blake2.rar` with a `Data CRC32` field grafted onto the file header: the +`FHFL_CRC32` flag (0x0004) is set, four bytes holding `0x83b27226` are inserted +after `Attributes`, `HeadSize` grows by four, and the header CRC32 is +recomputed. The payload's real CRC32 is `0x83b27227`, so the stored value is +wrong by one bit while the BLAKE2sp record is untouched and correct. + +RAR 7.12 and UnRAR 7.20 both test this archive clean. Corrupting the BLAKE2sp +digest instead, and leaving the CRC32 correct, makes both reject it. So when a +header carries both, BLAKE2sp is the authoritative check and the CRC32 is never +evaluated. + +WinRAR writes one field or the other, never both, so this shape only turns up +in archives from elsewhere. Hand-crafted for that reason. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/README-header-encrypted-stored-multivol.md b/arca-rar/vendor/rars/tests/fixtures/rar50/README-header-encrypted-stored-multivol.md new file mode 100644 index 00000000..a1caafce --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar50/README-header-encrypted-stored-multivol.md @@ -0,0 +1,13 @@ +# header_encrypted_stored_multivol.part{1,2,3}.rar + +WinRAR 7.12, `rar a -ma5 -v2k -m0 -hppassword`. One 4096-byte member, +`stored_4k.bin`, CRC32 `0xa087a9af`, split across three volumes. Password +`password`. + +Stored and header-encrypted and split, which is the combination that reaches +the stored-split verification path. rars used to apply the encrypted-checksum +transform there for any encrypted file rather than only when the crypt record +asks for it, so this archive failed with a checksum mismatch while extracting +the correct bytes. Compressed volume sets took a different path and were +unaffected, which is why the existing `encrypted_multivol` fixtures did not +catch it. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/README-subdata-underflow.md b/arca-rar/vendor/rars/tests/fixtures/rar50/README-subdata-underflow.md new file mode 100644 index 00000000..2f0cbce7 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar50/README-subdata-underflow.md @@ -0,0 +1,17 @@ +# subdata_size_underflow.rar + +`with_recovery.rar` with one byte changed: the `RR` service header's +`FHEXTRA_SUBDATA` record declares size 1 instead of 2, so its type vint fills +the record and the payload byte `0x0a` (10% recovery) dangles past the end. The +header CRC32 was recomputed over `[crc + 4, header_end)`. + +WinRAR 5.21 and earlier wrote this shape for real: they stored the `SUBDATA` +size one less than the payload they emitted. Because `SUBDATA` is the last +record in a service header, the shortfall always shows up as exactly one +leftover byte, which is how a reader recognises it. + +RAR 7.12 and UnRAR 7.20 both extract this archive and report the recovery +record. Neither prints the recovery percent, so what the fixture pins is that +the shape is readable, not that the reference readers keep the byte. + +Hand-crafted rather than produced by a WinRAR 5.21 build. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/README-zero-fill.md b/arca-rar/vendor/rars/tests/fixtures/rar50/README-zero-fill.md new file mode 100644 index 00000000..684ce04a --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar50/README-zero-fill.md @@ -0,0 +1,13 @@ +# zero_fill_out_of_window.rar + +One 240-byte member, `zerofill.bin`: eight `0x00` bytes followed by +`b"zero-fill regression payload "` repeated eight times. + +The leading eight bytes are stored as a match of length 8 at distance 4096, +emitted at output position 0, so the distance reaches past the start of the +stream. WinRAR fills such a region with zeroes rather than failing, which makes +this a valid archive whose BLAKE2sp hash matches only if the reader does the +same. + +Produced by `rars` with the encoder temporarily patched to splice that match +over the leading literals, then verified against UnRAR 7.20 and RAR 7.12. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/README-zeroed-password-check.md b/arca-rar/vendor/rars/tests/fixtures/rar50/README-zeroed-password-check.md new file mode 100644 index 00000000..d9d575e0 --- /dev/null +++ b/arca-rar/vendor/rars/tests/fixtures/rar50/README-zeroed-password-check.md @@ -0,0 +1,24 @@ +# zeroed_password_check.rar + +A WinRAR 7.12 archive (`rar a -ma5 -m0 -psecret`, one member `s.txt`) with the +file header's 8-byte `PswCheck` field overwritten with zeros, its trailing +4-byte checksum recomputed over those zeros, and the header CRC32 recomputed. +Password `secret`. + +WinRAR 5.21 and earlier wrote this shape for real: they set +`FHEXTRA_CRYPT_PSWCHECK` and then left the field zero. A reader that verifies +it rejects the correct password, so those archives cannot be opened at all. + +RAR 7.12 treats an all-zero field as no check. Measured on this archive and its +unmodified original: + +| `PswCheck` | Correct password | Wrong password | +|---|---|---| +| genuine | `All OK` | `Incorrect password for s.txt` | +| all zeros | `All OK` | `Checksum error in the encrypted file s.txt` | + +The changed wrong-password message is the evidence that the explicit check +stopped running and detection fell through to the data checksum. A wrong +password is still refused, just later and by a different mechanism. + +Hand-crafted, since WinRAR 7.12 does not emit this shape. diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/algorithm_version_2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/algorithm_version_2.rar new file mode 100644 index 00000000..f21dd3a5 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/algorithm_version_2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/algorithm_version_2_stored.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/algorithm_version_2_stored.rar new file mode 100644 index 00000000..87aaa2ee Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/algorithm_version_2_stored.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/ams_archive_name_rar721.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/ams_archive_name_rar721.rar new file mode 100644 index 00000000..1be25e12 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/ams_archive_name_rar721.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/crc32_wrong_beside_blake2sp.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/crc32_wrong_beside_blake2sp.rar new file mode 100644 index 00000000..9eb621f8 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/crc32_wrong_beside_blake2sp.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/empty_file.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/empty_file.rar new file mode 100644 index 00000000..ee0b2246 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/empty_file.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part1.rar new file mode 100644 index 00000000..32607a55 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part2.rar new file mode 100644 index 00000000..35a013fb Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part3.rar new file mode 100644 index 00000000..45789e5a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/encrypted_multivol.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/filter_arm.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_arm.rar new file mode 100644 index 00000000..29655c3e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_arm.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/filter_delta.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_delta.rar new file mode 100644 index 00000000..aefa439f Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_delta.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/filter_e8.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_e8.rar new file mode 100644 index 00000000..ebe66f32 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_e8.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/filter_e8e9.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_e8e9.rar new file mode 100644 index 00000000..ef752be8 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/filter_e8e9.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/first_block_without_tables.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/first_block_without_tables.rar new file mode 100644 index 00000000..b9560a27 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/first_block_without_tables.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted.rar new file mode 100644 index 00000000..832c2fd9 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_comment.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_comment.rar new file mode 100644 index 00000000..a67a122f Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_comment.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part1.rar new file mode 100644 index 00000000..10a2cbda Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part2.rar new file mode 100644 index 00000000..aafe0abd Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part3.rar new file mode 100644 index 00000000..c45ceb11 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/header_encrypted_stored_multivol.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/m1_fastest.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/m1_fastest.rar new file mode 100644 index 00000000..a24dae70 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/m1_fastest.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/m3_default.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/m3_default.rar new file mode 100644 index 00000000..cb0b82a1 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/m3_default.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/m5_max.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/m5_max.rar new file mode 100644 index 00000000..de6d291c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/m5_max.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multifile.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multifile.rar new file mode 100644 index 00000000..3910f6e3 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multifile.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part1.rar new file mode 100644 index 00000000..5b09ebce Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part2.rar new file mode 100644 index 00000000..b4626132 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part3.rar new file mode 100644 index 00000000..bdedd6c7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part1.rar new file mode 100644 index 00000000..437d03c7 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part1.rev b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part1.rev new file mode 100644 index 00000000..1cca54aa Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part1.rev differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part2.rar new file mode 100644 index 00000000..09e42597 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part2.rev b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part2.rev new file mode 100644 index 00000000..1eeb00f5 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part2.rev differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part3.rar new file mode 100644 index 00000000..bb0b26e6 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part4.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part4.rar new file mode 100644 index 00000000..7a01244e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part4.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part5.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part5.rar new file mode 100644 index 00000000..8a7c3da0 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/multivol_rev.part5.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/password_aes.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/password_aes.rar new file mode 100644 index 00000000..6cdc729a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/password_aes.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/password_crc32.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/password_crc32.rar new file mode 100644 index 00000000..2f6849ba Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/password_crc32.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/plaintext_stored_multivol.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/plaintext_stored_multivol.part2.rar new file mode 100644 index 00000000..0dc4c32d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/plaintext_stored_multivol.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid.rar new file mode 100644 index 00000000..57252c40 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part01.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part01.rar new file mode 100644 index 00000000..b37de406 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part01.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part02.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part02.rar new file mode 100644 index 00000000..614418b6 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part02.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part03.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part03.rar new file mode 100644 index 00000000..58388548 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part03.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part04.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part04.rar new file mode 100644 index 00000000..4acae7e9 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part04.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part05.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part05.rar new file mode 100644 index 00000000..b19cab9a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part05.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part06.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part06.rar new file mode 100644 index 00000000..74e6fd79 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/solid_multivol.part06.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/stored.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/stored.rar new file mode 100644 index 00000000..7871ab5c Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/stored.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/stored_blake2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_blake2.rar new file mode 100644 index 00000000..b373d923 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_blake2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part1.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part1.rar new file mode 100644 index 00000000..c9cc5cff Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part1.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part2.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part2.rar new file mode 100644 index 00000000..ddf848f4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part2.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part3.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part3.rar new file mode 100644 index 00000000..e26c17e4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/stored_multivol.part3.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/subdata_size_underflow.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/subdata_size_underflow.rar new file mode 100644 index 00000000..b21eb8f0 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/subdata_size_underflow.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/hardlink.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/hardlink.rar new file mode 100644 index 00000000..b8fff669 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/hardlink.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/invalid_hash_valid_htime_exfld.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/invalid_hash_valid_htime_exfld.rar new file mode 100644 index 00000000..27dad55a Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/invalid_hash_valid_htime_exfld.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_loop_bug.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_loop_bug.rar new file mode 100644 index 00000000..c539ffa0 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_loop_bug.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_multiple_files_solid.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_multiple_files_solid.rar new file mode 100644 index 00000000..2e53d7b1 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_multiple_files_solid.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_solid.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_solid.rar new file mode 100644 index 00000000..aaa138e4 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/libarchive_solid.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_hlink.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_hlink.rar new file mode 100644 index 00000000..bc1d2f32 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_hlink.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_solid.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_solid.rar new file mode 100644 index 00000000..277cfe72 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_solid.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_solid_qo.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_solid_qo.rar new file mode 100644 index 00000000..084caaef Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/rarfile_solid_qo.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/wild/symlink.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/symlink.rar new file mode 100644 index 00000000..0aebba5f Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/wild/symlink.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/winrar721_header_encrypted_quickopen.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/winrar721_header_encrypted_quickopen.rar new file mode 100644 index 00000000..aae4f7a2 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/winrar721_header_encrypted_quickopen.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/with_all_services.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/with_all_services.rar new file mode 100644 index 00000000..a0eccb92 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/with_all_services.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/with_comment.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/with_comment.rar new file mode 100644 index 00000000..9c8dc164 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/with_comment.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/with_quickopen.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/with_quickopen.rar new file mode 100644 index 00000000..18915b01 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/with_quickopen.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/with_recovery.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/with_recovery.rar new file mode 100644 index 00000000..721ba9a3 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/with_recovery.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/zero_fill_out_of_window.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/zero_fill_out_of_window.rar new file mode 100644 index 00000000..24ec392e Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/zero_fill_out_of_window.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/rar50/zeroed_password_check.rar b/arca-rar/vendor/rars/tests/fixtures/rar50/zeroed_password_check.rar new file mode 100644 index 00000000..b9e9ca71 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/rar50/zeroed_password_check.rar differ diff --git a/arca-rar/vendor/rars/tests/fixtures/tz/America_New_York b/arca-rar/vendor/rars/tests/fixtures/tz/America_New_York new file mode 100644 index 00000000..a8b9ab19 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/tz/America_New_York differ diff --git a/arca-rar/vendor/rars/tests/fixtures/tz/Asia_Kolkata b/arca-rar/vendor/rars/tests/fixtures/tz/Asia_Kolkata new file mode 100644 index 00000000..0014046d Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/tz/Asia_Kolkata differ diff --git a/arca-rar/vendor/rars/tests/fixtures/tz/Australia_Lord_Howe b/arca-rar/vendor/rars/tests/fixtures/tz/Australia_Lord_Howe new file mode 100644 index 00000000..069a95ad Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/tz/Australia_Lord_Howe differ diff --git a/arca-rar/vendor/rars/tests/fixtures/tz/Etc_GMT_5 b/arca-rar/vendor/rars/tests/fixtures/tz/Etc_GMT_5 new file mode 100644 index 00000000..c0b745f1 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/tz/Etc_GMT_5 differ diff --git a/arca-rar/vendor/rars/tests/fixtures/tz/Europe_London b/arca-rar/vendor/rars/tests/fixtures/tz/Europe_London new file mode 100644 index 00000000..5ad74220 Binary files /dev/null and b/arca-rar/vendor/rars/tests/fixtures/tz/Europe_London differ diff --git a/arca-rar/vendor/rars/tests/legacy_builder_sources.rs b/arca-rar/vendor/rars/tests/legacy_builder_sources.rs new file mode 100644 index 00000000..07d03f2a --- /dev/null +++ b/arca-rar/vendor/rars/tests/legacy_builder_sources.rs @@ -0,0 +1,340 @@ +#![cfg(feature = "write")] + +use rars::{ + ArchiveReader, ArchiveVersion, Builder, EntrySource, ErrorKind, WriteOperation, WriteProgress, + WriteProgressEvent, WriterResources, +}; +use std::io::Cursor; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; + +const FORMATS: [ArchiveVersion; 7] = [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, +]; + +#[test] +fn source_and_inline_members_have_identical_legacy_output() { + for format in FORMATS { + for (store, solid) in [(true, false), (false, false), (false, true)] { + let payload = b"legacy source payload with repeated data\n".repeat(16); + let mut inline = Builder::new(format).store(store).solid(solid); + let mut sources = Builder::new(format).store(store).solid(solid); + for (name, bytes) in [ + (b"first".as_slice(), payload.as_slice()), + (b"second", b"second input"), + (b"empty", b""), + ] { + inline + .add_bytes(name.to_vec(), bytes.to_vec(), None, None) + .unwrap(); + sources + .add_source(name.to_vec(), EntrySource::from_bytes(bytes), None, None) + .unwrap(); + } + let expected = inline.to_bytes().unwrap(); + let actual = sources.to_bytes().unwrap(); + let mut streamed = Vec::new(); + sources + .write_to(&mut streamed, &WriterResources::default(), None) + .unwrap(); + assert_eq!(streamed, expected); + assert_eq!(actual, expected, "{format:?} store={store} solid={solid}"); + let archive = ArchiveReader::read_owned(actual).unwrap(); + assert_eq!( + archive.read_member(b"first", None).unwrap().unwrap(), + payload + ); + assert_eq!( + archive.read_member(b"second", None).unwrap().unwrap(), + b"second input" + ); + assert_eq!(archive.read_member(b"empty", None).unwrap().unwrap(), b""); + } + } +} + +#[test] +fn compression_start_can_cancel_before_any_legacy_source_is_opened() { + struct Stop(AtomicBool); + impl WriteProgress for Stop { + fn report(&self, event: WriteProgressEvent<'_>) { + if matches!( + event, + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Compression, + .. + } + ) { + self.0.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + for format in FORMATS { + let opens = Arc::new(AtomicUsize::new(0)); + let mut builder = Builder::new(format).store(true); + for name in [b"first", b"other"] { + let opens = opens.clone(); + builder + .add_source( + name.to_vec(), + EntrySource::from_opener(1, move || { + opens.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(Cursor::new(vec![42]))) + }), + None, + None, + ) + .unwrap(); + } + let mut output = Vec::new(); + let error = builder + .write_to( + &mut output, + &WriterResources::default(), + Some(&Stop(AtomicBool::new(false))), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Cancelled); + assert_eq!(opens.load(Ordering::Relaxed), 0, "{format:?}"); + // A streaming sink may retain the archive prefix on failure. + } +} + +#[test] +fn stored_source_changes_are_detected_during_verified_emission() { + for format in FORMATS { + let opens = Arc::new(AtomicUsize::new(0)); + let mut builder = Builder::new(format).store(true); + let source_opens = opens.clone(); + builder + .add_source( + b"source".to_vec(), + EntrySource::from_opener(1, move || { + let value = if source_opens.fetch_add(1, Ordering::Relaxed) == 0 { + b'A' + } else { + b'B' + }; + Ok(Box::new(Cursor::new(vec![value]))) + }), + None, + None, + ) + .unwrap(); + let mut output = Vec::new(); + let error = builder + .write_to(&mut output, &WriterResources::default(), None) + .unwrap_err(); + assert_eq!( + error.kind(), + ErrorKind::SourceChanged, + "{format:?}: {error}" + ); + assert_eq!(error.entry_context().unwrap().0, b"source"); + // A streaming sink may retain the archive prefix on failure. + } +} + +#[test] +fn multiple_legacy_volume_inputs_are_refused_without_opening_them() { + for format in FORMATS { + let mut builder = Builder::new(format).store(true).volume_size(Some(64)); + for name in [b"first", b"other"] { + builder + .add_source( + name.to_vec(), + EntrySource::from_opener(1, || panic!("invalid volume plan opened a source")), + None, + None, + ) + .unwrap(); + } + assert_eq!( + builder.build_volumes(None).unwrap_err().kind(), + ErrorKind::InvalidArgument + ); + } +} + +#[test] +fn legacy_builder_emits_before_opening_later_sources() { + use std::io::Write; + use std::sync::Mutex; + struct Sink(Arc>>); + impl Write for Sink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + // Exercise short writes as well as direct emission. + let n = bytes.len().min(17); + self.0.lock().unwrap().extend_from_slice(&bytes[..n]); + Ok(n) + } + fn flush(&mut self) -> std::io::Result<()> { + panic!("builder must not flush caller-owned output") + } + } + for format in FORMATS { + for (store, solid) in [(true, false), (false, false), (false, true)] { + let output = Arc::new(Mutex::new(Vec::new())); + let mut builder = Builder::new(format).store(store).solid(solid); + builder + .add_bytes(b"first".to_vec(), b"first member".repeat(100), None, None) + .unwrap(); + let seen = output.clone(); + builder + .add_source( + b"second".to_vec(), + EntrySource::from_opener(6, move || { + assert!( + !seen.lock().unwrap().is_empty(), + "later source opened before emission" + ); + Ok(Box::new(Cursor::new(b"second"))) + }), + None, + None, + ) + .unwrap(); + let events = Mutex::new(Vec::new()); + let progress = |event: WriteProgressEvent<'_>| match event { + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Emission, + total_bytes, + .. + } => events.lock().unwrap().push((0, total_bytes, false)), + WriteProgressEvent::BytesWritten { completed_bytes } => { + events.lock().unwrap().push((completed_bytes, None, false)) + } + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Emission, + total_bytes, + .. + } => events + .lock() + .unwrap() + .push((total_bytes.unwrap(), total_bytes, true)), + _ => {} + }; + builder + .write_to( + &mut Sink(output.clone()), + &WriterResources::default(), + Some(&progress), + ) + .unwrap(); + let bytes = output.lock().unwrap().clone(); + let events = events.into_inner().unwrap(); + assert_eq!(events.first(), Some(&(0, None, false))); + assert_eq!( + events.last(), + Some(&(bytes.len() as u64, Some(bytes.len() as u64), true)) + ); + assert!(events.windows(2).all(|pair| pair[0].0 <= pair[1].0)); + let archive = ArchiveReader::read_owned(bytes).unwrap(); + assert_eq!( + archive.read_member(b"second", None).unwrap().unwrap(), + b"second" + ); + } + } +} + +#[test] +fn legacy_output_failure_stops_before_later_source_reads() { + use std::io::Write; + struct FailingSink; + impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("injected sink failure")) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + for format in FORMATS { + let mut builder = Builder::new(format).store(true); + builder + .add_source( + b"never-opened".to_vec(), + EntrySource::from_opener(1, || panic!("failed output opened a source")), + None, + None, + ) + .unwrap(); + let finished = AtomicBool::new(false); + let progress = |event: WriteProgressEvent<'_>| { + if matches!( + event, + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Emission, + .. + } + ) { + finished.store(true, Ordering::Relaxed); + } + }; + let error = builder + .write_to( + &mut FailingSink, + &WriterResources::default(), + Some(&progress), + ) + .unwrap_err(); + assert!( + error.to_string().contains("injected sink failure"), + "{format:?}: {error}" + ); + assert!(!finished.load(Ordering::Relaxed)); + } +} + +#[cfg(feature = "parallel")] +#[test] +fn ordinary_legacy_sources_keep_parallel_preparation() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + pool.install(|| { + let barrier = Arc::new(std::sync::Barrier::new(2)); + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + for value in 0..2u8 { + let barrier = barrier.clone(); + let opens = AtomicUsize::new(0); + builder + .add_source( + vec![b'a' + value], + EntrySource::from_opener(1, move || { + if opens.fetch_add(1, Ordering::Relaxed) == 0 { + // Both independent sources must enter preparation together. + barrier.wait(); + } + Ok(Box::new(Cursor::new(vec![value]))) + }), + None, + None, + ) + .unwrap(); + } + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for value in 0..2u8 { + assert_eq!( + archive + .read_member_at(value as usize, None) + .unwrap() + .unwrap(), + [value] + ); + } + }); +} diff --git a/arca-rar/vendor/rars/tests/legacy_codec_adapters.rs b/arca-rar/vendor/rars/tests/legacy_codec_adapters.rs new file mode 100644 index 00000000..d1fd3bb9 --- /dev/null +++ b/arca-rar/vendor/rars/tests/legacy_codec_adapters.rs @@ -0,0 +1,263 @@ +#![cfg(feature = "write")] + +use rars::codec::{ + rar13::{Unpack15, Unpack15Encoder}, + rar20::{Unpack20, Unpack20Encoder}, + rar29::{Unpack29, Unpack29Encoder}, +}; +use std::io::{self, Read, Write}; + +struct TinyReader<'a>(&'a [u8]); + +impl Read for TinyReader<'_> { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let len = out.len().min(self.0.len()).min(3); + out[..len].copy_from_slice(&self.0[..len]); + self.0 = &self.0[len..]; + Ok(len) + } +} + +struct FailedReader; + +impl Read for FailedReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "source refused", + )) + } +} + +struct FailedWriter; + +impl Write for FailedWriter { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::new(io::ErrorKind::BrokenPipe, "sink refused")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn assert_io_error(error: rars::codec::Error, kind: io::ErrorKind, message: &str) { + let rars::codec::Error::Io(source) = error else { + panic!("expected an I/O failure, got {error:?}"); + }; + let rars::Error::Io(detail) = *source else { + panic!("expected the library I/O context, got {source:?}"); + }; + assert_eq!(detail.kind, kind); + assert_eq!(detail.message, message); +} + +fn members() -> (Vec, Vec) { + let phrase = b"legacy solid decoder history alpha beta gamma "; + (phrase.repeat(8), phrase.repeat(4)) +} + +#[test] +fn rar29_tiny_x86_filter_members_preserve_incomplete_operands() { + let bytes = [0xe8, 0xff, 0xe9, 0x01]; + for length in 1..=4 { + for kind in [rars::FilterKind::E8, rars::FilterKind::E8E9] { + let input = &bytes[..length]; + let packed = Unpack29Encoder::new() + .encode_member_with_filter(input, rars::FilterSpec::whole(kind)) + .unwrap(); + let output = Unpack29::default().decode_member(&packed, length).unwrap(); + assert_eq!(output, input); + } + } +} + +#[test] +fn rar29_x86_filters_round_trip_the_negative_address_wrap_boundary() { + // Relative +0x00ffffff crosses the 16MiB boundary when the operand's + // position (one) is added; its coded operand wraps to -1. + for kind in [rars::FilterKind::E8, rars::FilterKind::E8E9] { + let input = [0xe8, 0xff, 0xff, 0xff, 0x00]; + let packed = Unpack29Encoder::new() + .encode_member_with_filter(&input, rars::FilterSpec::whole(kind)) + .unwrap(); + assert_eq!( + Unpack29::default() + .decode_member(&packed, input.len()) + .unwrap(), + input + ); + } + assert_eq!( + Unpack29Encoder::new().encode_member_with_filter( + b"x", + rars::FilterSpec::whole(rars::FilterKind::Delta { channels: 0 }) + ), + Err(rars::codec::Error::InvalidData( + "RAR 2.9 VM filter channel count is invalid" + )) + ); +} + +#[test] +fn rar15_public_adapters_preserve_cloned_solid_state_and_reset_non_solid_state() { + let (first, second) = members(); + let mut encoder = Unpack15Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + let mut decoder = Unpack15::default(); + assert_eq!( + decoder + .decode_member(&first_packed, first.len(), false) + .unwrap(), + first + ); + let mut copied = decoder.clone(); + let mut output = Vec::new(); + decoder + .decode_member_to(&second_packed, second.len(), true, &mut output) + .unwrap(); + assert_eq!(output, second); + output.clear(); + copied + .decode_member_from_reader( + &mut TinyReader(&second_packed), + second.len(), + true, + &mut output, + ) + .unwrap(); + assert_eq!(output, second); + + let independent = Unpack15Encoder::new().encode_member(&first).unwrap(); + output.clear(); + decoder + .decode_member_to(&independent, first.len(), false, &mut output) + .unwrap(); + assert_eq!(output, first); +} + +#[test] +fn rar20_public_adapters_preserve_cloned_solid_state() { + let (first, second) = members(); + let mut encoder = Unpack20Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + let mut decoder = Unpack20::default(); + let mut output = Vec::new(); + decoder + .decode_member_to(&first_packed, first.len(), &mut output) + .unwrap(); + assert_eq!(output, first); + let mut copied = decoder.clone(); + assert_eq!( + copied.decode_member(&second_packed, second.len()).unwrap(), + second + ); + output.clear(); + decoder + .decode_member_from_reader(&mut TinyReader(&second_packed), second.len(), &mut output) + .unwrap(); + assert_eq!(output, second); +} + +#[test] +fn rar29_public_adapters_preserve_cloned_solid_state_and_reset_each_non_solid_route() { + let (first, second) = members(); + let mut encoder = Unpack29Encoder::new(); + let first_packed = encoder.encode_member(&first).unwrap(); + let second_packed = encoder.encode_member(&second).unwrap(); + let mut decoder = Unpack29::default(); + assert_eq!( + decoder.decode_member(&first_packed, first.len()).unwrap(), + first + ); + let mut copied = decoder.clone(); + let mut output = Vec::new(); + decoder + .decode_member_to(&second_packed, second.len(), &mut output) + .unwrap(); + assert_eq!(output, second); + output.clear(); + copied + .decode_member_from_reader(&mut TinyReader(&second_packed), second.len(), &mut output) + .unwrap(); + assert_eq!(output, second); + + let independent = Unpack29Encoder::new().encode_member(&first).unwrap(); + decoder.reset_non_solid(); + assert_eq!( + decoder.decode_member(&independent, first.len()).unwrap(), + first + ); + assert_eq!( + decoder + .decode_non_solid_member(&independent, first.len()) + .unwrap(), + first + ); + output.clear(); + decoder + .decode_non_solid_member_to(&independent, first.len(), &mut output) + .unwrap(); + assert_eq!(output, first); + output.clear(); + decoder + .decode_non_solid_member_from_reader( + &mut TinyReader(&independent), + first.len(), + &mut output, + ) + .unwrap(); + assert_eq!(output, first); +} + +#[test] +fn rar15_public_adapters_keep_reader_and_sink_failures() { + let data = b"legacy codec failure context"; + let packed = Unpack15Encoder::new().encode_member(data).unwrap(); + let error = Unpack15::new() + .decode_member_from_reader(&mut FailedReader, data.len(), false, &mut Vec::new()) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::PermissionDenied, "source refused"); + let error = Unpack15::new() + .decode_member_to(&packed, data.len(), false, &mut FailedWriter) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::BrokenPipe, "sink refused"); +} + +#[test] +fn rar20_public_adapters_keep_reader_and_sink_failures() { + let data = b"legacy codec failure context"; + let packed = Unpack20Encoder::new().encode_member(data).unwrap(); + let error = Unpack20::new() + .decode_member_from_reader(&mut FailedReader, data.len(), &mut Vec::new()) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::PermissionDenied, "source refused"); + let error = Unpack20::new() + .decode_member_to(&packed, data.len(), &mut FailedWriter) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::BrokenPipe, "sink refused"); +} + +#[test] +fn rar29_public_adapters_keep_reader_and_sink_failures() { + let data = b"legacy codec failure context"; + let packed = Unpack29Encoder::new().encode_member(data).unwrap(); + let error = Unpack29::new() + .decode_member_from_reader(&mut FailedReader, data.len(), &mut Vec::new()) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::PermissionDenied, "source refused"); + let error = Unpack29::new() + .decode_member_to(&packed, data.len(), &mut FailedWriter) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::BrokenPipe, "sink refused"); + let error = Unpack29::new() + .decode_non_solid_member_from_reader(&mut FailedReader, data.len(), &mut Vec::new()) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::PermissionDenied, "source refused"); + let error = Unpack29::new() + .decode_non_solid_member_to(&packed, data.len(), &mut FailedWriter) + .unwrap_err(); + assert_io_error(error, io::ErrorKind::BrokenPipe, "sink refused"); +} diff --git a/arca-rar/vendor/rars/tests/legacy_name_encoding.rs b/arca-rar/vendor/rars/tests/legacy_name_encoding.rs new file mode 100644 index 00000000..b8b3b108 --- /dev/null +++ b/arca-rar/vendor/rars/tests/legacy_name_encoding.rs @@ -0,0 +1,218 @@ +#![cfg(feature = "write")] + +use rars::{ + filename::{decoded_name, LegacyNameEncoding}, + ArchiveReader, ArchiveVersion, Builder, +}; + +#[test] +fn strict_code_pages_are_explicit_and_locale_independent() { + for (label, bytes, expected) in [ + ("cp437", b"caf\x82".as_slice(), "café"), + ("CP850", b"\x9b".as_slice(), "ø"), + ("cp852", b"\x88".as_slice(), "Å‚"), + ("cp866", b"\x80".as_slice(), "Ð�"), + ("windows-1251", b"\xc0".as_slice(), "Ð�"), + ("windows-1252", b"\x80".as_slice(), "€"), + ("utf-8", "日本語".as_bytes(), "日本語"), + ] { + let encoding: LegacyNameEncoding = label.parse().unwrap(); + assert_eq!(encoding.decode(bytes).unwrap(), expected); + } + assert!("auto".parse::().is_err()); + assert!(LegacyNameEncoding::Windows1252.decode(b"\x81").is_err()); + assert!(LegacyNameEncoding::Windows1251.decode(b"\x98").is_err()); + assert!(LegacyNameEncoding::Utf8.decode(b"\xff").is_err()); + assert_eq!( + decoded_name(b"\x82", false, None).unwrap().as_ref(), + b"\x82" + ); + assert_eq!( + decoded_name("café".as_bytes(), true, Some(LegacyNameEncoding::Cp850)) + .unwrap() + .as_ref(), + "café".as_bytes() + ); +} + +#[test] +fn legacy_views_and_callbacks_preserve_original_identity() { + for format in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"caf\x82.txt".to_vec(), b"data".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + assert!(!member.name_is_unicode()); + assert_eq!( + member + .decoded_name(Some(LegacyNameEncoding::Cp850)) + .unwrap() + .as_ref(), + "café.txt".as_bytes() + ); + assert_eq!(member.meta.name, b"caf\x82.txt"); + archive + .extract_to(None, |meta| { + assert!(!meta.name_is_unicode); + assert_eq!(meta.name, b"caf\x82.txt"); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + } +} + +#[test] +fn unicode_names_take_precedence_in_metadata_and_extraction() { + for format in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes("café.txt".as_bytes().to_vec(), b"data".to_vec(), None, None) + .unwrap(); + if format == ArchiveVersion::Rar29 { + // Explicit full UTF-16 commands for café.txt, independent of fallback. + let mut wire = b"cafe.txt\0\0".to_vec(); + let units: Vec<_> = "café.txt".encode_utf16().collect(); + for group in units.chunks(4) { + wire.push(0xaa); + for unit in group { + wire.extend_from_slice(&unit.to_le_bytes()); + } + } + builder + .set_legacy_unicode_name("café.txt".as_bytes(), wire) + .unwrap(); + } + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + assert!(member.name_is_unicode()); + assert_eq!( + member + .decoded_name(Some(LegacyNameEncoding::Cp850)) + .unwrap() + .as_ref(), + "café.txt".as_bytes() + ); + archive + .extract_to(None, |meta| { + assert!(meta.name_is_unicode); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + } +} + +#[test] +fn rar3_direct_utf8_unicode_name_round_trips_without_a_separator() { + let name = "café.txt".as_bytes(); + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes(name.to_vec(), b"data".to_vec(), None, None) + .unwrap(); + builder + .set_legacy_unicode_name(name, name.to_vec()) + .unwrap(); + + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + assert_eq!(member.meta.name, name); + assert!(member.name_is_unicode()); + assert_eq!(member.decoded_name(None).unwrap().as_ref(), name); + + let invalid = b"\xff.txt"; + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes(invalid.to_vec(), b"data".to_vec(), None, None) + .unwrap(); + assert!(builder + .set_legacy_unicode_name(invalid, invalid.to_vec()) + .is_err()); +} + +#[test] +fn legacy_unicode_metadata_is_validated_and_renamed_with_its_member() { + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes(b"old.txt".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let before = builder.to_bytes().unwrap(); + assert!(builder + .set_legacy_unicode_name(b"old.txt", b"bad".to_vec()) + .is_err()); + assert_eq!(builder.to_bytes().unwrap(), before); + + let mut wire = b"old.txt\0\0".to_vec(); + for group in "old.txt".encode_utf16().collect::>().chunks(4) { + wire.push(0xaa); + for unit in group { + wire.extend_from_slice(&unit.to_le_bytes()); + } + } + builder + .set_legacy_unicode_name(b"old.txt", wire.clone()) + .unwrap(); + builder.rename(b"old.txt", b"new.txt".to_vec()).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = archive.members().next().unwrap(); + assert_eq!(member.meta.name, b"new.txt"); + assert!(member.name_is_unicode()); + + let mut old = Builder::new(ArchiveVersion::Rar15).store(true); + old.add_bytes(b"old.txt".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert!(old.set_legacy_unicode_name(b"old.txt", wire).is_err()); + + let mut volume = Builder::new(ArchiveVersion::Rar29) + .store(true) + .volume_size(Some(512)); + volume + .add_bytes(b"old.txt".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert!(volume + .set_legacy_unicode_name(b"old.txt", b"old.txt\0\0".to_vec()) + .is_err()); +} + +#[test] +fn split_legacy_names_keep_their_encoding_classification() { + let mut builder = Builder::new(ArchiveVersion::Rar29) + .store(true) + .volume_size(Some(512)); + builder + .add_bytes(b"caf\x82.txt".to_vec(), vec![42; 4096], None, None) + .unwrap(); + let archives: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + rars::extract_volumes_to(&archives, None, |meta| { + assert!(!meta.name_is_unicode); + assert_eq!( + decoded_name( + &meta.name, + meta.name_is_unicode, + Some(LegacyNameEncoding::Cp850) + ) + .unwrap() + .as_ref(), + "café.txt".as_bytes() + ); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); +} diff --git a/arca-rar/vendor/rars/tests/legacy_parallel_retention.rs b/arca-rar/vendor/rars/tests/legacy_parallel_retention.rs new file mode 100644 index 00000000..89b6a467 --- /dev/null +++ b/arca-rar/vendor/rars/tests/legacy_parallel_retention.rs @@ -0,0 +1,143 @@ +#![cfg(feature = "write")] +#![cfg(all( + feature = "parallel", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] + +use rars::{rar15_40, ArchiveReadOptions, ArchiveVersion, Builder, Error}; +use std::io::{self, Write}; +use std::sync::{Arc, Mutex}; + +const VERSIONS: [ArchiveVersion; 5] = [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, +]; + +fn archive(version: ArchiveVersion, stored: bool) -> rar15_40::Archive { + let mut builder = Builder::new(version).store(stored); + for index in 0..5 { + builder + .add_bytes( + index.to_string().into_bytes(), + vec![index as u8; index * 64], + None, + None, + ) + .unwrap(); + } + rar15_40::Archive::parse_owned(builder.to_bytes().unwrap()).unwrap() +} + +fn corrupt_member(archive: &mut rar15_40::Archive, index: usize) { + let file = archive + .blocks + .iter_mut() + .filter_map(|block| match block { + rar15_40::Block::File(file) => Some(file), + _ => None, + }) + .nth(index) + .unwrap(); + file.file_crc ^= 1; +} + +#[test] +fn publishes_one_worker_window_before_a_later_decode_failure() { + for workers in [1, 2] { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(workers) + .build() + .unwrap(); + pool.install(|| { + for version in VERSIONS { + for stored in [true, false] { + let mut archive = archive(version, stored); + corrupt_member(&mut archive, workers); + let mut opened = Vec::new(); + let error = archive + .extract_to_parallel_buffered(ArchiveReadOptions::new(), |meta| { + opened.push(meta.name.clone()); + Ok(Box::new(io::sink())) + }) + .unwrap_err(); + assert!(matches!(error.root_cause(), Error::Crc32Mismatch { .. })); + assert_eq!( + opened, + (0..workers) + .map(|index| index.to_string().into_bytes()) + .collect::>() + ); + } + } + }); + } +} + +struct Capture(Arc>>); + +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn preserves_order_and_content_across_windows_including_empty_members() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + pool.install(|| { + for version in VERSIONS { + for stored in [true, false] { + let archive = archive(version, stored); + let mut opened = Vec::new(); + let data = Arc::new(Mutex::new(Vec::new())); + archive + .extract_to_parallel_buffered(ArchiveReadOptions::new(), |meta| { + opened.push(meta.name.clone()); + Ok(Box::new(Capture(data.clone()))) + }) + .unwrap(); + assert_eq!(opened, [b"0", b"1", b"2", b"3", b"4"]); + assert_eq!( + *data.lock().unwrap(), + (0..5) + .flat_map(|i| vec![i as u8; i * 64]) + .collect::>() + ); + } + } + }); +} + +#[test] +fn publication_failure_stops_before_a_later_batch_is_decoded() { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap(); + pool.install(|| { + for version in VERSIONS { + let mut archive = archive(version, true); + corrupt_member(&mut archive, 2); + let mut opened = 0; + let error = archive + .extract_to_parallel_buffered(ArchiveReadOptions::new(), |_| { + opened += 1; + Err(io::Error::other("destination unavailable").into()) + }) + .unwrap_err(); + assert!(matches!(error.root_cause(), Error::Io(_))); + assert_eq!(opened, 1); + } + }); +} diff --git a/arca-rar/vendor/rars/tests/legacy_writer_cancellation.rs b/arca-rar/vendor/rars/tests/legacy_writer_cancellation.rs new file mode 100644 index 00000000..a39afa68 --- /dev/null +++ b/arca-rar/vendor/rars/tests/legacy_writer_cancellation.rs @@ -0,0 +1,398 @@ +#![cfg(feature = "write")] + +use rars::{ + ArchiveVersion, Builder, ErrorKind, WriteCancellation, WriteOperation, WriteProgress, + WriteProgressEvent, WriterResources, +}; +use std::{ + io::{self, Write}, + sync::atomic::{AtomicBool, Ordering}, +}; + +#[path = "support/scratch.rs"] +mod scratch; + +const FORMATS: [ArchiveVersion; 7] = [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, +]; + +#[test] +fn resource_cancellation_stops_source_loading() { + use rars::EntrySource; + use std::io::{Cursor, Read, Seek, SeekFrom}; + struct Source { + token: WriteCancellation, + bytes: Cursor>, + } + impl Read for Source { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + self.token.cancel(); + self.bytes.read(bytes) + } + } + impl Seek for Source { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.bytes.seek(pos) + } + } + for format in FORMATS { + for precancel in [false, true] { + let token = WriteCancellation::new(); + let source_token = token.clone(); + let source = EntrySource::from_opener(100_000, move || { + assert!(!precancel, "cancelled writer opened its source"); + Ok(Box::new(Source { + token: source_token.clone(), + bytes: Cursor::new(vec![42; 100_000]), + })) + }); + let mut builder = Builder::new(format).store(true); + builder + .add_source(b"source".to_vec(), source, None, None) + .unwrap(); + if precancel { + token.cancel(); + } + let resources = WriterResources::default().with_cancellation(token); + let mut output = Vec::new(); + assert_eq!( + builder + .write_to(&mut output, &resources, None) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + if precancel { + assert!(output.is_empty()); + } else { + // Only the archive header can precede source verification. + assert!(output.len() < 100_000); + } + } + } +} + +struct Stop { + operation: WriteOperation, + finished: bool, + cancelled: AtomicBool, +} +impl Stop { + fn new(operation: WriteOperation, finished: bool) -> Self { + Self { + operation, + finished, + cancelled: AtomicBool::new(false), + } + } +} +impl WriteProgress for Stop { + fn report(&self, event: WriteProgressEvent<'_>) { + let stop = match event { + WriteProgressEvent::Advanced { + operation, + completed_bytes, + .. + } => !self.finished && operation == self.operation && completed_bytes > 0, + WriteProgressEvent::BytesWritten { completed_bytes } => { + !self.finished && self.operation == WriteOperation::Emission && completed_bytes > 0 + } + WriteProgressEvent::OperationFinished { operation, .. } => { + self.finished && operation == self.operation + } + _ => false, + }; + if stop { + self.cancelled.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } +} + +fn builder(format: ArchiveVersion, store: bool) -> Builder { + let mut builder = Builder::new(format).store(store); + builder + .add_bytes( + b"payload".to_vec(), + b"legacy cancellation\n".repeat(8000), + None, + None, + ) + .unwrap(); + builder +} + +#[test] +fn cancellation_during_preparation_and_completion_preserves_destination() { + let scratch = scratch::case("legacy-writer-cancel"); + let destination = scratch.join("archive.rar"); + for format in FORMATS { + for store in [false, true] { + let builder = builder(format, store); + for operation in [WriteOperation::Compression, WriteOperation::Emission] { + for finished in [false, true] { + std::fs::write(&destination, b"original").unwrap(); + let stop = Stop::new(operation, finished); + let error = builder + .write_to_path(&destination, Some(&stop)) + .unwrap_err(); + assert_eq!( + error.kind(), + ErrorKind::Cancelled, + "{format:?}, {store}, {operation:?}" + ); + assert_eq!(std::fs::read(&destination).unwrap(), b"original"); + assert_eq!(std::fs::read_dir(&scratch).unwrap().count(), 1); + } + } + let passive = |_: WriteProgressEvent<'_>| {}; + assert_eq!( + builder.to_bytes().unwrap(), + builder.to_bytes_with_progress(Some(&passive)).unwrap() + ); + } + } +} + +#[test] +fn resource_token_interrupts_short_output_writes() { + struct Sink { + token: WriteCancellation, + calls: usize, + } + impl Write for Sink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.calls += 1; + self.token.cancel(); + Ok(bytes.len().min(17)) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for format in FORMATS { + let token = WriteCancellation::new(); + let resources = WriterResources::default().with_cancellation(token.clone()); + let mut sink = Sink { token, calls: 0 }; + assert_eq!( + builder(format, true) + .write_to(&mut sink, &resources, None) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + assert_eq!(sink.calls, 1); + } +} + +#[test] +fn cancellation_after_a_volume_does_not_return_a_partial_set() { + struct StopVolume(AtomicBool); + impl WriteProgress for StopVolume { + fn report(&self, event: WriteProgressEvent<'_>) { + if matches!(event, WriteProgressEvent::VolumeFinished { .. }) { + self.0.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + for format in FORMATS { + for store in [false, true] { + let stop = StopVolume(AtomicBool::new(false)); + let result = builder(format, store) + .volume_size(Some(64)) + .build_volumes(Some(&stop)); + assert_eq!( + result.unwrap_err().kind(), + ErrorKind::Cancelled, + "{format:?}, {store}" + ); + } + } +} + +#[test] +fn cancellation_from_final_zero_byte_progress_is_not_reported_as_success() { + struct StopAtFinalAdvance(AtomicBool); + impl WriteProgress for StopAtFinalAdvance { + fn report(&self, event: WriteProgressEvent<'_>) { + if matches!( + event, + WriteProgressEvent::Advanced { + operation: WriteOperation::Compression, + completed_bytes: 0, + total_bytes: 0, + .. + } + ) { + self.0.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + + let mut builder = Builder::new(ArchiveVersion::Rar30).store(true); + builder + .add_bytes(b"empty".to_vec(), Vec::new(), None, None) + .unwrap(); + let stop = StopAtFinalAdvance(AtomicBool::new(false)); + assert_eq!( + builder + .to_bytes_with_progress(Some(&stop)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); +} + +#[test] +fn compressed_legacy_volume_can_cancel_at_final_progress() { + use rars::{rar15_40, FeatureSet}; + use std::sync::atomic::AtomicUsize; + + struct StopAfterLastVolume { + final_volume: AtomicBool, + checks_after_final: AtomicUsize, + } + impl WriteProgress for StopAfterLastVolume { + fn report(&self, event: WriteProgressEvent<'_>) { + if let WriteProgressEvent::VolumeFinished { + volume_number, + total_volumes: Some(total_volumes), + .. + } = event + { + if volume_number == total_volumes { + self.final_volume.store(true, Ordering::Relaxed); + } + } + } + fn is_cancelled(&self) -> bool { + // The volume reporter checks once before returning. Cancel at + // the wrapper's final compression-progress update after that. + self.final_volume.load(Ordering::Relaxed) + && self.checks_after_final.fetch_add(1, Ordering::Relaxed) != 0 + } + } + + let data = b"legacy volume compression".repeat(100); + let entry = rar15_40::FileEntry { + name: b"file", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let progress = StopAfterLastVolume { + final_volume: AtomicBool::new(false), + checks_after_final: AtomicUsize::new(0), + }; + let result = rar15_40::write_compressed_volumes_with_progress( + entry, + rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::default()), + 1024, + Some(&progress), + ); + assert_eq!(result.unwrap_err().kind(), ErrorKind::Cancelled); + assert!(progress.final_volume.load(Ordering::Relaxed)); + assert!(progress.checks_after_final.load(Ordering::Relaxed) >= 2); +} + +#[test] +fn compressed_legacy_volume_preserves_cancellation_from_codec() { + use rars::{rar15_40, FeatureSet}; + + let data = b"a repeating legacy volume member ".repeat(4096); + let entry = rar15_40::FileEntry { + name: b"file", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let progress = Stop::new(WriteOperation::Compression, false); + let error = rar15_40::write_compressed_volumes_with_progress( + entry, + rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::default()), + 1024, + Some(&progress), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Cancelled); + assert!(progress.cancelled.load(Ordering::Relaxed)); +} + +#[test] +fn rar13_compressed_volume_preserves_cancellation_from_codec() { + use rars::rar13; + + let data = b"a repeating RAR 1.3 volume member ".repeat(4096); + let entry = rar13::FileEntry { + name: b"file", + data: &data, + file_time: 0, + file_attr: 0x20, + password: None, + file_comment: None, + }; + let progress = Stop::new(WriteOperation::Compression, false); + let error = rar13::write_compressed_volumes_with_progress( + entry, + rar13::WriterOptions::default(), + 1024, + Some(&progress), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Cancelled); + assert!(progress.cancelled.load(Ordering::Relaxed)); +} + +#[test] +fn rar29_filter_search_preserves_cancellation() { + use rars::{rar15_40, FeatureSet, FilterPolicy}; + + let mut state = 0x1357_9bdfu32; + let data: Vec = (0..16_384) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let entry = rar15_40::FileEntry { + name: b"binary", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let progress = Stop::new(WriteOperation::Compression, false); + let error = rar15_40::write_rar29_compressed_archive_with_filter_policy_and_progress( + &[entry], + rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::default()), + FilterPolicy::Auto, + Some(&progress), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Cancelled); + assert!(progress.cancelled.load(Ordering::Relaxed)); +} diff --git a/arca-rar/vendor/rars/tests/legacy_writer_errors.rs b/arca-rar/vendor/rars/tests/legacy_writer_errors.rs new file mode 100644 index 00000000..45a4bebf --- /dev/null +++ b/arca-rar/vendor/rars/tests/legacy_writer_errors.rs @@ -0,0 +1,944 @@ +#![cfg(feature = "write")] + +use rars::{ + rar13, rar15_40, ArchiveVersion, Builder, EntrySource, Error, ErrorKind, FeatureSet, + FilterKind, FilterPolicy, MemberCoding, WriteCancellation, WriterResources, +}; + +#[path = "support/scratch.rs"] +mod scratch; +use std::{ + io::{self, Cursor, Read, Seek, SeekFrom, Write}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; + +const FORMATS: [ArchiveVersion; 7] = [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, +]; + +#[test] +fn encrypted_legacy_volumes_preserve_non_utf8_password_failure_context() { + let payload = b"legacy compressed volume payload".repeat(32); + for format in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for header_encryption in [false, true] { + if format == ArchiveVersion::Rar29 && header_encryption { + continue; + } + let mut features = FeatureSet::store_only(); + features.header_encryption = header_encryption; + let entry = rar15_40::FileEntry { + name: b"member", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"\xff"), + file_comment: None, + }; + let error = rar15_40::write_compressed_volumes( + entry, + rar15_40::WriterOptions::new(format, features), + 64, + ) + .unwrap_err(); + assert_eq!( + error.root_cause(), + &Error::Rar30Crypto(rars::crypto::rar30::Error::NonUtf8Password) + ); + assert_eq!( + error.entry_context(), + Some((b"member".as_slice(), "encrypting volume member")) + ); + } + } +} + +#[test] +fn compressed_legacy_volume_rejects_an_invalid_target_before_encoding() { + let entry = rar15_40::FileEntry { + name: b"file", + data: b"payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let error = rar15_40::write_compressed_volumes( + entry, + rar15_40::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 1024, + ) + .unwrap_err(); + assert_eq!(error, Error::UnsupportedVersion(ArchiveVersion::Rar50)); +} + +#[test] +fn invalid_legacy_compression_level_fails_before_member_io_or_output() { + let entries = [rar15_40::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("invalid level opened the source")), + )]; + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_compression_level(6), + MemberCoding::Compressed, + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::UnsupportedFeature); + assert!(output.is_empty()); +} + +#[test] +fn legacy_header_encryption_requires_password_before_member_io_or_output() { + let entries = [rar15_40::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("missing password opened member")), + )]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(ArchiveVersion::Rar30, features), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert!(matches!(error, Error::UnsupportedWriterOption { .. })); + assert!(output.is_empty()); +} + +#[test] +fn oversized_legacy_comment_fails_before_opening_member_source() { + let entries = [rar15_40::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("oversized comment opened member")), + )]; + let comment = vec![b'x'; 65536]; + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(ArchiveVersion::Rar15, FeatureSet::store_only()), + MemberCoding::Stored, + Some(&comment), + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert!(!output.is_empty()); +} + +#[test] +fn embedded_comment_that_exceeds_file_header_size_is_rejected() { + for (comment_len, fits) in [(65486, true), (65487, false)] { + let mut builder = Builder::new(ArchiveVersion::Rar29).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(vec![b'x'; comment_len])) + .unwrap(); + if fits { + let archive = rars::ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + } else { + let error = builder.to_bytes().unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert_eq!(error.entry_context(), Some((b"file".as_slice(), "writing"))); + } + } +} + +#[test] +fn invalid_rar29_filters_fail_before_opening_a_member_or_writing_output() { + let entries = [rar15_40::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("invalid filter opened the source")), + )]; + let options = rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + let cases = [ + ( + FilterKind::Delta { channels: 0 }, + ErrorKind::InvalidArgument, + ), + ( + FilterKind::Delta { channels: 33 }, + ErrorKind::InvalidArgument, + ), + ( + FilterKind::Audio { channels: 0 }, + ErrorKind::InvalidArgument, + ), + ( + FilterKind::Audio { channels: 33 }, + ErrorKind::InvalidArgument, + ), + ( + FilterKind::Rgb { width: 0, pos_r: 0 }, + ErrorKind::InvalidArgument, + ), + ( + FilterKind::Rgb { width: 4, pos_r: 0 }, + ErrorKind::InvalidArgument, + ), + ( + FilterKind::Rgb { width: 3, pos_r: 3 }, + ErrorKind::InvalidArgument, + ), + (FilterKind::Arm, ErrorKind::UnsupportedFeature), + ]; + for (filter, expected) in cases { + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + options, + MemberCoding::Filtered(FilterPolicy::explicit(filter)), + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), expected, "{filter:?}: {error}"); + assert!(output.is_empty(), "{filter:?} wrote archive bytes"); + } + + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + options.with_method(rar15_40::Rar29Method::Ppmd), + MemberCoding::Filtered(FilterPolicy::Auto), + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert!(output.is_empty()); +} + +#[test] +fn rar13_family_rejects_filter_policy_before_member_io_or_output() { + let entries = [rar13::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("unsupported filter opened the source")), + )]; + for format in [ArchiveVersion::Rar13, ArchiveVersion::Rar14] { + let mut output = Vec::new(); + let error = rar13::write_streaming_archive_to( + &entries, + rar13::WriterOptions::new(format, FeatureSet::store_only()), + MemberCoding::Filtered(FilterPolicy::Auto), + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert!(matches!(error, Error::UnsupportedWriterOption { .. })); + assert!(output.is_empty()); + } +} + +#[test] +fn legacy_dictionary_size_is_validated_before_member_io() { + let entries = [rar15_40::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("invalid dictionary opened the source")), + )]; + for size in [0, 96 * 1024, 8 * 1024 * 1024] { + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_dictionary_size(size), + MemberCoding::Compressed, + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument, "{size}: {error}"); + assert!(output.is_empty()); + } +} + +#[test] +fn legacy_resource_limits_fail_before_opening_a_member_or_writing_output() { + let entries = [rar15_40::StreamingEntry::new( + b"member".to_vec(), + EntrySource::from_opener(1, || panic!("unsupported limit opened the source")), + )]; + let options = rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + for (resources, expected_feature) in [ + ( + WriterResources::default().with_max_preparation_bytes(1), + "preparation memory quota", + ), + ( + WriterResources::default().with_max_memory_bytes(1), + "aggregate managed-memory limit", + ), + ( + WriterResources::default() + .with_max_preparation_bytes(1) + .with_max_memory_bytes(1), + "aggregate managed-memory limit", + ), + ] { + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + options, + MemberCoding::Stored, + None, + &resources, + None, + &mut output, + ) + .unwrap_err(); + assert!( + matches!(error, Error::UnsupportedFamilyFeature { feature, .. } if feature == expected_feature), + "{error}" + ); + assert!(output.is_empty()); + } +} + +#[test] +fn legacy_source_length_failure_identifies_the_member_before_output() { + let root = scratch::case("legacy-missing-source-length"); + let entries = [rar15_40::StreamingEntry::new( + b"missing".to_vec(), + EntrySource::from_path(root.join("does-not-exist")), + )]; + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((&b"missing"[..], "reading source")) + ); + assert!(output.is_empty()); +} + +#[test] +fn legacy_archive_rejects_modern_target_before_encrypted_member_io() { + let entries = [rar15_40::StreamingEntry::new( + b"file".to_vec(), + EntrySource::from_opener(1, || panic!("wrong target opened member")), + ) + .with_password(b"secret".to_vec())]; + let mut output = Vec::new(); + let error = rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut output, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::UnsupportedVersion(ArchiveVersion::Rar50) + )); + assert!(output.is_empty()); +} + +#[test] +fn legacy_volume_preflight_rejects_unsupported_options_and_empty_payloads() { + let entry = rar15_40::StoredEntry { + name: b"member", + data: b"payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let plain = rar15_40::WriterOptions::new(ArchiveVersion::Rar30, FeatureSet::store_only()); + let error = rar15_40::write_stored_volumes( + entry, + rar15_40::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 8, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::UnsupportedVersion(ArchiveVersion::Rar50) + )); + + let error = rar15_40::write_stored_volumes( + rar15_40::StoredEntry { + file_comment: Some(b"comment"), + ..entry + }, + plain, + 8, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::UnsupportedFeature { + feature: "volume_file_comment", + .. + } + )); + + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let encrypted = rar15_40::WriterOptions::new(ArchiveVersion::Rar30, features); + let no_password = rar15_40::StoredEntry { + password: None, + ..entry + }; + assert!(matches!( + rar15_40::write_stored_volumes(no_password, encrypted, 8), + Err(Error::UnsupportedWriterOption { .. }) + )); + assert_eq!( + rar15_40::write_stored_volumes(entry, encrypted, 0) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + let empty = rar15_40::StoredEntry { data: b"", ..entry }; + assert_eq!( + rar15_40::write_stored_volumes(empty, plain, 8) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); + assert_eq!( + rar15_40::write_stored_volumes(empty, encrypted, 8) + .unwrap_err() + .kind(), + ErrorKind::InvalidArgument + ); +} + +#[test] +fn volume_validation_identifies_members_but_not_global_options() { + for format in FORMATS { + for store in [false, true] { + let mut builder = Builder::new(format).store(store).volume_size(Some(64)); + let name = vec![b'a'; 65536]; + builder + .add_bytes(name.clone(), b"payload".to_vec(), None, None) + .unwrap(); + let error = builder.build_volumes(None).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert_eq!( + error.entry_context(), + Some((name.as_slice(), "preparing volume member")) + ); + + let mut builder = Builder::new(format).store(store).volume_size(Some(0)); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let error = builder.build_volumes(None).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert!(error.entry_context().is_none()); + } + } +} + +#[test] +fn encrypted_and_plain_volume_framing_failures_identify_the_member() { + for encrypted in [false, true] { + let name = vec![b'a'; 65535]; + let mut builder = Builder::new(ArchiveVersion::Rar30) + .store(true) + .volume_size(Some(64)); + if encrypted { + builder = builder + .password(Some(b"secret".to_vec())) + .header_encryption(true); + } + builder + .add_bytes(name.clone(), b"payload".to_vec(), None, None) + .unwrap(); + let error = builder.build_volumes(None).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert_eq!( + error.entry_context(), + Some((name.as_slice(), "writing volume member")) + ); + } +} + +fn write( + format: ArchiveVersion, + source: EntrySource, + compressed: bool, + solid: bool, + resources: &WriterResources, + output: &mut dyn Write, +) -> rars::Result<()> { + let mut features = FeatureSet::store_only(); + features.solid = solid; + let coding = if compressed { + MemberCoding::Compressed + } else { + MemberCoding::Stored + }; + if matches!(format, ArchiveVersion::Rar13 | ArchiveVersion::Rar14) { + let entries = [ + rar13::StreamingEntry::new( + b"first".to_vec(), + EntrySource::from_bytes(b"good".to_vec()), + ), + rar13::StreamingEntry::new(b"second".to_vec(), source), + ]; + rar13::write_streaming_archive_to( + &entries, + rar13::WriterOptions::new(format, features), + coding, + None, + resources, + None, + output, + ) + } else { + let entries = [ + rar15_40::StreamingEntry::new( + b"first".to_vec(), + EntrySource::from_bytes(b"good".to_vec()), + ), + rar15_40::StreamingEntry::new(b"second".to_vec(), source), + ]; + rar15_40::write_streaming_archive_to( + &entries, + rar15_40::WriterOptions::new(format, features), + coding, + None, + resources, + None, + output, + ) + } +} + +#[test] +fn source_failures_identify_the_member_in_serial_and_parallel_preparation() { + for format in FORMATS { + for (compressed, solid) in [(false, false), (true, false), (true, true)] { + let source = EntrySource::from_opener(4, || { + Err(io::Error::new(io::ErrorKind::PermissionDenied, "source denied").into()) + }); + let error = write( + format, + source, + compressed, + solid, + &WriterResources::default(), + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!(error.entry_context(), Some((&b"second"[..], "preparing"))); + assert!( + matches!(error.root_cause(), Error::Io(io) if io.kind == std::io::ErrorKind::PermissionDenied) + ); + } + } +} + +#[test] +fn changed_contents_and_lengths_have_the_same_source_changed_category() { + for format in FORMATS { + for same_length in [false, true] { + let opens = Arc::new(AtomicUsize::new(0)); + let source = EntrySource::from_opener(4, move || { + let data = if opens.fetch_add(1, Ordering::Relaxed) == 0 { + b"good".to_vec() + } else if same_length { + b"evil".to_vec() + } else { + b"short".to_vec() + }; + Ok(Box::new(Cursor::new(data))) + }); + let error = write( + format, + source, + false, + false, + &WriterResources::default(), + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::SourceChanged, "{format:?}"); + assert_eq!(error.entry_context(), Some((&b"second"[..], "writing"))); + } + } +} + +#[test] +fn builder_source_materialization_identifies_the_member() { + for format in FORMATS { + for (store, solid) in [(true, false), (false, false), (false, true)] { + for actual_length in [1, 4] { + let mut builder = Builder::new(format).store(store).solid(solid); + builder + .add_source( + b"source".to_vec(), + EntrySource::from_opener(3, move || { + Ok(Box::new(Cursor::new(vec![1; actual_length]))) + }), + None, + None, + ) + .unwrap(); + for error in builder_errors(&builder) { + assert_eq!(error.kind(), ErrorKind::SourceChanged, "{format:?}"); + assert_eq!( + error.entry_context(), + Some((&b"source"[..], "reading source")) + ); + } + } + } + } +} + +fn builder_errors(builder: &Builder) -> [Error; 3] { + let resources = WriterResources::default(); + [ + builder.to_bytes().unwrap_err(), + builder + .to_bytes_with_resources(&resources, None) + .unwrap_err(), + builder + .write_to(&mut Vec::new(), &resources, None) + .unwrap_err(), + ] +} + +fn write_stored_source( + format: ArchiveVersion, + source: EntrySource, + resources: &WriterResources, + output: &mut dyn Write, + through_builder: bool, +) -> rars::Result<()> { + if through_builder { + let mut builder = Builder::new(format).store(true); + builder.add_source(b"second".to_vec(), source, None, None)?; + builder.write_to(output, resources, None) + } else { + write(format, source, false, false, resources, output) + } +} + +#[test] +fn builder_source_io_failures_keep_reading_context() { + struct FailingRead { + cursor: Cursor>, + fail_at: u64, + } + impl Read for FailingRead { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if self.cursor.position() >= self.fail_at { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "source read denied", + )) + } else { + self.cursor.read(output) + } + } + } + impl Seek for FailingRead { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.cursor.seek(position) + } + } + let root = scratch::case("builder-source-io-context"); + for format in FORMATS { + for (store, solid) in [(true, false), (false, false), (false, true)] { + // A path has a fallible length lookup. The other cases fail while + // opening, consuming advertised bytes, or probing for extra bytes. + for failure in 0..4 { + let source = match failure { + 0 => EntrySource::from_path(root.join("absent")), + 1 => EntrySource::from_opener(4, || { + Err( + io::Error::new(io::ErrorKind::PermissionDenied, "source open denied") + .into(), + ) + }), + _ => EntrySource::from_opener(4, move || { + Ok(Box::new(FailingRead { + cursor: Cursor::new(b"good".to_vec()), + fail_at: if failure == 2 { 0 } else { 4 }, + })) + }), + }; + let mut builder = Builder::new(format).store(store).solid(solid); + let name = b"source-\xff"; + builder + .add_source(name.to_vec(), source, None, None) + .unwrap(); + for error in builder_errors(&builder) { + assert_eq!(error.kind(), ErrorKind::Io, "{format:?}: {failure}"); + assert_eq!( + error.entry_context(), + Some((name.as_slice(), "reading source")) + ); + assert!(matches!(error.root_cause(), Error::Io(io) + if io.kind == if failure == 0 { io::ErrorKind::NotFound } else { io::ErrorKind::PermissionDenied })); + } + } + } + } +} + +#[test] +fn builder_source_context_preserves_existing_errors_and_cancellation() { + for format in FORMATS { + for store in [false, true] { + for original in [ + Error::SourceChanged("upstream read failed") + .at_entry(b"upstream".to_vec(), "opening"), + Error::Cancelled, + ] { + let mut builder = Builder::new(format).store(store); + builder + .add_source( + b"source".to_vec(), + EntrySource::from_opener(4, { + let original = original.clone(); + move || Err(original.clone()) + }), + None, + None, + ) + .unwrap(); + for error in builder_errors(&builder) { + assert_eq!(error, original); + } + } + } + } +} + +#[test] +fn builder_member_validation_keeps_preparation_context() { + for format in [ArchiveVersion::Rar13, ArchiveVersion::Rar14] { + let name = vec![b'a'; 256]; + let mut builder = Builder::new(format); + builder + .add_source( + name.clone(), + EntrySource::from_opener(4, || panic!("invalid member must not read its source")), + None, + None, + ) + .unwrap(); + for error in builder_errors(&builder) { + assert_eq!(error.kind(), ErrorKind::InvalidArgument); + assert_eq!(error.entry_context(), Some((name.as_slice(), "preparing"))); + } + } +} + +#[test] +fn member_payload_output_failure_retains_io_kind_and_member() { + use std::sync::atomic::AtomicBool; + struct Output(Arc); + impl Write for Output { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.0.load(Ordering::Relaxed) { + Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "payload sink closed", + )) + } else { + Ok(bytes.len()) + } + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for (format, through_builder) in FORMATS + .into_iter() + .flat_map(|format| [(format, false), (format, true)]) + { + let broken = Arc::new(AtomicBool::new(false)); + let source = EntrySource::from_opener(4, { + let broken = broken.clone(); + let opens = AtomicUsize::new(0); + move || { + if opens.fetch_add(1, Ordering::Relaxed) == 1 { + broken.store(true, Ordering::Relaxed); + } + Ok(Box::new(Cursor::new(b"good".to_vec()))) + } + }); + let error = write_stored_source( + format, + source, + &WriterResources::default(), + &mut Output(broken), + through_builder, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!(error.entry_context(), Some((&b"second"[..], "writing"))); + } +} + +#[test] +fn cancellation_and_shared_output_failures_do_not_blame_a_member() { + struct BrokenOutput; + impl Write for BrokenOutput { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::new(io::ErrorKind::BrokenPipe, "closed")) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for (format, through_builder) in FORMATS + .into_iter() + .flat_map(|format| [(format, false), (format, true)]) + { + let error = write_stored_source( + format, + EntrySource::from_bytes(b"good".to_vec()), + &WriterResources::default(), + &mut BrokenOutput, + through_builder, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!(error.entry_context(), None); + + let token = WriteCancellation::new(); + let source = EntrySource::from_opener(4, { + let token = token.clone(); + move || { + token.cancel(); + Ok(Box::new(Cursor::new(b"good".to_vec()))) + } + }); + let error = write_stored_source( + format, + source, + &WriterResources::default().with_cancellation(token), + &mut Vec::new(), + through_builder, + ) + .unwrap_err(); + assert_eq!(error, Error::Cancelled); + } +} + +#[test] +fn legacy_emission_propagates_failure_at_every_output_write() { + struct FailOnWrite { + fail_at: usize, + writes: usize, + } + impl Write for FailOnWrite { + fn write(&mut self, data: &[u8]) -> io::Result { + let call = self.writes; + self.writes += 1; + if call == self.fail_at { + Err(io::Error::new(io::ErrorKind::BrokenPipe, "closed output")) + } else { + Ok(data.len()) + } + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + for (format, compressed, solid) in FORMATS.into_iter().flat_map(|format| { + [ + (format, false, false), + (format, true, false), + (format, true, true), + ] + }) { + let run = |sink: &mut dyn Write| { + write( + format, + EntrySource::from_bytes(b"good".to_vec()), + compressed, + solid, + &WriterResources::default(), + sink, + ) + }; + let mut counting = FailOnWrite { + fail_at: usize::MAX, + writes: 0, + }; + run(&mut counting).unwrap(); + assert!(counting.writes >= 4, "{format:?}"); + + let mut member_failure = false; + let mut archive_failure = false; + for fail_at in 0..counting.writes { + let mut sink = FailOnWrite { fail_at, writes: 0 }; + let error = run(&mut sink).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{format:?} write {fail_at}"); + assert_eq!(sink.writes, fail_at + 1); + if let Some((name, operation)) = error.entry_context() { + assert!(name == b"first" || name == b"second"); + assert_eq!(operation, "writing"); + member_failure = true; + } else { + archive_failure = true; + } + } + assert!(member_failure, "{format:?}"); + assert!(archive_failure, "{format:?}"); + } +} diff --git a/arca-rar/vendor/rars/tests/member_selection.rs b/arca-rar/vendor/rars/tests/member_selection.rs new file mode 100644 index 00000000..d5b37b87 --- /dev/null +++ b/arca-rar/vendor/rars/tests/member_selection.rs @@ -0,0 +1,527 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReader, ArchiveVersion, Builder, ErrorKind}; + +#[test] +fn volume_lookup_counts_redirections_in_logical_member_indices() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let mut first = Builder::new(format).store(true); + first.add_directory(b"dir".to_vec(), None, None).unwrap(); + first + .add_unix_symlink(b"link".to_vec(), b"first".to_vec(), false, None, None) + .unwrap(); + first + .add_bytes(b"first".to_vec(), b"first payload".to_vec(), None, None) + .unwrap(); + let mut second = Builder::new(format).store(true); + second + .add_bytes(b"last".to_vec(), b"last payload".to_vec(), None, None) + .unwrap(); + let archives = [ + ArchiveReader::read_owned(first.to_bytes().unwrap()).unwrap(), + ArchiveReader::read_owned(second.to_bytes().unwrap()).unwrap(), + ]; + let members = rars::volume_members(&archives).unwrap(); + assert_eq!(members.len(), 4); + for (index, expected) in [ + None, + None, + Some(b"first payload".as_slice()), + Some(b"last payload"), + ] + .into_iter() + .enumerate() + { + assert_eq!( + rars::read_volume_member_at(&archives, index, None) + .unwrap() + .as_deref(), + expected, + "{format}, index {index}" + ); + } + assert_eq!( + rars::read_volume_member_at(&archives, 4, None).unwrap(), + None + ); + let selection = [3, 2, 1, 3, 0, usize::MAX]; + let expected = vec![ + Some(b"last payload".to_vec()), + Some(b"first payload".to_vec()), + None, + Some(b"last payload".to_vec()), + None, + None, + ]; + assert_eq!( + rars::read_volume_members_at(&archives, &selection, None).unwrap(), + expected + ); + let options = rars::ArchiveReadOptions::new().with_max_total_output_bytes(25); + assert_eq!( + rars::read_volume_members_at_with_options(&archives, &selection, options).unwrap(), + expected + ); + assert_eq!( + rars::read_volume_members_at_with_options( + &archives, + &selection, + options.with_max_total_output_bytes(24) + ) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + } +} + +fn mixed(format: ArchiveVersion, solid: bool, stored: bool) -> Archive { + let mut builder = Builder::new(format).solid(solid).store(stored); + builder + .add_directory(b"directory".to_vec(), None, None) + .unwrap(); + for name in [b"first".as_slice(), b"secret", b"last"] { + builder + .add_bytes(name.to_vec(), name.repeat(100), None, None) + .unwrap(); + } + builder + .set_entry_encryption(b"secret", Some(b"password".to_vec()), None) + .unwrap(); + ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() +} + +#[test] +fn batch_reads_share_budgets_and_decode_solid_dependencies_once() { + use rars::{ArchiveReadOptions, ReadCancellation}; + for format in ArchiveVersion::ALL { + for solid in [false, true] { + let archive = mixed(format, solid, false); + assert_eq!(archive.is_solid(), solid, "{format}"); + let limit = if solid { 1500 } else { 900 }; + let options = + ArchiveReadOptions::with_password(b"password").with_max_total_output_bytes(limit); + let expected = vec![ + Some(b"last".repeat(100)), + None, + Some(b"first".repeat(100)), + Some(b"last".repeat(100)), + None, + ]; + assert_eq!( + archive + .read_members_at_with_options(&[3, 0, 1, 3, usize::MAX], options) + .unwrap(), + expected, + "{format}, solid={solid}" + ); + assert_eq!( + archive + .read_members_at_with_options( + &[3, 1, 3], + options.with_max_total_output_bytes(limit - 1) + ) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + let without_password = archive.read_members_at(&[3, 1], None); + if solid { + assert_eq!( + without_password.unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + } else { + assert_eq!( + without_password.unwrap(), + vec![Some(b"last".repeat(100)), Some(b"first".repeat(100))] + ); + } + assert_eq!( + archive.read_members_at(&[1], None).unwrap(), + vec![Some(b"first".repeat(100))] + ); + assert_eq!( + archive.read_members_at(&[0, usize::MAX], None).unwrap(), + vec![None, None] + ); + assert!(archive.read_members_at(&[], None).unwrap().is_empty()); + let token = ReadCancellation::new(); + token.cancel(); + for selection in [&[][..], &[0, usize::MAX][..], &[3][..]] { + assert_eq!( + archive + .read_members_at_with_options(selection, options.with_cancellation(&token)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + } + } + } +} + +#[test] +fn batch_reads_verify_selected_payloads_and_preserve_redirection_indices() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_unix_symlink(b"link".to_vec(), b"first".to_vec(), false, None, None) + .unwrap(); + builder + .add_bytes(b"first".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"empty".to_vec(), vec![], None, None) + .unwrap(); + builder + .add_bytes(b"bad".to_vec(), b"bad checksum".to_vec(), None, None) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + if let Archive::Rar50Plus(inner) = &mut archive { + let bad = inner + .blocks + .iter_mut() + .filter_map(|block| match block { + rars::rar50::Block::File(file) => Some(file), + _ => None, + }) + .last() + .unwrap(); + bad.data_crc32 = Some(bad.data_crc32.unwrap() ^ 1); + bad.hash = None; // Exercise CRC verification rather than the stronger hash. + } + assert_eq!( + archive.read_members_at(&[2, 0, 1, 2], None).unwrap(), + vec![Some(vec![]), None, Some(b"payload".to_vec()), Some(vec![])] + ); + assert_eq!( + archive.read_members_at(&[1, 3], None).unwrap_err().kind(), + ErrorKind::ChecksumMismatch + ); +} + +#[test] +fn batch_reads_reduce_payload_io_compared_with_repeated_solid_reads() { + use std::io::{self, Cursor, Read, Seek, SeekFrom}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + struct Counted { + source: Cursor>, + bytes: Arc, + } + impl Read for Counted { + fn read(&mut self, out: &mut [u8]) -> io::Result { + let count = self.source.read(out)?; + self.bytes.fetch_add(count, Ordering::Relaxed); + Ok(count) + } + } + impl Seek for Counted { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.source.seek(position) + } + } + for format in [ArchiveVersion::Rar29, ArchiveVersion::Rar50] { + let mut builder = Builder::new(format).solid(true); + for name in [b"first".as_slice(), b"last"] { + builder + .add_bytes(name.to_vec(), name.repeat(100), None, None) + .unwrap(); + } + let bytes = Arc::new(AtomicUsize::new(0)); + let archive = ArchiveReader::read_reader(Counted { + source: Cursor::new(builder.to_bytes().unwrap()), + bytes: Arc::clone(&bytes), + }) + .unwrap(); + bytes.store(0, Ordering::Relaxed); + let selection = [1, 0, 1]; + let expected: Vec<_> = selection + .iter() + .map(|&index| archive.read_member_at(index, None).unwrap()) + .collect(); + let repeated = bytes.swap(0, Ordering::Relaxed); + assert_eq!(archive.read_members_at(&selection, None).unwrap(), expected); + let batch = bytes.load(Ordering::Relaxed); + assert!( + batch > 0 && batch < repeated, + "{format}: {batch} bytes in batch vs {repeated} repeated" + ); + } +} + +#[test] +fn plaintext_selection_ignores_independent_encrypted_members_before_and_after_it() { + for format in ArchiveVersion::ALL { + for stored in [false, true] { + let archive = mixed(format, false, stored); + for (index, name) in [(1, b"first".as_slice()), (3, b"last")] { + for password in [None, Some(b"wrong".as_slice())] { + assert_eq!( + archive.read_member(name, password).unwrap().unwrap(), + name.repeat(100), + "{format}" + ); + assert_eq!( + archive.read_member_at(index, password).unwrap().unwrap(), + name.repeat(100), + "{format}" + ); + } + } + assert_eq!( + archive.read_member(b"secret", None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!( + archive.read_member_at(2, None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!( + archive + .read_member(b"secret", Some(b"password")) + .unwrap() + .unwrap(), + b"secret".repeat(100) + ); + assert!(archive.read_member(b"secret", Some(b"wrong")).is_err()); + assert_eq!( + archive.test(None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!(archive.read_member(b"missing", None).unwrap(), None); + assert_eq!(archive.read_member(b"directory", None).unwrap(), None); + assert_eq!(archive.read_member_at(0, None).unwrap(), None); + assert_eq!(archive.read_member_at(10, None).unwrap(), None); + } + } +} + +#[test] +fn solid_selection_stops_after_target_but_verifies_encrypted_predecessors() { + for format in ArchiveVersion::ALL { + let archive = mixed(format, true, false); + assert_eq!( + archive.read_member(b"first", None).unwrap().unwrap(), + b"first".repeat(100) + ); + assert_eq!( + archive.read_member_at(1, None).unwrap().unwrap(), + b"first".repeat(100) + ); + assert_eq!( + archive.read_member(b"last", None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!( + archive.read_member_at(3, None).unwrap_err().kind(), + ErrorKind::PasswordRequired + ); + assert_eq!( + archive + .read_member(b"last", Some(b"password")) + .unwrap() + .unwrap(), + b"last".repeat(100) + ); + assert!(archive.read_member(b"last", Some(b"wrong")).is_err()); + } +} + +#[test] +fn selection_preserves_target_integrity_and_last_duplicate_name_identity() { + for solid in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar29).solid(solid); + builder + .add_bytes(b"first".to_vec(), b"first payload".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"second".to_vec(), b"second payload".to_vec(), None, None) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + if let Archive::Rar15To40(legacy) = &mut archive { + let mut files = legacy.blocks.iter_mut().filter_map(|block| match block { + rars::rar15_40::Block::File(file) => Some(file), + _ => None, + }); + let first = files.next().unwrap(); + first.file_crc ^= 1; + first.name = b"same".to_vec(); + files.next().unwrap().name = b"same".to_vec(); + } + assert_eq!( + archive.read_member_at(0, None).unwrap_err().kind(), + ErrorKind::ChecksumMismatch + ); + if solid { + assert_eq!( + archive.read_member(b"same", None).unwrap_err().kind(), + ErrorKind::ChecksumMismatch + ); + } else { + assert_eq!( + archive.read_member(b"same", None).unwrap().unwrap(), + b"second payload" + ); + assert_eq!( + archive.read_member_at(1, None).unwrap().unwrap(), + b"second payload" + ); + } + assert_eq!( + archive.test(None).unwrap_err().kind(), + ErrorKind::ChecksumMismatch + ); + } +} + +#[test] +fn option_bearing_reads_apply_budgets_to_selected_payloads_and_solid_dependencies() { + use rars::ArchiveReadOptions; + for format in ArchiveVersion::ALL { + for solid in [false, true] { + let archive = mixed(format, solid, false); + let required = if solid { 1500 } else { 400 }; + let options = ArchiveReadOptions::with_password(b"password") + .with_max_total_output_bytes(required); + for _ in 0..2 { + assert_eq!( + archive + .read_member_with_options(b"last", options) + .unwrap() + .unwrap(), + b"last".repeat(100) + ); + assert_eq!( + archive + .read_member_at_with_options(3, options) + .unwrap() + .unwrap(), + b"last".repeat(100) + ); + } + let limited = options.with_max_total_output_bytes(required - 1); + assert!( + archive.read_member_with_options(b"last", limited).is_err(), + "{format}, solid={solid}" + ); + assert_eq!( + archive + .read_member_at_with_options(3, limited) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + let member_limit = options.with_max_member_output_bytes(400); + assert_eq!( + archive + .read_member_with_options(b"last", member_limit) + .is_err(), + solid + ); + assert!(archive + .test_with_options(options.with_max_total_output_bytes(1499)) + .is_err()); + archive + .test_with_options(options.with_max_total_output_bytes(1500)) + .unwrap(); + // Failed policy-bearing calls do not retain policy in the archive. + archive.test(Some(b"password")).unwrap(); + } + } +} + +#[test] +fn option_bearing_lookup_keeps_duplicate_identity_and_checks_cancellation_before_empty_selection() { + use rars::{ArchiveReadOptions, ReadCancellation}; + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .allow_duplicate_names(true); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"same".to_vec(), b"first".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"same".to_vec(), b"second".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"empty".to_vec(), Vec::new(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let options = ArchiveReadOptions::new().with_max_member_output_bytes(5); + assert_eq!( + archive + .read_member_at_with_options(1, options) + .unwrap() + .unwrap(), + b"first" + ); + assert!(archive.read_member_with_options(b"same", options).is_err()); + let zero = options.with_max_total_output_bytes(0); + assert_eq!( + archive.read_member_with_options(b"empty", zero).unwrap(), + Some(Vec::new()) + ); + assert_eq!( + archive.read_member_with_options(b"missing", zero).unwrap(), + None + ); + assert_eq!(archive.read_member_at_with_options(0, zero).unwrap(), None); + let token = ReadCancellation::new(); + token.cancel(); + let cancelled = options.with_cancellation(&token); + for name in [b"same".as_slice(), b"empty", b"missing", b"dir"] { + assert_eq!( + archive + .read_member_with_options(name, cancelled) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + } + for index in [0, 1, 3, 99] { + assert_eq!( + archive + .read_member_at_with_options(index, cancelled) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + } + assert_eq!( + archive.test_with_options(cancelled).unwrap_err().kind(), + ErrorKind::Cancelled + ); +} + +#[test] +fn option_bearing_helpers_keep_rar5_dictionary_admission() { + use rars::ArchiveReadOptions; + let mut builder = Builder::new(ArchiveVersion::Rar50).compression_level(Some(1)); + builder + .add_bytes(b"file".to_vec(), b"repeated".repeat(1000), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let options = ArchiveReadOptions::new().with_rar50_dictionary_size_limit(0); + assert_eq!( + archive + .read_member_with_options(b"file", options) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!( + archive + .read_member_at_with_options(0, options) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!( + archive.test_with_options(options).unwrap_err().kind(), + ErrorKind::ResourceLimit + ); + archive.test(None).unwrap(); +} diff --git a/arca-rar/vendor/rars/tests/ppmd_regressions.rs b/arca-rar/vendor/rars/tests/ppmd_regressions.rs new file mode 100644 index 00000000..c79a0a9b --- /dev/null +++ b/arca-rar/vendor/rars/tests/ppmd_regressions.rs @@ -0,0 +1,93 @@ +use rars::crc32::crc32; +use rars::rar15_40::Archive; +use rars::{ArchiveReadOptions, Result}; +use std::cell::RefCell; +use std::io::{Result as IoResult, Write}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar15_40/ppmd") + .join(name) +} + +struct CollectWriter { + data: Rc>>, +} + +struct CollectedEntry { + data: Vec, +} + +impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.data.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +fn collect_extract(archive: &Archive) -> Result> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(ArchiveReadOptions::default(), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(_meta, data)| CollectedEntry { + data: data.borrow().clone(), + }) + .collect()) +} + +#[test] +fn ppmd_block_can_emit_embedded_lz_distance_matches() { + let expected = std::fs::read(fixture("ppmd_lz_match.txt")).unwrap(); + let archive = Archive::parse_path(fixture("ppmd_lz_match_rar300.rar")).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"repeated_phrase_64k.txt"); + assert_eq!(files[0].method, 0x35); + assert_eq!(files[0].unp_ver, 29); + assert_eq!(files[0].pack_size, 193); + assert_eq!(files[0].unp_size, 44_544); + assert_eq!(files[0].file_crc, 0x884fab33); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, expected); + assert_eq!(crc32(&extracted[0].data), 0x884fab33); +} + +#[test] +fn ppmd_block_can_emit_embedded_one_byte_lz_repeats() { + let archive = Archive::parse_path(fixture("ppmd_lz_repeat_rar3.cbr")).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"testfile.jpg"); + assert_eq!(files[0].method, 0x35); + assert_eq!(files[0].unp_ver, 29); + assert_eq!(files[0].pack_size, 182); + assert_eq!(files[0].unp_size, 220); + assert_eq!(files[0].file_crc, 0xda70b16c); + assert_eq!(files[1].name, b"testfile.png"); + assert_eq!(files[1].method, 0x35); + assert_eq!(files[1].unp_ver, 29); + assert_eq!(files[1].pack_size, 84); + assert_eq!(files[1].unp_size, 87); + assert_eq!(files[1].file_crc, 0xafb0ac62); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(crc32(&extracted[0].data), 0xda70b16c); + assert_eq!(crc32(&extracted[1].data), 0xafb0ac62); +} diff --git a/arca-rar/vendor/rars/tests/rar13_fixtures.rs b/arca-rar/vendor/rars/tests/rar13_fixtures.rs new file mode 100644 index 00000000..4d97c38a --- /dev/null +++ b/arca-rar/vendor/rars/tests/rar13_fixtures.rs @@ -0,0 +1,875 @@ +#![cfg(feature = "write")] + +use rars::rar13::{extract_volumes_to, file_checksum, Archive, Entry}; +use rars::{detect_archive_family, find_archive_start, ArchiveFamily, Error}; +use std::cell::RefCell; +use std::io::{Result as IoResult, Write}; +use std::rc::Rc; + +const EMPTY: &[u8] = include_bytes!("fixtures/rar13/EMPTY.RAR"); +const BIG80K: &[u8] = include_bytes!("fixtures/rar13/BIG80K.RAR"); +const MULTIFIL: &[u8] = include_bytes!("fixtures/rar13/MULTIFIL.RAR"); +const REPEATB: &[u8] = include_bytes!("fixtures/rar13/REPEATB.RAR"); +const WITHDIR: &[u8] = include_bytes!("fixtures/rar13/WITHDIR.RAR"); +const COMMENT: &[u8] = include_bytes!("fixtures/rar13/COMMENT.RAR"); +const FCOMM: &[u8] = include_bytes!("fixtures/rar13/FCOMM.RAR"); +const README_PASSWORD: &[u8] = include_bytes!("fixtures/rar13/README_password=password.rar"); +const README_COMPRESSED: &[u8] = include_bytes!("fixtures/rar13/README.RAR"); +const README_STORE: &[u8] = include_bytes!("fixtures/rar13/README_store.rar"); +const README_EXPECTED: &[u8] = include_bytes!("fixtures/rar13/README"); +const CMULTI_EXPECTED: &[u8] = include_bytes!("fixtures/rar13/CMULTI.TXT"); +const STOREPWD: &[u8] = include_bytes!("fixtures/rar13/STOREPWD.RAR"); +const SFXSRC: &[u8] = include_bytes!("fixtures/rar13/SFXSRC.EXE"); +const SOLID: &[u8] = include_bytes!("fixtures/rar13/SOLID.RAR"); +const SOLID_FLAG_CLEARED: &[u8] = include_bytes!("fixtures/rar13/solid_flag_cleared.rar"); +const MULTIVOL_RAR: &[u8] = include_bytes!("fixtures/rar13/MULTIVOL.RAR"); +const MULTIVOL_R00: &[u8] = include_bytes!("fixtures/rar13/MULTIVOL.R00"); +const MULTIVOL_R01: &[u8] = include_bytes!("fixtures/rar13/MULTIVOL.R01"); +const MULTIVOL_R02: &[u8] = include_bytes!("fixtures/rar13/MULTIVOL.R02"); +const CMULTIV_RAR: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.RAR"); +const CMULTIV_R00: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R00"); +const CMULTIV_R01: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R01"); +const CMULTIV_R02: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R02"); +const CMULTIV_R03: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R03"); +const CMULTIV_R04: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R04"); +const CMULTIV_R05: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R05"); +const CMULTIV_R06: &[u8] = include_bytes!("fixtures/rar13/CMULTIV.R06"); +const RAR140_NOAV: &[u8] = include_bytes!("fixtures/rar13/rar140_av/rar140_noav_baseline.rar"); +const RAR140_AV: &[u8] = include_bytes!("fixtures/rar13/rar140_av/rar140_av_patched.rar"); +const ENCRYPTED_SPLIT: [&[u8]; 3] = [ + include_bytes!("fixtures/rar13/encrypted_split/ESPLIT.RAR"), + include_bytes!("fixtures/rar13/encrypted_split/ESPLIT.R00"), + include_bytes!("fixtures/rar13/encrypted_split/ESPLIT.R01"), +]; + +struct CollectWriter { + data: Rc>>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CollectedEntry { + name: Vec, + data: Vec, + file_time: u32, + file_attr: u8, + is_directory: bool, +} + +impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.data.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +/// RAR 1.4 stores `LHD_SOLID` at +17 bit 4 and never reads it back: solid +/// continuation follows the archive-level `MHD_SOLID` flag and position. +/// +/// The fixture has that bit cleared on the second member, which RAR 7.12 and +/// UnRAR 7.20 both extract regardless. The member is 44 packed bytes standing +/// for 2700 unpacked, so a byte-exact result is only reachable with the window +/// carried over, which is what makes this a test of the flag being ignored +/// rather than of the archive merely parsing. +#[test] +fn ignores_a_cleared_solid_flag() { + let archive = Archive::parse(SOLID_FLAG_CLEARED).unwrap(); + let extracted = collect_extract(&archive, None).unwrap(); + + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data.len(), 2700); + assert_eq!(extracted[1].data, extracted[0].data); +} + +#[test] +fn decrypts_a_rar1402_compressed_member_across_volume_boundaries() { + let volumes: Vec<_> = ENCRYPTED_SPLIT + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(volumes + .iter() + .all(|volume| volume.entries[0].is_encrypted())); + assert!(volumes.iter().all(|volume| !volume.entries[0].is_stored())); + + let extracted = collect_extract_volumes(&volumes, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"SOURCE.TXT"); + assert_eq!(extracted[0].data, CMULTI_EXPECTED[..32768]); + assert_eq!( + collect_extract_volumes(&volumes, None).unwrap_err(), + Error::NeedPassword + ); + assert_eq!( + collect_extract_volumes(&volumes, Some(b"wrong")).unwrap_err(), + Error::WrongPasswordOrCorruptData + ); +} + +fn collect_extract( + archive: &Archive, + password: Option<&[u8]>, +) -> Result, Error> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(password, |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + file_attr: meta.file_attr, + is_directory: meta.is_directory, + }) + .collect()) +} + +fn collect_extract_volumes( + archives: &[Archive], + password: Option<&[u8]>, +) -> Result, Error> { + let entries = RefCell::new(Vec::new()); + extract_volumes_to(archives, password, |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + file_attr: meta.file_attr, + is_directory: meta.is_directory, + }) + .collect()) +} + +fn collect_entry( + archive: &Archive, + entry: &Entry, + password: Option<&[u8]>, +) -> Result, Error> { + let data = Rc::new(RefCell::new(Vec::new())); + entry.write_to( + archive, + password, + &mut CollectWriter { + data: Rc::clone(&data), + }, + )?; + let data = data.borrow().clone(); + Ok(data) +} + +#[test] +fn detects_real_rar1402_archive() { + let sig = detect_archive_family(README_STORE).expect("signature"); + assert_eq!(sig.family, ArchiveFamily::Rar13); + assert_eq!(sig.offset, 0); + assert_eq!(sig.length, 4); +} + +#[test] +fn extract_to_reports_rar13_entry_context_on_write_failure() { + struct FailWriter; + + impl Write for FailWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("sink failed")) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let archive = Archive::parse(README_STORE).unwrap(); + let error = archive + .extract_to(None, |_meta| Ok(Box::new(FailWriter))) + .unwrap_err(); + + match error { + Error::AtEntry { + name, + operation, + source, + } => { + assert_eq!(name, b"README"); + assert_eq!(operation, "extracting"); + assert!(matches!(*source, Error::Io(_))); + } + other => panic!("expected entry context, got {other:?}"), + } +} + +#[test] +fn parses_rar140_inline_av_shape_fixture() { + let archive = Archive::parse(RAR140_AV).expect("parse RAR 1.40 AV fixture"); + assert_eq!(archive.main.flags, 0xa0); + assert_eq!(archive.main.head_size, 53); + assert!(archive.main.has_authenticity_verification()); + + let av = archive + .authenticity_verification() + .expect("parse AV") + .expect("AV present"); + assert_eq!(av.size, 44); + assert_eq!(av.prefix, *b"\x1ai\x6d\x02\xda\xae"); + assert_eq!(av.cipher_body.len(), 38); + assert_eq!( + archive.authenticity_verification_status().unwrap(), + rars::rar13::AuthenticityVerificationStatus::StructurallyPresent + ); + + let extracted = collect_extract(&archive, None).expect("extract AV-bearing archive"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"hello\n"); +} + +#[test] +fn reports_absent_av_on_rar140_control_fixture() { + let archive = Archive::parse(RAR140_NOAV).expect("parse RAR 1.40 baseline fixture"); + assert!(!archive.main.has_authenticity_verification()); + assert!(archive.authenticity_verification().unwrap().is_none()); + assert_eq!( + archive.authenticity_verification_status().unwrap(), + rars::rar13::AuthenticityVerificationStatus::Absent + ); +} + +#[test] +fn decodes_real_rar1402_stored_file() { + let archive = Archive::parse(README_STORE).expect("parse RAR 1.402 archive"); + assert_eq!(archive.main.head_size, 7); + assert_eq!(archive.main.flags, 0x80); + assert_eq!(archive.entries.len(), 1); + + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"README"); + assert_eq!(entry.header.pack_size, README_EXPECTED.len() as u32); + assert_eq!(entry.header.unp_size, README_EXPECTED.len() as u32); + assert_eq!(entry.header.file_crc, 0xe079); + assert_eq!(entry.header.head_size, 27); + assert_eq!(entry.header.file_attr, 0x20); + assert_eq!(entry.header.flags, 0); + assert_eq!(entry.header.unp_ver, 2); + assert_eq!(entry.header.method, 0); + let decoded = collect_entry(&archive, entry, None).expect("stored data"); + assert_eq!(decoded, README_EXPECTED); + + let extracted = collect_extract(&archive, None).expect("extract stored archive"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"README"); + assert_eq!(extracted[0].data, README_EXPECTED); + assert!(!extracted[0].is_directory); +} + +#[test] +fn real_rar1402_stored_checksum_matches_rolling_sum_rotate() { + let archive = Archive::parse(README_STORE).expect("parse RAR 1.402 archive"); + let entry = &archive.entries[0]; + let decoded = collect_entry(&archive, entry, None).expect("stored data"); + + assert_eq!(entry.header.file_crc, 0xe079); + assert_eq!(file_checksum(&decoded), 0xe079); + entry + .verify_checksum(&decoded) + .expect("RAR 1.3 rolling checksum"); +} + +#[test] +fn decodes_real_rar1402_compressed_file() { + let archive = Archive::parse(README_COMPRESSED).expect("parse compressed RAR 1.402 archive"); + assert_eq!(archive.entries.len(), 1); + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"README"); + assert_eq!(entry.header.pack_size, 1078); + assert_eq!(entry.header.unp_size, README_EXPECTED.len() as u32); + assert_eq!(entry.header.file_crc, 0xe079); + assert_eq!(entry.header.method, 3); + assert!(!entry.is_stored()); + + let extracted = collect_extract(&archive, None).expect("extract compressed archive"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"README"); + assert_eq!(extracted[0].data, README_EXPECTED); + assert_eq!(file_checksum(&extracted[0].data), 0xe079); +} + +#[test] +fn decodes_real_rar1402_compressed_window_wrap_file() { + let archive = Archive::parse(BIG80K).expect("parse BIG80K RAR 1.402 archive"); + let extracted = collect_extract(&archive, None).expect("extract BIG80K archive"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"BIG80K.TXT"); + assert_eq!(extracted[0].data.len(), 80 * 1024); + assert_eq!( + file_checksum(&extracted[0].data), + archive.entries[0].header.file_crc + ); +} + +#[test] +fn decodes_real_rar1402_repeating_pattern_file() { + let archive = Archive::parse(REPEATB).expect("parse REPEATB RAR 1.402 archive"); + let extracted = collect_extract(&archive, None).expect("extract REPEATB archive"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"REPEATB.BIN"); + assert_eq!(extracted[0].data, expected_repeatb()); +} + +#[test] +fn decodes_real_rar1402_solid_archive() { + let archive = Archive::parse(SOLID).expect("parse SOLID RAR 1.402 archive"); + assert_eq!(archive.main.flags, 0x88); + assert!(archive.main.is_solid()); + assert_eq!(archive.entries.len(), 3); + + let extracted = collect_extract(&archive, None).expect("extract solid archive"); + assert_eq!(extracted.len(), 3); + assert_eq!(extracted[0].name, b"BIG80K.TXT"); + assert_eq!(extracted[0].data.len(), 80 * 1024); + assert_eq!( + file_checksum(&extracted[0].data), + archive.entries[0].header.file_crc + ); + assert_eq!(extracted[1].name, b"HELLO.TXT"); + assert_eq!(extracted[1].data, b"Hello, RAR 1.402 fixture world.\r\n"); + assert_eq!(extracted[2].name, b"TINY.TXT"); + assert_eq!(extracted[2].data, b"AAAAAAAA\r\n"); +} + +#[test] +fn parses_empty_stored_file() { + let archive = Archive::parse(EMPTY).expect("parse empty RAR 1.402 archive"); + assert_eq!(archive.main.flags, 0x80); + assert_eq!(archive.entries.len(), 1); + + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"EMPTY.BIN"); + assert_eq!(entry.header.pack_size, 0); + assert_eq!(entry.header.unp_size, 0); + assert_eq!(entry.header.file_crc, 0); + assert!(entry.is_stored()); + assert_eq!( + collect_entry(&archive, entry, None).expect("empty data"), + b"" + ); + entry.verify_checksum(b"").expect("empty checksum"); + + let extracted = collect_extract(&archive, None).expect("extract empty archive"); + assert_eq!(extracted[0].name, b"EMPTY.BIN"); + assert!(extracted[0].data.is_empty()); +} + +#[test] +fn parses_multiple_file_headers() { + let archive = Archive::parse(MULTIFIL).expect("parse multi-file RAR 1.402 archive"); + assert_eq!(archive.entries.len(), 2); + + let first = &archive.entries[0]; + assert_eq!(first.name, b"HELLO.TXT"); + assert_eq!(first.header.pack_size, 33); + assert_eq!(first.header.unp_size, 33); + assert_eq!(first.header.file_crc, 0x7a6e); + assert!(first.is_stored()); + assert_eq!( + collect_entry(&archive, first, None).expect("stored HELLO.TXT"), + b"Hello, RAR 1.402 fixture world.\r\n" + ); + first + .verify_checksum(b"Hello, RAR 1.402 fixture world.\r\n") + .expect("HELLO.TXT checksum"); + + let second = &archive.entries[1]; + assert_eq!(second.name, b"TINY.TXT"); + assert_eq!(second.header.pack_size, 7); + assert_eq!(second.header.unp_size, 10); + assert_eq!(second.header.file_crc, 0x0642); + assert_eq!(second.header.method, 3); + assert!(!second.is_stored()); + let extracted = collect_extract(&archive, None).expect("extract mixed archive"); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"HELLO.TXT"); + assert_eq!(extracted[0].data, b"Hello, RAR 1.402 fixture world.\r\n"); + assert_eq!(extracted[1].name, b"TINY.TXT"); + assert_eq!(extracted[1].data, b"AAAAAAAA\r\n"); +} + +#[test] +fn parses_directory_entry_and_following_file() { + let archive = Archive::parse(WITHDIR).expect("parse directory RAR 1.402 archive"); + assert_eq!(archive.entries.len(), 2); + + let dir = &archive.entries[0]; + assert_eq!(dir.name, b"SUBDIR"); + assert_eq!(dir.header.file_attr, 0x10); + assert!(dir.is_directory()); + assert_eq!(dir.header.pack_size, 0); + assert_eq!(dir.header.unp_size, 0); + + let file = &archive.entries[1]; + assert_eq!(file.name, b"SUBDIR\\INNER.TXT"); + assert!(!file.is_directory()); + assert!(file.is_stored()); + assert_eq!( + collect_entry(&archive, file, None).expect("stored inner file"), + b"Inside subdir.\r\n" + ); + file.verify_checksum(b"Inside subdir.\r\n") + .expect("inner file checksum"); + + let extracted = collect_extract(&archive, None).expect("extract directory archive"); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"SUBDIR"); + assert!(extracted[0].is_directory); + assert!(extracted[0].data.is_empty()); + assert_eq!(extracted[1].name, b"SUBDIR\\INNER.TXT"); + assert_eq!(extracted[1].data, b"Inside subdir.\r\n"); +} + +#[test] +fn parses_encrypted_compressed_file_metadata() { + let archive = Archive::parse(README_PASSWORD).expect("parse encrypted RAR 1.402 archive"); + assert_eq!(archive.entries.len(), 1); + let entry = &archive.entries[0]; + + assert_eq!(entry.name, b"README"); + assert_eq!(entry.header.pack_size, 1078); + assert_eq!(entry.header.unp_size, README_EXPECTED.len() as u32); + assert_eq!(entry.header.file_crc, 0xe079); + assert_eq!(entry.header.flags, 0x04); + assert_eq!(entry.header.method, 3); + assert!(entry.is_encrypted()); + assert!(!entry.is_stored()); +} + +#[test] +fn decodes_real_rar1402_encrypted_compressed_file() { + let archive = + Archive::parse(README_PASSWORD).expect("parse encrypted compressed RAR 1.402 archive"); + assert!(collect_extract(&archive, None).is_err()); + + let extracted = + collect_extract(&archive, Some(b"password")).expect("extract encrypted compressed archive"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"README"); + assert_eq!(extracted[0].data, README_EXPECTED); + assert_eq!(file_checksum(&extracted[0].data), 0xe079); +} + +#[test] +fn extract_to_decodes_real_rar1402_encrypted_compressed_file() { + #[derive(Clone)] + struct SharedWriter(std::rc::Rc>>); + + impl std::io::Write for SharedWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let archive = + Archive::parse(README_PASSWORD).expect("parse encrypted compressed RAR 1.402 archive"); + let extracted = std::rc::Rc::new(std::cell::RefCell::new(Vec::new())); + archive + .extract_to(Some(b"password"), |meta| { + assert_eq!(meta.name, b"README"); + Ok(Box::new(SharedWriter(extracted.clone()))) + }) + .expect("stream encrypted compressed archive"); + + let extracted = extracted.borrow(); + assert_eq!(&*extracted, README_EXPECTED); + assert_eq!(file_checksum(&extracted), 0xe079); +} + +#[test] +fn rejects_wrong_password_for_encrypted_compressed_file() { + let archive = + Archive::parse(README_PASSWORD).expect("parse encrypted compressed RAR 1.402 archive"); + assert!(collect_extract(&archive, Some(b"wrong-password")).is_err()); +} + +#[test] +fn rejects_corrupt_stored_payload_checksum() { + let mut corrupt = README_STORE.to_vec(); + let last = corrupt.last_mut().expect("non-empty fixture"); + *last ^= 0x01; + + let archive = Archive::parse(&corrupt).expect("parse corrupt stored archive"); + match collect_extract(&archive, None) { + Err(Error::CrcMismatch { .. }) => {} + Err(Error::AtEntry { source, .. }) if matches!(*source, Error::CrcMismatch { .. }) => {} + other => panic!("expected checksum error, got {other:?}"), + } +} + +#[test] +fn rejects_truncated_compressed_payload() { + let truncated = &README_COMPRESSED[..README_COMPRESSED.len() - 1]; + let err = Archive::parse(truncated).expect_err("truncated archive must not parse"); + assert_eq!(err, Error::TooShort); +} + +#[test] +fn decodes_real_rar1402_encrypted_stored_file() { + let archive = Archive::parse(STOREPWD).expect("parse encrypted stored RAR 1.402 archive"); + assert_eq!(archive.entries.len(), 1); + let entry = &archive.entries[0]; + + assert_eq!(entry.name, b"SECRET.TXT"); + assert_eq!(entry.header.pack_size, 27); + assert_eq!(entry.header.unp_size, 27); + assert_eq!(entry.header.file_crc, 0x4423); + assert_eq!(entry.header.flags, 0x04); + assert_eq!(entry.header.method, 0); + assert!(entry.is_encrypted()); + assert!(entry.is_stored()); + assert!(matches!( + collect_entry(&archive, entry, None), + Err(Error::NeedPassword) + )); + + let decoded = collect_entry(&archive, entry, Some(b"password")).expect("decrypt stored data"); + assert_eq!(decoded, b"Stored encrypted fixture.\r\n"); + entry + .verify_checksum(&decoded) + .expect("encrypted stored checksum"); + + let extracted = + collect_extract(&archive, Some(b"password")).expect("extract encrypted stored archive"); + assert_eq!(extracted[0].name, b"SECRET.TXT"); + assert_eq!(extracted[0].data, b"Stored encrypted fixture.\r\n"); +} + +#[test] +fn detects_and_parses_rar14_sfx_archive() { + let sig = find_archive_start(SFXSRC, 128 * 1024).expect("SFX embedded signature"); + assert_eq!(sig.family, ArchiveFamily::Rar13); + assert_eq!(sig.offset, 6491); + + let archive = Archive::parse(SFXSRC).expect("parse RAR 1.402 SFX archive"); + assert_eq!(archive.sfx_offset, 6491); + assert_eq!(archive.entries.len(), 1); + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"HELLO.TXT"); + assert!(entry.is_stored()); + assert_eq!( + collect_entry(&archive, entry, None).expect("stored SFX payload"), + b"Hello, RAR 1.402 fixture world.\r\n" + ); +} + +#[test] +fn parses_old_style_multivolume_parts() { + let cases = [ + (MULTIVOL_RAR, false, true, 19_962, 0x5ec8), + (MULTIVOL_R00, true, true, 19_962, 0x5147), + (MULTIVOL_R01, true, true, 19_962, 0xda0b), + (MULTIVOL_R02, true, false, 5_650, 0x4649), + ]; + + for (bytes, split_before, split_after, pack_size, file_crc) in cases { + let archive = Archive::parse(bytes).expect("parse multivolume RAR 1.402 part"); + assert_eq!(archive.main.flags, 0x81); + assert!(archive.main.is_volume()); + assert_eq!(archive.entries.len(), 1); + + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"RANDOM.BIN"); + assert_eq!(entry.header.pack_size, pack_size); + assert_eq!(entry.header.unp_size, 65_536); + assert_eq!(entry.header.file_crc, file_crc); + assert_eq!(entry.is_split_before(), split_before); + assert_eq!(entry.is_split_after(), split_after); + assert!(entry.is_stored()); + } +} + +#[test] +fn reassembles_old_style_stored_multivolume_file() { + let volumes = [ + Archive::parse(MULTIVOL_RAR).expect("parse first volume"), + Archive::parse(MULTIVOL_R00).expect("parse second volume"), + Archive::parse(MULTIVOL_R01).expect("parse third volume"), + Archive::parse(MULTIVOL_R02).expect("parse fourth volume"), + ]; + + let extracted = collect_extract_volumes(&volumes, None).expect("join stored volumes"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"RANDOM.BIN"); + assert_eq!(extracted[0].data.len(), 65_536); + assert_eq!(file_checksum(&extracted[0].data), 0x4649); +} + +#[test] +fn parses_old_style_compressed_multivolume_parts() { + let cases = [ + (CMULTIV_RAR, false, true, 1_962, 0x8523), + (CMULTIV_R00, true, true, 1_962, 0x8523), + (CMULTIV_R01, true, true, 1_962, 0x8523), + (CMULTIV_R02, true, true, 1_962, 0x8523), + (CMULTIV_R03, true, true, 1_962, 0x87cd), + (CMULTIV_R04, true, true, 1_962, 0x87cd), + (CMULTIV_R05, true, true, 1_962, 0x87cd), + (CMULTIV_R06, true, false, 533, 0x87cd), + ]; + + for (bytes, split_before, split_after, pack_size, file_crc) in cases { + let archive = Archive::parse(bytes).expect("parse compressed multivolume RAR 1.402 part"); + assert_eq!(archive.main.flags, 0x81); + assert!(archive.main.is_volume()); + assert_eq!(archive.entries.len(), 1); + + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"CMULTI.TXT"); + assert_eq!(entry.header.pack_size, pack_size); + assert_eq!(entry.header.unp_size, 98_304); + assert_eq!(entry.header.file_crc, file_crc); + assert_eq!(entry.is_split_before(), split_before); + assert_eq!(entry.is_split_after(), split_after); + assert!(!entry.is_stored()); + } +} + +#[test] +fn reassembles_old_style_compressed_multivolume_file() { + let volumes = [ + Archive::parse(CMULTIV_RAR).expect("parse first compressed volume"), + Archive::parse(CMULTIV_R00).expect("parse second compressed volume"), + Archive::parse(CMULTIV_R01).expect("parse third compressed volume"), + Archive::parse(CMULTIV_R02).expect("parse fourth compressed volume"), + Archive::parse(CMULTIV_R03).expect("parse fifth compressed volume"), + Archive::parse(CMULTIV_R04).expect("parse sixth compressed volume"), + Archive::parse(CMULTIV_R05).expect("parse seventh compressed volume"), + Archive::parse(CMULTIV_R06).expect("parse eighth compressed volume"), + ]; + + let extracted = collect_extract_volumes(&volumes, None).expect("join compressed volumes"); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"CMULTI.TXT"); + assert_eq!(extracted[0].data, CMULTI_EXPECTED); + assert_eq!(file_checksum(&extracted[0].data), 0x87cd); +} + +#[test] +fn parses_archive_comment_main_header_extension() { + let archive = Archive::parse(COMMENT).expect("parse comment RAR 1.402 archive"); + assert_eq!(archive.main.flags, 0x92); + assert_eq!(archive.main.head_size, 43); + assert_eq!(archive.main.extra.len(), 36); + assert!(archive.main.has_archive_comment()); + assert!(archive.main.has_packed_comment()); + assert_eq!(archive.entries.len(), 1); + + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"HELLO.TXT"); + assert!(entry.is_stored()); + assert_eq!( + collect_entry(&archive, entry, None).expect("stored commented payload"), + b"Hello, comment fixture.\r\n" + ); + entry + .verify_checksum(b"Hello, comment fixture.\r\n") + .expect("comment fixture payload checksum"); +} + +#[test] +fn decodes_packed_archive_comment() { + let archive = Archive::parse(COMMENT).expect("parse comment RAR 1.402 archive"); + let comment = archive + .archive_comment() + .expect("decode RAR 1.402 archive comment") + .expect("archive comment"); + assert_eq!(comment, b"This is the archive comment.\r\n"); +} + +#[test] +fn parses_and_decodes_file_comment_header_extension() { + let archive = Archive::parse(FCOMM).expect("parse file-comment RAR 1.402 archive"); + assert_eq!(archive.entries.len(), 1); + + let entry = &archive.entries[0]; + assert_eq!(entry.name, b"HELLO.TXT"); + assert_eq!(entry.header.flags, 0x08); + assert_eq!(entry.header.head_size, 38); + assert_eq!(entry.extra, b"\x06\x00FCOM\r\n"); + assert_eq!( + entry + .file_comment() + .expect("decode file comment") + .expect("file comment"), + b"FCOM\r\n" + ); + assert_eq!( + collect_entry(&archive, entry, None).expect("stored file-comment payload"), + b"Hello, file comment fixture.\r\n" + ); +} + +fn expected_repeatb() -> Vec { + let mut out = Vec::with_capacity(256 * 32); + for _ in 0..32 { + out.extend(0u8..=255); + } + out +} + +/// The streaming writer and the in-memory one are the same writer with a +/// different way of getting at the bytes, so the archives they produce have to +/// be the same archives. +#[test] +fn streaming_and_buffered_writers_agree_byte_for_byte() { + use rars::rar13::{ + write_compressed_archive_with_comment, write_stored_archive_with_comment, + write_streaming_archive_to, FileEntry, StoredEntry, StreamingEntry, WriterOptions, + }; + use rars::{ArchiveVersion, EntrySource, FeatureSet, MemberCoding, WriterResources}; + + let text = b"the quick brown fox jumps over the lazy dog\n".repeat(400); + let counted: Vec = (0..40_000u32).map(|index| index as u8).collect(); + let members: Vec<(&[u8], &[u8])> = vec![ + (b"TEXT.TXT", &text), + (b"COUNT.BIN", &counted), + (b"EMPTY.DAT", b""), + ]; + + for solid in [false, true] { + for coding in [MemberCoding::Stored, MemberCoding::Compressed] { + let mut features = FeatureSet::store_only(); + features.solid = solid && coding.compresses(); + let options = WriterOptions::new(ArchiveVersion::Rar14, features); + let comment: Option<&[u8]> = Some(b"shared comment"); + + let buffered = match &coding { + MemberCoding::Stored => { + let entries: Vec<_> = members + .iter() + .map(|(name, data)| StoredEntry { + name, + data, + file_time: 0, + file_attr: 0, + password: None, + file_comment: None, + }) + .collect(); + write_stored_archive_with_comment(&entries, options, comment) + } + _ => { + let entries: Vec<_> = members + .iter() + .map(|(name, data)| FileEntry { + name, + data, + file_time: 0, + file_attr: 0, + password: None, + file_comment: None, + }) + .collect(); + write_compressed_archive_with_comment(&entries, options, comment) + } + }; + + let streamed_entries: Vec<_> = members + .iter() + .map(|(name, data)| { + StreamingEntry::new(name.to_vec(), EntrySource::from_bytes(data.to_vec())) + }) + .collect(); + let mut streamed = Vec::new(); + let streamed_result = write_streaming_archive_to( + &streamed_entries, + options, + coding.clone(), + comment, + &WriterResources::default(), + None, + &mut streamed, + ); + + let label = format!("solid={solid} {coding:?}"); + match buffered { + Ok(buffered) => { + assert!( + streamed_result.is_ok(), + "{label}: streaming refused what the buffered writer accepted: \ + {streamed_result:?}" + ); + assert_eq!(streamed, buffered, "{label}: archives differ"); + let archive = Archive::parse(&streamed).unwrap(); + let extracted = collect_extract(&archive, None) + .unwrap_or_else(|error| panic!("{label}: {error:?}")); + assert_eq!(extracted.len(), members.len(), "{label}"); + for (got, (_, want)) in extracted.iter().zip(&members) { + assert_eq!(&got.data, want, "{label}"); + } + } + Err(buffered) => { + let streamed_error = streamed_result.expect_err(&format!( + "{label}: streaming accepted what the buffered writer refused" + )); + assert_eq!( + streamed_error.to_string(), + buffered.to_string(), + "{label}: the two paths refused it differently" + ); + } + } + } + } +} + +/// RAR 1.3 has no filters, so a filter policy has to be turned away rather +/// than routed into a compression engine it does not have. +#[test] +fn a_filter_policy_is_refused_by_name() { + use rars::rar13::{write_streaming_archive_to, StreamingEntry, WriterOptions}; + use rars::{ + ArchiveVersion, EntrySource, FeatureSet, FilterPolicy, MemberCoding, WriterOption, + WriterResources, + }; + + let entries = vec![StreamingEntry::new( + b"A.TXT".to_vec(), + EntrySource::from_bytes(b"payload".to_vec()), + )]; + let mut out = Vec::new(); + let error = write_streaming_archive_to( + &entries, + WriterOptions::new(ArchiveVersion::Rar14, FeatureSet::store_only()), + MemberCoding::Filtered(FilterPolicy::Auto), + None, + &WriterResources::default(), + None, + &mut out, + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::UnsupportedWriterOption { + option: WriterOption::Filter, + .. + } + )); + assert!(out.is_empty(), "nothing should have been written"); +} diff --git a/arca-rar/vendor/rars/tests/rar15_40_fixtures.rs b/arca-rar/vendor/rars/tests/rar15_40_fixtures.rs new file mode 100644 index 00000000..c259f8e9 --- /dev/null +++ b/arca-rar/vendor/rars/tests/rar15_40_fixtures.rs @@ -0,0 +1,8370 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::crc32::crc32; +use rars::rar15_40::{ + extract_volumes_to, repair_rev3_volumes_to, write_compressed_archive, + write_compressed_archive_with_comment, write_compressed_volumes, + write_rar29_compressed_archive_with_filter_policy, write_stored_archive, + write_stored_archive_with_comment, write_stored_volumes, Archive, Block, FileEntry, FilterKind, + FilterPolicy, FilterSpec, NewSubKind, ProtectHeader, Rar29Method, StoredEntry, WriterOptions, +}; +use rars::{ + detect_archive_family, ArchiveFamily, ArchiveReadOptions, ArchiveVersion, Error, FeatureSet, +}; +use std::cell::RefCell; +use std::io::{Result as IoResult, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::rc::Rc; + +/// What the writer declares for RAR 2.9, and so the distance limit PPMd has to +/// encode within. Passing anything else here compares the codec against an +/// archive the writer would not have written. +const RAR29_DICTIONARY: usize = 1024 * 1024; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar15_40") + .join(name) +} + +const RARS_GENERATED_PAYLOAD: &[u8] = b"rar15 oracle payload\n"; +const RARS_GENERATED_SECOND: &[u8] = b"rar15 oracle second file\n"; + +fn level_sensitive_payload() -> Vec { + let pattern: Vec = (0..=255).collect(); + let mut data = Vec::new(); + for round in 0..8u8 { + data.extend_from_slice(&pattern); + for index in 0..32u8 { + let mut decoy = pattern.clone(); + for byte in decoy.iter_mut().skip(7).step_by(11) { + *byte = byte.wrapping_add(index).wrapping_add(round).wrapping_add(1); + } + data.extend_from_slice(&decoy); + } + data.extend_from_slice(&pattern); + } + data +} + +const RARS_GENERATED_FIXTURE_BYTES: &[(&str, usize, u32)] = &[ + ("comments.rar", 133, 0x5694_01c1), + ("compressed.rar", 87, 0x13ca_0571), + ("encrypted.rar", 87, 0x04af_0eb7), + ("solid.rar", 153, 0x8deb_98d1), + ("split-encrypted.r00", 73, 0xf109_611e), + ("split-encrypted.r01", 67, 0x43b5_c214), + ("split-encrypted.rar", 73, 0x5989_8212), + ("split-store.r00", 73, 0x6926_da6f), + ("split-store.r01", 64, 0x1f25_c044), + ("split-store.rar", 73, 0x5440_b247), + ("stored.rar", 84, 0x470d_272b), +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CollectedEntry { + name: Vec, + data: Vec, + file_time: u32, + attr: u32, + host_os: u8, + is_directory: bool, +} + +struct CollectWriter { + data: Rc>>, +} + +impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.data.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +/// A DOS timestamp steps in two seconds and holds nothing smaller, so the +/// extended time field carries the odd second and up to three bytes of +/// 100-nanosecond ticks. Ignoring it costs up to a second on every RAR 1.5-4.x +/// archive that has one. +/// +/// The RAR 4.20 fixture exercises the whole path: present, odd second, and the +/// full three sub-second bytes. RAR 7.12 restores it as +/// 2026-04-27 06:54:43.704088300 UTC, which is the DOS 06:54:42 plus both +/// parts, and rars now agrees to the nanosecond. +#[test] +fn decodes_the_extended_modification_time() { + let bytes = std::fs::read(fixture("rar420/ext_time_rar420.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + let refinement = file.mtime_refinement().expect("fixture carries an mtime"); + assert!(refinement.add_second); + assert_eq!(refinement.nanoseconds, 704_088_300); + + // The DOS field itself is unchanged: it is the base the refinement adds to. + assert_eq!(file.file_time & 0x1f, 42 / 2); + assert_eq!(file.metadata().mtime_refinement, Some(refinement)); +} + +/// A header without the extended field has nothing to add, and one whose mtime +/// nibble is absent must not be read as a zero refinement either. +#[test] +fn reports_no_refinement_without_an_extended_time() { + let bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + for file in archive.files() { + if file.ext_time.is_empty() { + assert_eq!(file.mtime_refinement(), None); + } + } +} + +/// A block whose declared extent is zero leaves the next position equal to the +/// current one, and a walk that just seeks and repeats spins there forever. +/// +/// RAR 7.12 and UnRAR 7.20 both reject this archive and neither hangs. rars +/// gets there by refusing any header below the 7-byte minimum, which keeps the +/// block extent positive; a direct next > current comparison would do as well. +#[test] +fn rejects_a_block_header_that_does_not_advance() { + let data = std::fs::read(fixture("zero_head_size.rar")).unwrap(); + assert!(Archive::parse(&data) + .and_then(|archive| collect_extract(&archive)) + .is_err()); +} + +/// A compressed file header can legitimately describe an empty member without +/// carrying a codec bitstream. UnRAR 7.20 and libarchive 3.8.5 both extract +/// this fixture as an empty file; entering the RAR29 decoder would instead ask +/// it to manufacture a block header from no input. +#[test] +fn extracts_a_compressed_empty_member_without_a_bitstream() { + let data = std::fs::read(fixture("empty_compressed_payload_rar30.rar")).unwrap(); + let archive = Archive::parse(&data).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"empty.bin"); + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + assert_eq!(file.pack_size, 0); + assert_eq!(file.unp_size, 0); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"empty.bin"); + assert!(extracted[0].data.is_empty()); +} + +/// Below `UnpVer` 20 the per-file solid flag is written but never read: solid +/// continuation follows the archive-level flag and position instead. +/// +/// The fixture is a solid RAR 1.5 pair with `FHD_SOLID` cleared on the second +/// member and the header CRC recomputed. Extraction can only succeed by +/// carrying the window across, since that member is 46 packed bytes standing +/// for 2700 unpacked, so a byte-exact result proves the flag was ignored. +#[test] +fn rar15_ignores_a_cleared_solid_flag_below_unpack20() { + let data = std::fs::read(fixture("solid_flag_cleared_rar15.rar")).unwrap(); + let archive = Archive::parse(&data).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data.len(), 2700); + assert_eq!(extracted[1].data.len(), 2700); + assert_eq!(extracted[0].data, extracted[1].data); +} + +fn collect_extract(archive: &Archive) -> Result, Error> { + collect_extract_with_password(archive, None) +} + +fn collect_extract_with_password( + archive: &Archive, + password: Option<&[u8]>, +) -> Result, Error> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(read_options(password), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) + .collect()) +} + +fn collect_file( + archive: &Archive, + file: &rars::rar15_40::FileHeader, +) -> Result { + collect_file_with_password(archive, file, None) +} + +fn collect_file_with_password( + archive: &Archive, + file: &rars::rar15_40::FileHeader, + password: Option<&[u8]>, +) -> Result { + let meta = file.metadata(); + let data = Rc::new(RefCell::new(Vec::new())); + file.write_to( + archive, + password, + &mut CollectWriter { + data: Rc::clone(&data), + }, + )?; + let data = data.borrow().clone(); + Ok(CollectedEntry { + name: meta.name, + data, + file_time: meta.file_time, + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) +} + +fn collect_extract_volumes(archives: &[Archive]) -> Result, Error> { + collect_extract_volumes_with_password(archives, None) +} + +fn collect_extract_volumes_with_password( + archives: &[Archive], + password: Option<&[u8]>, +) -> Result, Error> { + let entries = RefCell::new(Vec::new()); + extract_volumes_to(archives, read_options(password), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time, + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) + .collect()) +} + +fn read_options(password: Option<&[u8]>) -> ArchiveReadOptions<'_> { + match password { + Some(password) => ArchiveReadOptions::with_password(password), + None => ArchiveReadOptions::default(), + } +} + +#[derive(Debug, Clone)] +struct ReferenceUnrar { + wineprefix: String, + unrar: String, +} + +fn reference_unrar(prefix_env: &str, unrar_env: &str) -> Option { + let wineprefix = match std::env::var(prefix_env) { + Ok(value) => value, + Err(_) => { + eprintln!("skipping reference test: set {prefix_env} to the WinRAR Wine prefix"); + return None; + } + }; + let unrar = std::env::var(unrar_env) + .unwrap_or_else(|_| format!("{wineprefix}/drive_c/Program Files (x86)/WinRAR/UnRAR.exe")); + if !Path::new(&unrar).is_file() { + eprintln!("skipping reference test: missing reference tool {unrar}"); + return None; + } + Some(ReferenceUnrar { wineprefix, unrar }) +} + +fn reference_unrar300() -> Option { + reference_unrar("RARS_WINRAR300_PREFIX", "RARS_UNRAR300") +} + +fn reference_unrar420() -> Option { + reference_unrar("RARS_WINRAR420_PREFIX", "RARS_UNRAR420") +} + +fn run_reference_unrar(reference: &ReferenceUnrar, archive_path: &Path) -> std::process::Output { + let wine_archive = format!("Z:{}", archive_path.to_string_lossy().replace('/', "\\")); + Command::new("env") + .arg(format!("WINEPREFIX={}", reference.wineprefix)) + .arg("wine") + .arg(&reference.unrar) + .arg("t") + .arg("-inul") + .arg(wine_archive) + .output() + .unwrap() +} + +fn repair_rev3_volumes( + data_volumes: &[Option<&[u8]>], + recovery_count: usize, + recovery_volumes: &[(usize, &[u8])], +) -> Result>, Error> { + let mut repaired = Vec::new(); + repair_rev3_volumes_to( + data_volumes, + recovery_count, + recovery_volumes, + |_, bytes| { + repaired.push(bytes.to_vec()); + Ok(()) + }, + )?; + Ok(repaired) +} + +fn write_rar29_auto(entries: &[FileEntry<'_>], options: WriterOptions) -> Vec { + write_rar29_compressed_archive_with_filter_policy(entries, options, FilterPolicy::Auto).unwrap() +} + +fn write_rar29_filter( + entries: &[FileEntry<'_>], + options: WriterOptions, + kind: FilterKind, +) -> Vec { + write_rar29_compressed_archive_with_filter_policy( + entries, + options, + FilterPolicy::Explicit(FilterSpec::whole(kind)), + ) + .unwrap() +} + +fn write_rar29_filter_range( + entries: &[FileEntry<'_>], + options: WriterOptions, + kind: FilterKind, + range: std::ops::Range, +) -> Vec { + write_rar29_compressed_archive_with_filter_policy( + entries, + options, + FilterPolicy::Explicit(FilterSpec::range(kind, range)), + ) + .unwrap() +} + +#[test] +fn detects_rar15_40_signature_family() { + let bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let sig = detect_archive_family(&bytes).unwrap(); + + assert_eq!(sig.family, ArchiveFamily::Rar15To40); + assert_eq!(sig.offset, 0); + assert_eq!(sig.length, 7); +} + +#[test] +fn generated_rar29_e8_filtered_archive_round_trips() { + let payload = b"\xe8\0\0\0\0rar29 e8 filter payload\n".repeat(16); + let entries = [FileEntry { + name: b"rar29-e8-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::E8, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + assert!(file.pack_size < file.unp_size); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-e8-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn forced_rar29_lz_round_trips_with_each_filter_policy() { + let payload = b"\xe8\0\0\0\0forced LZ and filter policy\n".repeat(32); + let entries = [FileEntry { + name: b"forced-lz.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Lz); + for policy in [ + FilterPolicy::None, + FilterPolicy::Auto, + FilterPolicy::explicit(FilterKind::E8), + ] { + let bytes = + write_rar29_compressed_archive_with_filter_policy(&entries, options, policy).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } +} + +#[test] +fn forced_rar29_lz_stores_unprofitable_and_empty_members() { + let data = b"short"; + for payload in [data.as_slice(), b"".as_slice()] { + let entries = [FileEntry { + name: b"small.bin", + data: payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Lz); + for policy in [FilterPolicy::None, FilterPolicy::Auto] { + let bytes = + write_rar29_compressed_archive_with_filter_policy(&entries, options, policy) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.method, 0x30); + assert_eq!(file.pack_size, payload.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } + } +} + +#[test] +fn rar29_level_zero_stores_even_with_a_filter_policy() { + let payload = b"\xe8\0\0\0\0level zero does not filter".repeat(16); + let entries = [FileEntry { + name: b"stored.bin", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_compression_level(0); + for policy in [ + FilterPolicy::None, + FilterPolicy::Auto, + FilterPolicy::explicit(FilterKind::E8), + ] { + let bytes = + write_rar29_compressed_archive_with_filter_policy(&entries, options, policy).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.method, 0x30); + assert_eq!(file.pack_size, payload.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } +} + +#[test] +fn solid_rar29_writer_respects_forced_lz_and_ppmd_engines() { + let mut features = FeatureSet::store_only(); + features.solid = true; + let payload = b"solid engine choice repeats this phrase across members\n".repeat(256); + let entries = [ + FileEntry { + name: b"first.txt", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"second.txt", + data: &payload, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + for method in [Rar29Method::Lz, Rar29Method::Ppmd] { + let options = WriterOptions::new(ArchiveVersion::Rar29, features).with_method(method); + let bytes = write_compressed_archive(&entries, options).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert_ne!(files[0].method, 0x30, "{method:?}"); + assert!(files[1].is_solid(), "{method:?}"); + for file in &files { + let packed = file.packed_data(&archive).unwrap(); + assert_eq!(packed[0] & 0x80 != 0, method == Rar29Method::Ppmd); + } + for member in collect_extract(&archive).unwrap() { + assert_eq!(member.data, payload, "{method:?}"); + } + } +} + +#[test] +fn generated_rar29_auto_filtered_archive_round_trips() { + let payload = b"\xe8\0\0\0\0rar29 auto filtered payload\n".repeat(16); + let entries = [FileEntry { + name: b"rar29-auto-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_auto( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + let plain = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + ) + .unwrap(); + let plain_archive = Archive::parse(&plain).unwrap(); + let plain_file = plain_archive.files().next().unwrap(); + assert!(file.pack_size <= plain_file.pack_size); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-auto-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_auto_filtered_archive_considers_delta_candidates() { + let payload: Vec = (0..768).map(|index| (index / 3) as u8).collect(); + let entries = [FileEntry { + name: b"rar29-auto-delta-candidate.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let auto = write_rar29_auto(&entries, options); + let explicit = write_rar29_filter(&entries, options, FilterKind::Delta { channels: 3 }); + let auto_archive = Archive::parse(&auto).unwrap(); + let explicit_archive = Archive::parse(&explicit).unwrap(); + let auto_file = auto_archive.files().next().unwrap(); + let explicit_file = explicit_archive.files().next().unwrap(); + + assert!(auto_file.pack_size <= explicit_file.pack_size); + let extracted = collect_extract(&auto_archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-auto-delta-candidate.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_auto_filtered_archive_considers_audio_candidates() { + let payload: Vec = (0..512) + .map(|index| (128 + ((index * 9) % 73) - ((index * 5) % 41)) as u8) + .collect(); + let entries = [FileEntry { + name: b"rar29-auto-audio-candidate.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let auto = write_rar29_auto(&entries, options); + let explicit = write_rar29_filter(&entries, options, FilterKind::Audio { channels: 2 }); + let auto_archive = Archive::parse(&auto).unwrap(); + let explicit_archive = Archive::parse(&explicit).unwrap(); + let auto_file = auto_archive.files().next().unwrap(); + let explicit_file = explicit_archive.files().next().unwrap(); + + assert!(auto_file.pack_size <= explicit_file.pack_size); + let extracted = collect_extract(&auto_archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-auto-audio-candidate.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_auto_filtered_archive_considers_rgb_candidates() { + let width = 24; + let mut payload = Vec::new(); + for y in 0..32 { + for x in 0..8 { + payload.extend_from_slice(&[ + (x * 23 + y * 3) as u8, + (x * 5 + y * 17) as u8, + (x * 13 + y * 19) as u8, + ]); + } + } + let entries = [FileEntry { + name: b"rar29-auto-rgb-candidate.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let auto = write_rar29_auto(&entries, options); + let explicit = write_rar29_filter(&entries, options, FilterKind::Rgb { width, pos_r: 0 }); + let auto_archive = Archive::parse(&auto).unwrap(); + let explicit_archive = Archive::parse(&explicit).unwrap(); + let auto_file = auto_archive.files().next().unwrap(); + let explicit_file = explicit_archive.files().next().unwrap(); + + assert!(auto_file.pack_size <= explicit_file.pack_size); + let extracted = collect_extract(&auto_archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-auto-rgb-candidate.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_auto_filtered_archive_considers_itanium_candidates() { + let payload = bytearray_like_itanium_payload(512); + let entries = [FileEntry { + name: b"rar29-auto-itanium-candidate.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let auto = write_rar29_auto(&entries, options); + let explicit = write_rar29_filter(&entries, options, FilterKind::Itanium); + let auto_archive = Archive::parse(&auto).unwrap(); + let explicit_archive = Archive::parse(&explicit).unwrap(); + let auto_file = auto_archive.files().next().unwrap(); + let explicit_file = explicit_archive.files().next().unwrap(); + + assert!(auto_file.pack_size <= explicit_file.pack_size); + let extracted = collect_extract(&auto_archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-auto-itanium-candidate.bin"); + assert_eq!(extracted[0].data, payload); +} + +fn bytearray_like_itanium_payload(len: usize) -> Vec { + let mut payload = vec![0u8; len]; + for (index, byte) in payload.iter_mut().enumerate() { + *byte = (index * 11 + 5) as u8; + } + for chunk in payload.chunks_mut(32) { + if chunk.len() > 21 { + chunk[16] = 22; + chunk[21] = 20; + } + } + payload +} + +#[test] +fn generated_rar29_segmented_e8_filtered_archive_round_trips() { + let mut payload = b"unfiltered prefix before x86 segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(b"\xe8\0\0\0\0rar29 segmented e8 filter payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after x86 segment\n"); + let entries = [FileEntry { + name: b"rar29-segmented-e8-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::E8, + filter_start..filter_end, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-segmented-e8-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_solid_e8_filtered_archive_round_trips() { + let first = b"\xe8\0\0\0\0rar29 solid e8 filtered first payload\n".repeat(12); + let second = b"\xe8\0\0\0\0rar29 solid e8 filtered second payload\n".repeat(12); + let entries = [ + FileEntry { + name: b"rar29-solid-e8-first.bin", + data: &first, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar29-solid-e8-second.bin", + data: &second, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterKind::E8, + ); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-solid-e8-first.bin"); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].name, b"rar29-solid-e8-second.bin"); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn generated_rar29_encrypted_e8_filtered_archive_round_trips() { + let payload = b"\xe8\0\0\0\0rar29 encrypted e8 filtered payload\n".repeat(12); + let entries = [FileEntry { + name: b"rar29-encrypted-e8-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let features = FeatureSet::store_only(); + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterKind::E8, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert!(file.is_encrypted()); + assert!(file.salt.is_some()); + assert!(matches!( + collect_extract_with_password(&archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"rar29-encrypted-e8-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar30_header_encrypted_e8_filtered_archive_round_trips() { + let payload = b"\xe8\0\0\0\0rar30 header encrypted e8 filtered payload\n".repeat(12); + let entries = [FileEntry { + name: b"rar30-header-encrypted-e8-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar30, features), + FilterKind::E8, + ); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + assert!(archive.main.has_encrypted_headers()); + let file = archive.files().next().unwrap(); + assert_eq!(file.name, b"rar30-header-encrypted-e8-filtered.bin"); + assert!(file.is_encrypted()); + assert!(file.salt.is_some()); + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 3.00 and 4.20 Wine prefixes"] +fn reference_unrar_accepts_rar29_solid_e8_filter_record() { + let Some(unrar300) = reference_unrar300() else { + return; + }; + let Some(unrar420) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-solid-e8-ref"); + let archive_path = dir.join("solid-e8.rar"); + let first = b"\xe8\0\0\0\0rar29 solid e8 first reference payload\n".repeat(12); + let second = b"\xe8\0\0\0\0rar29 solid e8 second reference payload\n".repeat(12); + let entries = [ + FileEntry { + name: b"solid-e8-first.bin", + data: &first, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-e8-second.bin", + data: &second, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterKind::E8, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + for reference in [&unrar300, &unrar420] { + let output = run_reference_unrar(reference, &archive_path); + assert!( + output.status.success(), + "{} rejected solid E8 archive: status={:?}\nstdout={}\nstderr={}", + reference.unrar, + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 4.20 Wine prefix"] +fn reference_unrar_accepts_rar29_segmented_e8_filter_record() { + let Some(reference) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-e8-ref"); + let archive_path = dir.join("segmented-e8.rar"); + let mut payload = b"unfiltered prefix before x86 segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(b"\xe8\0\0\0\0rar29 segmented e8 reference payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after x86 segment\n"); + let entries = [FileEntry { + name: b"segmented-e8.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::E8, + filter_start..filter_end, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 4.20 rejected segmented E8 archive: status={:?}\nstdout={}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 4.20 Wine prefix"] +fn reference_unrar_accepts_rar29_segmented_e8e9_filter_record() { + let Some(reference) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-e8e9-ref"); + let archive_path = dir.join("segmented-e8e9.rar"); + let mut payload = b"unfiltered prefix before x86 segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(b"\xe9\0\0\0\0rar29 segmented e8e9 reference payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after x86 segment\n"); + let entries = [FileEntry { + name: b"segmented-e8e9.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::E8E9, + filter_start..filter_end, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 4.20 rejected segmented E8E9 archive: status={:?}\nstdout={}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn generated_rar29_e8e9_filtered_archive_round_trips() { + let payload = b"\xe9\0\0\0\0rar29 e8e9 filter payload\n".repeat(16); + let entries = [FileEntry { + name: b"rar29-e8e9-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::E8E9, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + assert!(file.pack_size < file.unp_size); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-e8e9-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_delta_filtered_archive_round_trips() { + let payload: Vec = (0..384).map(|index| (index * 17 + 3) as u8).collect(); + let entries = [FileEntry { + name: b"rar29-delta-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Delta { channels: 3 }, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-delta-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_segmented_delta_filtered_archive_round_trips() { + let mut payload = b"unfiltered prefix before delta segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..384).map(|index| (index * 17 + 3) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after delta segment\n"); + let entries = [FileEntry { + name: b"rar29-segmented-delta-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Delta { channels: 3 }, + filter_start..filter_end, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-segmented-delta-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 4.20 Wine prefix"] +fn reference_unrar_accepts_rar29_segmented_delta_filter_record() { + let Some(reference) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-delta-ref"); + let archive_path = dir.join("segmented-delta.rar"); + let mut payload = b"unfiltered prefix before delta segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..384).map(|index| (index * 17 + 3) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after delta segment\n"); + let entries = [FileEntry { + name: b"segmented-delta.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Delta { channels: 3 }, + filter_start..filter_end, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 4.20 rejected segmented DELTA archive: status={:?}\nstdout={}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn generated_rar29_itanium_filtered_archive_round_trips() { + let mut payload = vec![0u8; 48]; + payload[16] = 22; + payload[21] = 20; + payload.extend_from_slice(b"rar29 itanium format payload\n"); + let entries = [FileEntry { + name: b"rar29-itanium-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Itanium, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-itanium-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_segmented_itanium_filtered_archive_round_trips() { + let mut payload = b"unfiltered prefix before itanium segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(&[0; 48]); + payload[filter_start + 16] = 22; + payload[filter_start + 21] = 20; + payload.extend_from_slice(b"rar29 segmented itanium format payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after itanium segment\n"); + let entries = [FileEntry { + name: b"rar29-segmented-itanium-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Itanium, + filter_start..filter_end, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-segmented-itanium-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 4.20 Wine prefix"] +fn reference_unrar_accepts_rar29_segmented_itanium_filter_record() { + let Some(reference) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-itanium-ref"); + let archive_path = dir.join("segmented-itanium.rar"); + let mut payload = b"unfiltered prefix before itanium segment ".to_vec(); + let filter_start = payload.len(); + payload.extend_from_slice(&[0; 48]); + payload[filter_start + 16] = 22; + payload[filter_start + 21] = 20; + payload.extend_from_slice(b"rar29 segmented itanium reference payload\n"); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after itanium segment\n"); + let entries = [FileEntry { + name: b"segmented-itanium.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Itanium, + filter_start..filter_end, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 4.20 rejected segmented ITANIUM archive: status={:?}\nstdout={}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn generated_rar29_rgb_filtered_archive_round_trips() { + let width = 12; + let payload: Vec = (0..96).map(|index| (index * 31 + 13) as u8).collect(); + let entries = [FileEntry { + name: b"rar29-rgb-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Rgb { width, pos_r: 0 }, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-rgb-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_segmented_rgb_filtered_archive_round_trips() { + let width = 12; + let mut payload = b"unfiltered prefix before rgb segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..96).map(|index| (index * 31 + 13) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after rgb segment\n"); + let entries = [FileEntry { + name: b"rar29-segmented-rgb-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Rgb { width, pos_r: 0 }, + filter_start..filter_end, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-segmented-rgb-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 4.20 Wine prefix"] +fn reference_unrar_accepts_rar29_segmented_rgb_filter_record() { + let Some(reference) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-rgb-ref"); + let archive_path = dir.join("segmented-rgb.rar"); + let width = 12; + let mut payload = b"unfiltered prefix before rgb segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..96).map(|index| (index * 31 + 13) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after rgb segment\n"); + let entries = [FileEntry { + name: b"segmented-rgb.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Rgb { width, pos_r: 0 }, + filter_start..filter_end, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 4.20 rejected segmented RGB archive: status={:?}\nstdout={}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn generated_rar29_audio_filtered_archive_round_trips() { + let payload: Vec = (0..160) + .map(|index| (index * 9 + index / 5) as u8) + .collect(); + let entries = [FileEntry { + name: b"rar29-audio-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Audio { channels: 2 }, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-audio-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_segmented_audio_filtered_archive_round_trips() { + let mut payload = b"unfiltered prefix before audio segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..160).map(|index| (index * 9 + index / 5) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after audio segment\n"); + let entries = [FileEntry { + name: b"rar29-segmented-audio-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Audio { channels: 2 }, + filter_start..filter_end, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"rar29-segmented-audio-filtered.bin"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_large_delta_filter_uses_multiple_vm_records() { + let payload: Vec = (0..240_064) + .map(|index| (index * 11 + index / 5 + index / 251) as u8) + .collect(); + let entries = [FileEntry { + name: b"rar29-large-delta-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Delta { channels: 4 }, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn generated_rar29_large_audio_filter_uses_multiple_redeclared_vm_records() { + let payload: Vec = (0..240_064) + .map(|index| (index * 7 + index / 3 + index / 257) as u8) + .collect(); + let entries = [FileEntry { + name: b"rar29-large-audio-filtered.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_filter( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Audio { channels: 4 }, + ); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 3.00 Wine prefix"] +fn reference_unrar300_accepts_rar29_segmented_filter_records() { + let Some(reference) = reference_unrar300() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-unrar300-ref"); + + let mut e8 = b"unfiltered prefix before x86 segment ".to_vec(); + let e8_start = e8.len(); + e8.extend_from_slice(b"\xe8\0\0\0\0rar29 segmented e8 reference payload\n"); + let e8_end = e8.len(); + e8.extend_from_slice(b"unfiltered suffix after x86 segment\n"); + write_reference_segmented_archive( + &dir.join("segmented-e8.rar"), + b"segmented-e8.bin", + &e8, + |entries| { + write_rar29_compressed_archive_with_filter_policy( + entries, + reference_rar29_options(), + FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8, e8_start..e8_end)), + ) + }, + ); + + let mut e8e9 = b"unfiltered prefix before x86 segment ".to_vec(); + let e8e9_start = e8e9.len(); + e8e9.extend_from_slice(b"\xe9\0\0\0\0rar29 segmented e8e9 reference payload\n"); + let e8e9_end = e8e9.len(); + e8e9.extend_from_slice(b"unfiltered suffix after x86 segment\n"); + write_reference_segmented_archive( + &dir.join("segmented-e8e9.rar"), + b"segmented-e8e9.bin", + &e8e9, + |entries| { + write_rar29_compressed_archive_with_filter_policy( + entries, + reference_rar29_options(), + FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8E9, e8e9_start..e8e9_end)), + ) + }, + ); + + let mut delta = b"unfiltered prefix before delta segment ".to_vec(); + let delta_start = delta.len(); + delta.extend((0..384).map(|index| (index * 17 + 3) as u8)); + let delta_end = delta.len(); + delta.extend_from_slice(b"unfiltered suffix after delta segment\n"); + write_reference_segmented_archive( + &dir.join("segmented-delta.rar"), + b"segmented-delta.bin", + &delta, + |entries| { + write_rar29_compressed_archive_with_filter_policy( + entries, + reference_rar29_options(), + FilterPolicy::Explicit(FilterSpec::range( + FilterKind::Delta { channels: 3 }, + delta_start..delta_end, + )), + ) + }, + ); + + let mut itanium = b"unfiltered prefix before itanium segment ".to_vec(); + let itanium_start = itanium.len(); + itanium.extend_from_slice(&[0; 48]); + itanium[itanium_start + 16] = 22; + itanium[itanium_start + 21] = 20; + itanium.extend_from_slice(b"rar29 segmented itanium reference payload\n"); + let itanium_end = itanium.len(); + itanium.extend_from_slice(b"unfiltered suffix after itanium segment\n"); + write_reference_segmented_archive( + &dir.join("segmented-itanium.rar"), + b"segmented-itanium.bin", + &itanium, + |entries| { + write_rar29_compressed_archive_with_filter_policy( + entries, + reference_rar29_options(), + FilterPolicy::Explicit(FilterSpec::range( + FilterKind::Itanium, + itanium_start..itanium_end, + )), + ) + }, + ); + + let mut rgb = b"unfiltered prefix before rgb segment ".to_vec(); + let rgb_start = rgb.len(); + rgb.extend((0..96).map(|index| (index * 31 + 13) as u8)); + let rgb_end = rgb.len(); + rgb.extend_from_slice(b"unfiltered suffix after rgb segment\n"); + write_reference_segmented_archive( + &dir.join("segmented-rgb.rar"), + b"segmented-rgb.bin", + &rgb, + |entries| { + write_rar29_compressed_archive_with_filter_policy( + entries, + reference_rar29_options(), + FilterPolicy::Explicit(FilterSpec::range( + FilterKind::Rgb { + width: 12, + pos_r: 0, + }, + rgb_start..rgb_end, + )), + ) + }, + ); + + let mut audio = b"unfiltered prefix before audio segment ".to_vec(); + let audio_start = audio.len(); + audio.extend((0..160).map(|index| (index * 9 + index / 5) as u8)); + let audio_end = audio.len(); + audio.extend_from_slice(b"unfiltered suffix after audio segment\n"); + write_reference_segmented_archive( + &dir.join("segmented-audio.rar"), + b"segmented-audio.bin", + &audio, + |entries| { + write_rar29_compressed_archive_with_filter_policy( + entries, + reference_rar29_options(), + FilterPolicy::Explicit(FilterSpec::range( + FilterKind::Audio { channels: 2 }, + audio_start..audio_end, + )), + ) + }, + ); + + for archive_path in [ + "segmented-e8.rar", + "segmented-e8e9.rar", + "segmented-delta.rar", + "segmented-itanium.rar", + "segmented-rgb.rar", + "segmented-audio.rar", + ] { + let archive_path = dir.join(archive_path); + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 3.00 rejected {}: status={:?}\nstdout={}\nstderr={}", + archive_path.display(), + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +fn reference_rar29_options() -> WriterOptions { + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) +} + +fn write_reference_segmented_archive( + archive_path: &Path, + name: &'static [u8], + payload: &[u8], + write: impl FnOnce(&[FileEntry<'_>]) -> rars::Result>, +) { + let entries = [FileEntry { + name, + data: payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write(&entries).unwrap(); + std::fs::write(archive_path, bytes).unwrap(); +} + +#[test] +#[ignore = "requires WinRAR/UnRAR 4.20 Wine prefix"] +fn reference_unrar_accepts_rar29_segmented_audio_filter_record() { + let Some(reference) = reference_unrar420() else { + return; + }; + + let dir = scratch::case("rars-rar29-segmented-audio-ref"); + let archive_path = dir.join("segmented-audio.rar"); + let mut payload = b"unfiltered prefix before audio segment ".to_vec(); + let filter_start = payload.len(); + payload.extend((0..160).map(|index| (index * 9 + index / 5) as u8)); + let filter_end = payload.len(); + payload.extend_from_slice(b"unfiltered suffix after audio segment\n"); + let entries = [FileEntry { + name: b"segmented-audio.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_rar29_filter_range( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterKind::Audio { channels: 2 }, + filter_start..filter_end, + ); + std::fs::write(&archive_path, bytes).unwrap(); + + let output = run_reference_unrar(&reference, &archive_path); + assert!( + output.status.success(), + "UnRAR 4.20 rejected segmented AUDIO archive: status={:?}\nstdout={}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn extract_to_reports_rar15_entry_context_on_write_failure() { + struct FailWriter; + + impl Write for FailWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("sink failed")) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let bytes = std::fs::read(fixture("rars_generated/stored.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let error = archive + .extract_to(ArchiveReadOptions::default(), |_meta| { + Ok(Box::new(FailWriter)) + }) + .unwrap_err(); + + match error { + Error::AtEntry { + name, + operation, + source, + } => { + assert_eq!(name, b"payload.txt"); + assert_eq!(operation, "extracting"); + assert!(matches!(*source, Error::Io(_))); + } + other => panic!("expected entry context, got {other:?}"), + } +} + +#[test] +fn extracts_file_data_when_a_nested_legacy_comment_header_is_malformed() { + let entry = StoredEntry { + name: b"file", + data: b"payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"note"), + }; + let original = write_stored_archive(&[entry], WriterOptions::default()).unwrap(); + let archive = Archive::parse(&original).unwrap(); + let file = archive.files().next().unwrap(); + let nested = file.block.offset + 32 + file.name.len(); + assert_eq!(original[nested + 2], 0x75); + for (kind, size) in [(0x7f, 17u16), (0x75, 12), (0x75, u16::MAX)] { + let mut bytes = original.clone(); + bytes[nested + 2] = kind; + bytes[nested + 5..nested + 7].copy_from_slice(&size.to_le_bytes()); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!( + archive.files().next().unwrap().file_comment().unwrap(), + None + ); + assert_eq!(collect_extract(&archive).unwrap()[0].data, b"payload"); + } +} + +#[test] +fn extracts_stored_newsub_comments_with_and_without_encryption() { + let plain = b"stored service comment"; + for password in [None, Some(b"secret".as_slice())] { + let entry = StoredEntry { + name: b"CMT", + data: plain, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password, + file_comment: None, + }; + let mut bytes = write_stored_archive( + &[entry], + WriterOptions::new(ArchiveVersion::Rar30, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let start = file.block.offset; + bytes[start + 2] = 0x7a; // NEWSUB_HEAD has the same file payload layout. + let crc = crc32(&bytes[start + 2..start + file.block.head_size as usize]) as u16; + bytes[start..start + 2].copy_from_slice(&crc.to_le_bytes()); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.files().next().is_none()); + assert!(archive.new_subs().next().unwrap().file.is_stored()); + let options = ArchiveReadOptions::with_optional_password(password); + assert_eq!( + archive + .archive_comment_with_options(options) + .unwrap() + .unwrap(), + plain + ); + assert_eq!( + archive + .archive_comment_with_options( + options.with_max_member_output_bytes(plain.len() as u64) + ) + .unwrap() + .unwrap(), + plain + ); + assert_eq!( + archive + .archive_comment_with_options( + options.with_max_member_output_bytes(plain.len() as u64 - 1) + ) + .unwrap_err() + .kind(), + rars::ErrorKind::ResourceLimit + ); + if password.is_some() { + assert_eq!( + archive.archive_comment().unwrap_err().kind(), + rars::ErrorKind::PasswordRequired + ); + } + } +} + +#[test] +fn extracts_compressed_legacy_comment_blocks_with_rar15_and_rar20_coding() { + let plain = b"compressed old comment compressed old comment\n"; + for (version, comment_version) in [ + (ArchiveVersion::Rar15, 15), + (ArchiveVersion::Rar20, 20), + (ArchiveVersion::Rar20, 26), + ] { + let entry = FileEntry { + name: b"file", + data: plain, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let original = write_compressed_archive( + &[entry], + WriterOptions::new(version, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&original).unwrap(); + let file = archive.files().next().unwrap(); + assert!( + !file.is_stored(), + "{version:?} must exercise comment decoding" + ); + let packed = file.packed_data(&archive).unwrap(); + let mut comment = vec![0, 0, 0x75, 0, 0]; + comment.extend_from_slice(&((13 + packed.len()) as u16).to_le_bytes()); + comment.extend_from_slice(&(plain.len() as u16).to_le_bytes()); + comment.push(comment_version); + comment.push(file.method); + comment.extend_from_slice(&((crc32(plain) & 0xffff) as u16).to_le_bytes()); + let crc = crc32(&comment[2..13]) as u16; + comment[..2].copy_from_slice(&crc.to_le_bytes()); + comment.extend_from_slice(&packed); + let mut bytes = original[..20].to_vec(); + bytes[10] |= 2; // MHD_COMMENT; main flags at signature+3. + bytes[12..14].copy_from_slice(&((13 + comment.len()) as u16).to_le_bytes()); + let crc = crc32(&bytes[9..20]) as u16; + bytes[7..9].copy_from_slice(&crc.to_le_bytes()); + bytes.extend_from_slice(&comment); + bytes.extend_from_slice(&original[20..]); + let parsed = Archive::parse(&bytes).unwrap(); + assert_eq!(parsed.archive_comment().unwrap().unwrap(), plain); + assert_eq!( + parsed + .archive_comment_with_options( + ArchiveReadOptions::new().with_max_reader_workspace_bytes(1024), + ) + .unwrap_err() + .kind(), + rars::ErrorKind::ResourceLimit, + "{version:?} compressed comment must allocate a decoder", + ); + assert_eq!( + parsed + .archive_comment_with_options( + ArchiveReadOptions::new().with_max_member_output_bytes(plain.len() as u64), + ) + .unwrap() + .unwrap(), + plain + ); + assert_eq!( + parsed + .archive_comment_with_options( + ArchiveReadOptions::new().with_max_member_output_bytes(plain.len() as u64 - 1), + ) + .unwrap_err() + .kind(), + rars::ErrorKind::ResourceLimit + ); + assert_eq!(collect_extract(&parsed).unwrap()[0].data, plain); + } +} + +#[test] +fn extracts_stored_solid_flagged_member_via_parallel_entry_point() { + let entry = StoredEntry { + name: b"file", + data: b"payload", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let mut bytes = write_stored_archive( + &[entry], + WriterOptions::new(ArchiveVersion::Rar30, FeatureSet::store_only()), + ) + .unwrap(); + let parsed = Archive::parse(&bytes).unwrap(); + let file = parsed.files().next().unwrap(); + let start = file.block.offset; + bytes[start + 3] |= 0x10; + let crc = crc32(&bytes[start + 2..start + file.block.head_size as usize]) as u16; + bytes[start..start + 2].copy_from_slice(&crc.to_le_bytes()); + let archive = Archive::parse(&bytes).unwrap(); + assert!(!archive.main.is_solid()); + assert!(archive.files().next().unwrap().is_solid()); + let output = Rc::new(RefCell::new(Vec::new())); + archive + .extract_to_parallel_buffered(ArchiveReadOptions::new(), |_| { + Ok(Box::new(CollectWriter { + data: output.clone(), + })) + }) + .unwrap(); + assert_eq!(*output.borrow(), b"payload"); +} + +#[test] +fn rejects_unsupported_stored_split_cipher_without_decoding_or_diagnostics() { + let entry = StoredEntry { + name: b"file", + data: &[42; 120], + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"secret"), + file_comment: None, + }; + let parts = write_stored_volumes(entry, WriterOptions::default(), 64).unwrap(); + assert!(parts.len() > 1); + let mut archives: Vec<_> = parts + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect(); + for archive in &mut archives { + for block in &mut archive.blocks { + if let Block::File(file) = block { + file.unp_ver = 17; + } + } + } + let error = extract_volumes_to( + &archives, + ArchiveReadOptions::with_password(b"secret"), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::UnsupportedEncryption { + unpack_version: 17, + .. + } + )); +} + +#[test] +fn extracts_stored_members_through_public_member_writer_for_each_cipher() { + for version in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for password in [None, Some(b"password".as_slice())] { + let entry = StoredEntry { + name: b"stored.txt", + data: b"stored member data", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password, + file_comment: None, + }; + let bytes = write_stored_archive( + &[entry], + WriterOptions::new(version, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.is_stored()); + let mut output = Vec::new(); + file.write_to(&archive, password, &mut output).unwrap(); + assert_eq!( + output, + b"stored member data", + "{version:?}, encrypted {}", + password.is_some() + ); + if password.is_some() && version == ArchiveVersion::Rar20 { + let mut rar26 = file.clone(); + rar26.unp_ver = 26; + let mut output = Vec::new(); + rar26.write_to(&archive, password, &mut output).unwrap(); + assert_eq!(output, b"stored member data"); + } + if password.is_some() && version == ArchiveVersion::Rar15 { + let mut unsupported = file.clone(); + unsupported.unp_ver = 17; + let mut output = Vec::new(); + assert!(matches!( + unsupported.write_to(&archive, password, &mut output), + Err(Error::UnsupportedEncryption { + unpack_version: 17, + .. + }) + )); + assert!(output.is_empty()); + } + if password.is_some() && version != ArchiveVersion::Rar15 { + let mut truncated = file.clone(); + truncated.packed_range.end -= 1; + let mut output = Vec::new(); + assert!(matches!( + truncated.write_to(&archive, password, &mut output), + Err(Error::WrongPasswordOrCorruptData) + )); + assert!(output.is_empty()); + } + } + } +} + +#[test] +fn writes_store_only_rar15_archive_that_reader_extracts() { + let entries = [ + StoredEntry { + name: b"hello.txt", + data: b"Hello from a generated RAR 1.5 archive.\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + StoredEntry { + name: b"tiny.bin", + data: b"\x00\x01\x02\x03", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + + let bytes = write_stored_archive(&entries, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"hello.txt"); + assert_eq!(files[0].unp_ver, 15); + assert_eq!(files[0].method, 0x30); + assert_eq!(files[0].file_crc, crc32(entries[0].data)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); +} + +#[test] +fn writes_stored_rar15_archive_comment_that_reader_decodes() { + let features = FeatureSet::store_only(); + let entries = [StoredEntry { + name: b"hello.txt", + data: b"hello with comment\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_stored_archive_with_comment( + &entries, + WriterOptions::new(ArchiveVersion::Rar15, features), + Some(b"archive note\n"), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert!(archive.main.has_archive_comment()); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"archive note\n"[..]) + ); + assert_eq!(collect_extract(&archive).unwrap()[0].data, entries[0].data); +} + +#[test] +fn writes_rar15_file_comments_that_reader_decodes() { + let features = FeatureSet::store_only(); + + let stored = [StoredEntry { + name: b"stored-comment.txt", + data: b"stored file comment payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"stored note\r\n"), + }]; + let stored_bytes = + write_stored_archive(&stored, WriterOptions::new(ArchiveVersion::Rar15, features)).unwrap(); + let stored_archive = Archive::parse(&stored_bytes).unwrap(); + let stored_file = stored_archive.files().next().unwrap(); + assert!(stored_file.has_file_comment()); + assert_eq!( + stored_file.file_comment().unwrap().as_deref(), + Some(&b"stored note\r\n"[..]) + ); + assert_eq!( + collect_extract(&stored_archive).unwrap()[0].data, + stored[0].data + ); + + let compressed = [FileEntry { + name: b"compressed-comment.txt", + data: b"compressed file comment payload compressed file comment payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"compressed note"), + }]; + let compressed_bytes = write_compressed_archive( + &compressed, + WriterOptions::new(ArchiveVersion::Rar15, features), + ) + .unwrap(); + let compressed_archive = Archive::parse(&compressed_bytes).unwrap(); + let compressed_file = compressed_archive.files().next().unwrap(); + assert!(compressed_file.has_file_comment()); + assert_eq!( + compressed_file.file_comment().unwrap().as_deref(), + Some(&b"compressed note"[..]) + ); + assert_eq!( + collect_extract(&compressed_archive).unwrap()[0].data, + compressed[0].data + ); +} + +#[test] +fn writes_rar20_old_style_comments_that_reader_decodes() { + let archive_features = FeatureSet::store_only(); + let stored = [StoredEntry { + name: b"rar20-commented.txt", + data: b"rar20 archive comment payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let archive_bytes = write_stored_archive_with_comment( + &stored, + WriterOptions::new(ArchiveVersion::Rar20, archive_features), + Some(b"rar20 archive note\r\n"), + ) + .unwrap(); + let archive = Archive::parse(&archive_bytes).unwrap(); + assert!(archive.main.has_archive_comment()); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(b"rar20 archive note\r\n".as_slice()) + ); + assert_eq!(collect_extract(&archive).unwrap()[0].data, stored[0].data); + + let file_features = FeatureSet::store_only(); + let compressed = [FileEntry { + name: b"rar20-file-commented.txt", + data: b"rar20 compressed file comment payload payload payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"rar20 file note"), + }]; + let file_bytes = write_compressed_archive_with_comment( + &compressed, + WriterOptions::new(ArchiveVersion::Rar20, file_features), + None, + ) + .unwrap(); + let archive = Archive::parse(&file_bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.unp_ver, 20); + assert!(file.has_file_comment()); + assert_eq!( + file.file_comment().unwrap().as_deref(), + Some(b"rar20 file note".as_slice()) + ); + assert_eq!( + collect_extract(&archive).unwrap()[0].data, + compressed[0].data + ); +} + +#[test] +fn writes_rar29_old_style_comments_that_reader_decodes() { + let archive_features = FeatureSet::store_only(); + let stored = [StoredEntry { + name: b"rar29-commented.txt", + data: b"rar29 archive comment payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let archive_bytes = write_stored_archive_with_comment( + &stored, + WriterOptions::new(ArchiveVersion::Rar29, archive_features), + Some(b"rar29 archive note\r\n"), + ) + .unwrap(); + let archive = Archive::parse(&archive_bytes).unwrap(); + assert!(archive.main.has_archive_comment()); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(b"rar29 archive note\r\n".as_slice()) + ); + assert_eq!(collect_extract(&archive).unwrap()[0].data, stored[0].data); + + let file_features = FeatureSet::store_only(); + let compressed = [FileEntry { + name: b"rar29-file-commented.txt", + data: b"rar29 compressed file comment payload payload payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"rar29 file note"), + }]; + let file_bytes = write_compressed_archive_with_comment( + &compressed, + WriterOptions::new(ArchiveVersion::Rar29, file_features), + None, + ) + .unwrap(); + let archive = Archive::parse(&file_bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.unp_ver, 29); + assert!(file.has_file_comment()); + assert_eq!( + file.file_comment().unwrap().as_deref(), + Some(b"rar29 file note".as_slice()) + ); + assert_eq!( + collect_extract(&archive).unwrap()[0].data, + compressed[0].data + ); +} + +#[test] +fn writes_rar29_archive_comment_with_empty_auto_members() { + let features = FeatureSet::store_only(); + let entries = [ + FileEntry { + name: b"file1.txt", + data: b"", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"file2.txt", + data: b"", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + + let bytes = write_compressed_archive_with_comment( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + Some(b"RARcomment"), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(b"RARcomment".as_slice()) + ); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert!(files.iter().all(|file| file.method == 0x30)); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert!(extracted.iter().all(|entry| entry.data.is_empty())); +} + +#[test] +fn writes_rar3_newsub_archive_comment_that_reader_decodes() { + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + let features = FeatureSet::store_only(); + let entries = [FileEntry { + name: b"rar3-commented.txt", + data: b"rar3 NEWSUB comment payload payload payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let bytes = write_compressed_archive_with_comment( + &entries, + WriterOptions::new(target, features), + Some(b"rar3 NEWSUB archive note\r\n"), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert!(!archive.main.has_archive_comment()); + let subblocks: Vec<_> = archive.new_subs().collect(); + assert_eq!(subblocks.len(), 1); + assert_eq!(subblocks[0].kind, NewSubKind::ArchiveComment); + assert_eq!(subblocks[0].file.name, b"CMT"); + assert_eq!(subblocks[0].file.unp_ver, 29); + assert_eq!(subblocks[0].file.method, 0x33); + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(b"rar3 NEWSUB archive note\r\n".as_slice()) + ); + assert_eq!(collect_extract(&archive).unwrap()[0].data, entries[0].data); + } +} + +#[test] +fn rar15_writer_downgrades_unix_metadata_for_unrar_compatibility() { + let unix_regular_file = 0o100664; + let entries = [StoredEntry { + name: b"unix-mode.txt", + data: b"RAR15 stored data\n", + file_time: 0x5a21_0000, + file_attr: unix_regular_file, + host_os: 3, + password: None, + file_comment: None, + }]; + + let rar15 = write_stored_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar15, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&rar15).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 15); + assert_eq!(file.host_os, 0); + assert_eq!(file.attr, 0x20); + assert!(!file.is_directory()); + assert_eq!(collect_extract(&archive).unwrap()[0].data, entries[0].data); + + let rar20 = write_stored_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&rar20).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 20); + assert_eq!(file.host_os, 3); + assert_eq!(file.attr, unix_regular_file); +} + +#[test] +fn writes_compressed_rar15_archive_that_reader_extracts() { + let entries = [ + FileEntry { + name: b"alpha.txt", + data: b"alpha beta gamma alpha beta gamma alpha beta gamma\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"binary.bin", + data: b"\x00\x01\x02\x03\x00\x01\x02\x03\x00\x01\x02\x03", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + + let bytes = write_compressed_archive(&entries, WriterOptions::default()).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"alpha.txt"); + assert_eq!(files[0].unp_ver, 15); + assert_eq!(files[0].method, 0x33); + assert_eq!(files[0].file_crc, crc32(entries[0].data)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); +} + +#[test] +fn rar15_writer_levels_control_unpack15_encoder_policy() { + let mut data: Vec<_> = (0..5000).map(|index| (index * 73 + 19) as u8).collect(); + data.extend_from_within(..256); + let entries = [FileEntry { + name: b"rar15-level-policy.bin", + data: &data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let mut packed_sizes = [0; 6]; + for level in 0..=5 { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar15, FeatureSet::store_only()) + .with_compression_level(level), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.method, 0x30 + level); + packed_sizes[usize::from(level)] = file.pack_size; + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); + } + assert!(packed_sizes[5] < packed_sizes[1]); +} + +#[test] +fn rar15_solid_level_zero_stores_each_member_without_a_solid_dependency() { + let entries = [ + FileEntry { + name: b"first", + data: b"repeated repeated repeated", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"second", + data: b"repeated repeated repeated again", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar15, features).with_compression_level(0), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.files().all(|file| file.method == 0x30)); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); +} + +#[test] +fn stored_writer_refuses_solid_option_before_emitting_members() { + let entries = [ + StoredEntry { + name: b"first", + data: b"shared shared shared", + file_time: 0, + file_attr: 0x20, + host_os: 0, + password: None, + file_comment: None, + }, + StoredEntry { + name: b"second", + data: b"shared shared shared again", + file_time: 0, + file_attr: 0x20, + host_os: 0, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + for target in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ] { + assert!( + matches!( + write_stored_archive(&entries, WriterOptions::new(target, features)), + Err(Error::UnsupportedWriterOption { .. }) + ), + "{target:?}" + ); + } +} + +#[test] +fn later_legacy_solid_level_zero_stores_each_member() { + let entries = [ + FileEntry { + name: b"first", + data: b"shared shared shared", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"second", + data: b"shared shared shared again", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + for target in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(target, features).with_compression_level(0), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!( + archive.files().all(|file| file.method == 0x30), + "{target:?}" + ); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entries[0].data, "{target:?}"); + assert_eq!(extracted[1].data, entries[1].data, "{target:?}"); + } +} + +#[test] +fn writes_literal_compressed_rar20_archive_that_reader_extracts() { + let entries = [ + FileEntry { + name: b"rar20-alpha.txt", + data: b"RAR 2.0 literal writer baseline alpha alpha alpha\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar20-binary.bin", + data: b"\x00\xff\x00\xff\x10\x20\x30\x40\x00\xff", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar20-run.bin", + data: &[b'A'; 1024], + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar20-repeat.bin", + data: &b"abc123xyz-"[..].repeat(128), + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 4); + assert!(files.iter().all(|file| file.unp_ver == 20)); + assert_eq!(files[0].method, 0x30); + assert_eq!(files[1].method, 0x30); + assert_eq!(files[2].method, 0x33); + assert_eq!(files[3].method, 0x33); + assert_eq!(files[0].file_crc, crc32(entries[0].data)); + assert_eq!(files[1].file_crc, crc32(entries[1].data)); + assert_eq!(files[2].file_crc, crc32(entries[2].data)); + assert_eq!(files[3].file_crc, crc32(entries[3].data)); + assert!(files.iter().all(|file| !file.is_solid())); + assert!(files[2].pack_size < files[2].unp_size / 4); + assert!(files[3].pack_size < files[3].unp_size / 2); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 4); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); + assert_eq!(extracted[2].data, entries[2].data); + assert_eq!(extracted[3].data, entries[3].data); +} + +#[test] +fn rar20_writer_uses_audio_block_for_pcm_like_payload_when_smaller() { + let mut payload = Vec::new(); + for sample in 0..8192i16 { + let left = sample.wrapping_mul(3).wrapping_add(200); + let right = sample.wrapping_mul(3).wrapping_sub(200); + payload.extend_from_slice(&left.to_le_bytes()); + payload.extend_from_slice(&right.to_le_bytes()); + } + let lz_packed = rars::codec::rar20::unpack20_encode_literals(&payload).unwrap(); + let entries = [FileEntry { + name: b"rar20-audio.wav", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 20); + assert_eq!(file.method, 0x33); + assert!(file.pack_size < lz_packed.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn rar20_writer_uses_audio_block_for_boat_modern_english_fixture() { + let source = std::fs::read(fixture("rar250/unpack20_audio_text.rar")).unwrap(); + let source_archive = Archive::parse(&source).unwrap(); + let source_entries = collect_extract(&source_archive).unwrap(); + let payload = &source_entries[0].data; + let lz_packed = rars::codec::rar20::unpack20_encode_literals(payload).unwrap(); + let entries = [FileEntry { + name: b"BoatModernEnglish.wav", + data: payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 20); + assert_eq!(file.method, 0x33); + assert!(file.pack_size < lz_packed.len() as u64); + assert!(file.pack_size < file.unp_size / 2); + assert_eq!(collect_extract(&archive).unwrap()[0].data, *payload); +} + +#[test] +fn rar20_writer_uses_repeat_last_for_regular_pcm_fixture() { + let source = std::fs::read(fixture("rar250/AUDIO.RAR")).unwrap(); + let source_archive = Archive::parse(&source).unwrap(); + let source_file = source_archive.files().next().unwrap(); + let source_entries = collect_extract(&source_archive).unwrap(); + let payload = &source_entries[0].data; + let entries = [FileEntry { + name: b"PCM_LR.WAV", + data: payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.unp_ver, 20); + assert_eq!(file.method, 0x33); + assert!(file.pack_size < file.unp_size / 8); + assert!(file.pack_size < source_file.pack_size * 2); + assert_eq!(collect_extract(&archive).unwrap()[0].data, *payload); +} + +#[test] +fn rar20_writer_stamps_requested_method_levels() { + let payload = b"rar20 level method payload alpha beta gamma\n".repeat(64); + let entries = [FileEntry { + name: b"rar20-level.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for level in 1..=5 { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()) + .with_compression_level(level), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30 + level); + assert_eq!(file.block.flags & 0x00e0, 0); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } +} + +#[test] +fn rar20_writer_stores_member_when_lz_payload_would_grow() { + let payload = (0..41u16) + .map(|index| { + let value = index.wrapping_mul(73).wrapping_add(index >> 3); + value as u8 + }) + .collect::>(); + let entries = [FileEntry { + name: b"rar20-randomish.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()) + .with_compression_level(5), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert!(matches!(file.method, 0x30 | 0x35)); + assert!(file.pack_size <= payload.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn writes_encrypted_rar20_archives_that_reader_extracts_with_password() { + let features = FeatureSet::store_only(); + let options = WriterOptions::new(ArchiveVersion::Rar20, features); + + let stored = [StoredEntry { + name: b"rar20-secret-store.txt", + data: b"RAR 2.0 encrypted stored payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let stored_bytes = write_stored_archive(&stored, options).unwrap(); + let stored_archive = Archive::parse(&stored_bytes).unwrap(); + let stored_file = stored_archive.files().next().unwrap(); + assert_eq!(stored_file.unp_ver, 20); + assert!(stored_file.is_encrypted()); + assert_eq!(stored_file.pack_size % 16, 0); + assert!(matches!( + collect_extract(&stored_archive), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&stored_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&stored_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored[0].data); + + let compressed = [FileEntry { + name: b"rar20-secret-compressed.txt", + data: b"RAR 2.0 encrypted compressed payload payload payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let compressed_bytes = write_compressed_archive(&compressed, options).unwrap(); + let compressed_archive = Archive::parse(&compressed_bytes).unwrap(); + let compressed_file = compressed_archive.files().next().unwrap(); + assert_eq!(compressed_file.unp_ver, 20); + assert!(compressed_file.is_encrypted()); + assert_eq!(compressed_file.pack_size % 16, 0); + assert!(matches!( + collect_extract(&compressed_archive), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&compressed_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&compressed_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed[0].data); +} + +#[test] +fn writes_encrypted_rar29_archives_that_reader_extracts_with_password() { + let features = FeatureSet::store_only(); + let options = WriterOptions::new(ArchiveVersion::Rar29, features); + + let stored = [StoredEntry { + name: b"rar29-secret-store.txt", + data: b"RAR 2.9 encrypted stored payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let stored_bytes = write_stored_archive(&stored, options).unwrap(); + let stored_archive = Archive::parse(&stored_bytes).unwrap(); + let stored_file = stored_archive.files().next().unwrap(); + assert_eq!(stored_file.unp_ver, 29); + assert!(stored_file.is_encrypted()); + assert!(stored_file.salt.is_some()); + assert_eq!(stored_file.pack_size % 16, 0); + assert!(matches!( + collect_extract(&stored_archive), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&stored_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&stored_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored[0].data); + + let compressed = [FileEntry { + name: b"rar29-secret-compressed.txt", + data: b"RAR 2.9 encrypted compressed payload payload payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let compressed_bytes = write_compressed_archive(&compressed, options).unwrap(); + let compressed_archive = Archive::parse(&compressed_bytes).unwrap(); + let compressed_file = compressed_archive.files().next().unwrap(); + assert_eq!(compressed_file.unp_ver, 29); + assert!(compressed_file.is_encrypted()); + assert!(compressed_file.salt.is_some()); + assert_eq!(compressed_file.pack_size % 16, 0); + assert!(matches!( + collect_extract(&compressed_archive), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&compressed_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&compressed_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed[0].data); +} + +#[test] +fn writes_literal_compressed_rar29_archive_that_reader_extracts() { + let entries = [ + FileEntry { + name: b"rar29-alpha.txt", + data: b"RAR 2.9 literal writer baseline alpha alpha alpha\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar29-binary.bin", + data: b"\x00\xff\x00\xff\x10\x20\x30\x40\x00\xff", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar29-run.bin", + data: &[b'Z'; 1024], + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"rar29-repeat.bin", + data: &b"abc123xyz-"[..].repeat(128), + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 4); + assert!(files.iter().all(|file| file.unp_ver == 29)); + assert!(files + .iter() + .all(|file| matches!(file.method, 0x30 | 0x33 | 0x35))); + assert_eq!(files[0].file_crc, crc32(entries[0].data)); + assert_eq!(files[1].file_crc, crc32(entries[1].data)); + assert_eq!(files[2].file_crc, crc32(entries[2].data)); + assert_eq!(files[3].file_crc, crc32(entries[3].data)); + assert!(files.iter().all(|file| !file.is_solid())); + assert!(files[2].pack_size < files[2].unp_size / 4); + assert!(files[3].pack_size < files[3].unp_size / 2); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 4); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); + assert_eq!(extracted[2].data, entries[2].data); + assert_eq!(extracted[3].data, entries[3].data); +} + +#[test] +fn writes_solid_compressed_rar29_rar30_and_rar40_archives_that_reader_extracts() { + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let entries = [ + FileEntry { + name: b"solid-one.txt", + data: b"solid writer baseline alpha beta gamma alpha beta gamma\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-two.txt", + data: b"solid writer baseline alpha beta gamma delta epsilon\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = + write_compressed_archive(&entries, WriterOptions::new(target, features)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 2); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + let independent_lz = rars::codec::rar29::unpack29_encode_literals(entries[1].data).unwrap(); + assert!(files[1].pack_size < independent_lz.len() as u64); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); + } +} + +#[test] +fn writes_solid_compressed_rar20_archive_that_reader_extracts() { + let shared = b"RAR 2.0 solid writer shared dictionary phrase alpha beta gamma.\n"; + let first_data = shared.repeat(64); + let mut second_data = shared.repeat(32); + second_data.extend_from_slice(b"second member tail\n"); + let entries = [ + FileEntry { + name: b"solid-rar20-one.txt", + data: &first_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-rar20-two.txt", + data: &second_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, features), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 2); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + assert!(files.iter().all(|file| file.unp_ver == 20)); + assert!(files[1].pack_size < files[0].pack_size); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, first_data); + assert_eq!(extracted[1].data, second_data); +} + +#[test] +fn writes_solid_rar20_archive_across_table_boundary_and_multiple_history_members() { + let first_data: Vec<_> = (0u8..=255).cycle().take(4096).collect(); + let phrase = b"RAR 2.0 solid table boundary phrase alpha beta gamma.\n"; + let second_data = phrase.repeat(96); + let mut third_data = phrase.repeat(24); + third_data.extend_from_slice(b"third member literal tail after history matches\n"); + let entries = [ + FileEntry { + name: b"solid-rar20-table.bin", + data: &first_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-rar20-boundary.txt", + data: &second_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-rar20-third.txt", + data: &third_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar20, features), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 3); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + assert!(files[2].is_solid()); + assert!(files.iter().all(|file| file.unp_ver == 20)); + + let independent_third = write_compressed_archive( + &[entries[2]], + WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()), + ) + .unwrap(); + let independent_third = Archive::parse(&independent_third).unwrap(); + let independent_third = independent_third.files().next().unwrap(); + assert!(files[2].pack_size < independent_third.pack_size); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 3); + assert_eq!(extracted[0].data, first_data); + assert_eq!(extracted[1].data, second_data); + assert_eq!(extracted[2].data, third_data); +} + +#[test] +fn compressed_rar29_writer_stores_incompressible_member_when_smaller() { + let mut state = 0x1234_5678u32; + let data: Vec<_> = (0..8192) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let entries = [FileEntry { + name: b"randomish.bin", + data: &data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30); + assert_eq!(file.pack_size, data.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); +} + +#[test] +fn solid_rar29_writer_stores_incompressible_member_and_resets_solid_run() { + let first_data = b"solid reset phrase alpha beta gamma ".repeat(96); + let mut state = 0x2468_ace0u32; + let randomish: Vec<_> = (0..8192) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let second_data = b"solid reset phrase alpha beta gamma ".repeat(64); + let entries = [ + FileEntry { + name: b"solid-before.txt", + data: &first_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-random.bin", + data: &randomish, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-after.txt", + data: &second_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 3); + assert_eq!(files[1].method, 0x30); + assert_eq!(files[1].pack_size, randomish.len() as u64); + assert!(!files[0].is_solid()); + assert!(!files[1].is_solid()); + assert!(!files[2].is_solid()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first_data); + assert_eq!(extracted[1].data, randomish); + assert_eq!(extracted[2].data, second_data); +} + +#[test] +fn solid_rar29_explicit_filter_codes_members_into_one_chain() { + let first_data = b"solid filtered phrase alpha beta gamma ".repeat(96); + let mut state = 0x1357_9bdfu32; + let randomish: Vec<_> = (0..8192) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let second_data = b"solid filtered phrase alpha beta gamma ".repeat(64); + let entries = [ + FileEntry { + name: b"solid-filter-before.txt", + data: &first_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-filter-random.bin", + data: &randomish, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"solid-filter-after.txt", + data: &second_data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::E8)), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 3); + // The middle member does not compress, so it is stored, exactly as it + // would be without a filter. That rebuilds the encoder, so the chain ends + // there and the member after it starts a fresh one rather than claiming a + // history the decoder will not have. + assert_eq!(files[1].method, 0x30); + assert!(!files[0].is_solid()); + assert!(!files[1].is_solid()); + assert!(!files[2].is_solid()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first_data); + assert_eq!(extracted[1].data, randomish); + assert_eq!(extracted[2].data, second_data); +} + +/// `--no-filter` reaches the writer as a filter policy, and only an explicit +/// filter was routed through the solid chain, so asking for no filter used to +/// cost the whole benefit of `--solid` while the members still claimed to +/// continue a chain that was never built. +#[test] +fn a_solid_chain_survives_asking_for_no_filter() { + let mut state = 0x51ed_2701u32; + let payload: Vec = (0..900) + .flat_map(|index| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + format!("line {index:05} {state:08x} unfiltered body\n").into_bytes() + }) + .collect(); + let entry = |name: &'static [u8]| FileEntry { + name, + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let entries = [entry(b"first.bin"), entry(b"second.bin")]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterPolicy::None, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(files[1].is_solid()); + assert!( + files[1].pack_size * 8 < files[0].pack_size, + "the second member has to reach the chain, got {} against {}", + files[1].pack_size, + files[0].pack_size + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); + assert_eq!(extracted[1].data, payload); +} + +/// An explicit filter over a zero-byte member asks for a 0..0 range, which the +/// codec refuses. One empty file in a directory used to compress every other +/// member and then fail at the last step, writing no archive at all. +#[test] +fn an_empty_member_is_written_under_every_explicit_filter() { + let payload = b"filter me \xe8\x00\x00\x00\x00 and me too\n".repeat(64); + for kind in [ + FilterKind::E8, + FilterKind::E8E9, + FilterKind::Delta { channels: 1 }, + FilterKind::Itanium, + FilterKind::Rgb { width: 3, pos_r: 0 }, + FilterKind::Audio { channels: 2 }, + ] { + let entries = [ + FileEntry { + name: b"empty.bin", + data: b"", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"payload.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterPolicy::Explicit(FilterSpec::whole(kind)), + ) + .unwrap_or_else(|err| panic!("{kind:?}: {err}")); + + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, b""); + assert_eq!(extracted[1].data, payload); + } +} + +/// The empty member must go through the chain's encoder like any other. Storing +/// it ends the chain in the headers while leaving the encoder's history in +/// place, so the member after it decodes against a dictionary its own flags +/// told the reader to discard. +#[test] +fn an_empty_member_keeps_a_filtered_solid_chain_intact() { + // Low internal repetition, so the first member packs large and a repeat of + // it can only be small if the chain is genuinely reaching back. + let mut state = 0x2545_f491u32; + let payload: Vec = (0..900) + .flat_map(|index| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let mut record = format!("record {index:05} {state:08x} ").into_bytes(); + // An x86 call for the filter to find, so the filter is doing work + // rather than being a no-op the chain could hide. + record.extend_from_slice(&[0xe8, 0, 0, 0, 0, b'\n']); + record + }) + .collect(); + let entries = [ + FileEntry { + name: b"first.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"empty.bin", + data: b"", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"second.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::E8)), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!( + files[1].is_solid(), + "the empty member must not end the chain" + ); + assert!(files[2].is_solid(), "the member after it must continue it"); + assert!( + files[2].pack_size * 8 < files[0].pack_size, + "a repeat across the empty member should still reach the chain, \ + got {} against {}", + files[2].pack_size, + files[0].pack_size + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); + assert_eq!(extracted[1].data, b""); + assert_eq!(extracted[2].data, payload); +} + +/// A filtered member used to be coded on its own whatever the archive said, +/// while its header still claimed to continue the chain. +#[test] +fn a_solid_rar29_chain_survives_a_filter_and_stays_solid() { + let member = |seed: u32, len: usize| -> Vec { + let mut state = seed; + let mut out = Vec::with_capacity(len); + while out.len() < len { + state = state.wrapping_mul(1_103_515_245).wrapping_add(12_345); + if out.len() % 64 == 0 { + out.push(0xe8); + out.extend_from_slice(&(state >> 8).to_le_bytes()[..3]); + } else { + out.push((state >> 16) as u8); + } + } + out.truncate(len); + out + }; + let first = member(1, 120_000); + // The second member repeats the first, so only a real chain can pack it + // into almost nothing. + let second = first.clone(); + let entries = [ + FileEntry { + name: b"chain-one.bin", + data: &first, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"chain-two.bin", + data: &second, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features), + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::E8)), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!( + files[1].is_solid(), + "the second member must continue the chain" + ); + assert!( + files[1].pack_size * 8 < files[0].pack_size, + "a repeat of the first member should pack to almost nothing, \ + got {} against {}", + files[1].pack_size, + files[0].pack_size + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); +} + +/// Calls to a handful of fixed addresses, which is what the x86 filters are +/// for. An x86 call operand is relative to the instruction after it, so the +/// same target reads differently at every call site; converting to absolute +/// leaves three values the match finder eats. Filler in between keeps the +/// member from being one long repeat and differs per seed, so the second +/// member of the chain is not simply a copy of the first. +fn call_heavy_x86(seed: u32, calls: usize) -> Vec { + const TARGETS: [u32; 3] = [0x1000, 0x2400, 0x3800]; + let mut data = Vec::with_capacity(calls * 16); + let mut state = seed | 1; + for index in 0..calls { + let target = TARGETS[index % TARGETS.len()]; + let relative = target.wrapping_sub(data.len() as u32 + 5); + data.push(0xe8); + data.extend_from_slice(&relative.to_le_bytes()); + for _ in 0..11 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + data.push((state >> 24) as u8 | 0x40); + } + } + data +} + +fn solid_rar29_entries<'a>(first: &'a [u8], second: &'a [u8]) -> [FileEntry<'a>; 2] { + [ + FileEntry { + name: b"searched-first.bin", + data: first, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"searched-second.bin", + data: second, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ] +} + +/// A solid archive used to refuse the search outright, on the grounds that a +/// candidate can only be measured against the history behind it. It can be: +/// the chain codes each one for real and keeps the smallest. +#[test] +fn a_solid_rar29_chain_searches_for_a_filter() { + let first = call_heavy_x86(0x1234_5678, 400); + let second = call_heavy_x86(0x2bad_c0de, 400); + let entries = solid_rar29_entries(&first, &second); + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = WriterOptions::new(ArchiveVersion::Rar29, features); + + let searched = + write_rar29_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::Auto) + .unwrap(); + let plain = + write_rar29_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::None) + .unwrap(); + + assert!( + searched.len() < plain.len(), + "the search should have found the x86 filter, got {} bytes against {} unfiltered", + searched.len(), + plain.len() + ); + + let archive = Archive::parse(&searched).unwrap(); + assert!(archive.main.is_solid()); + let files: Vec<_> = archive.files().collect(); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); +} + +/// The unfiltered member is one of the candidates, so the search cannot make a +/// member larger however unpromising its bytes are. +#[test] +fn a_solid_rar29_search_never_loses_to_leaving_the_bytes_alone() { + let mut state = 0x8765_4321u32; + let noise: Vec = (0..40_000) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + (state >> 24) as u8 + }) + .collect(); + let entries = solid_rar29_entries(&noise, &noise); + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = WriterOptions::new(ArchiveVersion::Rar29, features); + + let searched = + write_rar29_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::Auto) + .unwrap(); + let plain = + write_rar29_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::None) + .unwrap(); + + assert!( + searched.len() <= plain.len(), + "the search wrote {} bytes where leaving the data alone wrote {}", + searched.len(), + plain.len() + ); + let archive = Archive::parse(&searched).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, noise); + assert_eq!(extracted[1].data, noise); +} + +#[test] +fn auto_filtered_rar29_writer_stores_incompressible_member_when_smaller() { + let mut state = 0x8765_4321u32; + let data: Vec<_> = (0..8192) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let entries = [FileEntry { + name: b"auto-randomish.bin", + data: &data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterPolicy::Auto, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30); + assert_eq!(file.pack_size, data.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); +} + +#[test] +fn auto_filtered_rar29_writer_chooses_ppmd_for_text_when_smaller() { + let mut payload = Vec::new(); + for index in 0..512 { + payload.extend_from_slice(b"rar29 auto ppmd text with repeated words and punctuation. "); + payload.extend_from_slice( + format!("line {index:04}: alpha beta gamma alpha beta gamma\n").as_bytes(), + ); + } + let lz_packed = rars::codec::rar29::unpack29_encode_literals(&payload).unwrap(); + let ppmd_packed = rars::codec::rar29::unpack29_encode_ppmd(&payload, RAR29_DICTIONARY).unwrap(); + assert!( + ppmd_packed.len() < lz_packed.len(), + "fixture must exercise the auto-policy PPMd candidate" + ); + + let entries = [FileEntry { + name: b"auto-ppmd.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterPolicy::Auto, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + // The method byte is the level asked for, here the default 0x33. PPMd is + // signalled in the stream, so the pack size against a known PPMd encode is + // what shows which engine answered. + assert_eq!(file.method, 0x33); + assert_eq!(file.pack_size, ppmd_packed.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn default_rar29_writer_uses_auto_policy_for_text() { + let payload = b"rar29 default auto text alpha beta gamma alpha beta gamma\n".repeat(512); + let entries = [FileEntry { + name: b"default-auto-text.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let ppmd_packed = rars::codec::rar29::unpack29_encode_ppmd(&payload, RAR29_DICTIONARY).unwrap(); + + // The method byte is the level asked for, here the default 0x33. PPMd is + // signalled in the stream, so the pack size against a known PPMd encode is + // what shows which engine answered. + assert_eq!(file.method, 0x33); + assert_eq!(file.pack_size, ppmd_packed.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn auto_filtered_rar29_writer_improves_x86_relative_calls() { + let mut payload = Vec::new(); + payload.extend((0..2048).map(|index| (index * 37 + 11) as u8)); + let code_start = payload.len(); + let call_target = code_start + 0x1800; + for index in 0..512usize { + payload.extend_from_slice(&[0x55, 0x8b, 0xec, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = payload.len(); + payload.push(0xe8); + let next = call_pos + 5; + let relative = (call_target as i64 - next as i64) as i32; + payload.extend_from_slice(&relative.to_le_bytes()); + payload.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5d, 0xc3]); + } + payload.extend((0..2048).map(|index| (index * 53 + 7) as u8)); + let entries = [FileEntry { + name: b"x86-calls.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let plain_packed = rars::codec::rar29::unpack29_encode_literals(&payload).unwrap(); + let auto = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterPolicy::Auto, + ) + .unwrap(); + let auto = Archive::parse(&auto).unwrap(); + let auto_file = auto.files().next().unwrap(); + + assert_eq!(auto_file.method, 0x33); + assert!( + auto_file.pack_size * 2 < plain_packed.len() as u64, + "auto-filtered x86 payload should be much smaller than plain RAR29 LZ" + ); + assert_eq!(collect_extract(&auto).unwrap()[0].data, payload); +} + +#[test] +fn default_rar29_writer_uses_auto_policy_for_x86() { + let mut payload = Vec::new(); + payload.extend((0..2048).map(|index| (index * 37 + 11) as u8)); + let code_start = payload.len(); + let call_target = code_start + 0x1800; + for index in 0..512usize { + payload.extend_from_slice(&[0x55, 0x8b, 0xec, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = payload.len(); + payload.push(0xe8); + let next = call_pos + 5; + let relative = (call_target as i64 - next as i64) as i32; + payload.extend_from_slice(&relative.to_le_bytes()); + payload.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5d, 0xc3]); + } + payload.extend((0..2048).map(|index| (index * 53 + 7) as u8)); + let entries = [FileEntry { + name: b"default-auto-x86.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let default = write_compressed_archive(&entries, options).unwrap(); + let explicit_auto = + write_rar29_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::Auto) + .unwrap(); + let default_archive = Archive::parse(&default).unwrap(); + let auto_archive = Archive::parse(&explicit_auto).unwrap(); + + assert_eq!(default, explicit_auto); + assert_eq!(default_archive.files().next().unwrap().method, 0x33); + assert_eq!(collect_extract(&auto_archive).unwrap()[0].data, payload); +} + +#[test] +fn rar29_family_writer_levels_increase_lz_match_effort() { + let payload = level_sensitive_payload(); + let entries = [FileEntry { + name: b"rar29-level-effort.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let level_one = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(1), + ) + .unwrap(); + let level_three = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(3), + ) + .unwrap(); + let level_one_archive = Archive::parse(&level_one).unwrap(); + let level_three_archive = Archive::parse(&level_three).unwrap(); + let level_one_file = level_one_archive.files().next().unwrap(); + let level_three_file = level_three_archive.files().next().unwrap(); + + assert_eq!(level_one_file.method, 0x31, "{target:?} level 1"); + assert_eq!(level_three_file.method, 0x33, "{target:?} level 3"); + assert!( + level_three_file.pack_size < level_one_file.pack_size, + "{target:?}" + ); + assert_eq!( + collect_extract(&level_one_archive).unwrap()[0].data, + payload + ); + assert_eq!( + collect_extract(&level_three_archive).unwrap()[0].data, + payload + ); + } +} + +/// The declared dictionary is fitted to the member rather than fixed per +/// target: the smallest size the format encodes that reaches past the content, +/// floored at what the target's decoders expect. The pairs below are what +/// WinRAR declares for members of that size in our own fixtures. +/// +/// A fixed default was wrong in both directions. 128K lost half the ratio on +/// content whose repeats sit further apart than that, and RAR 2.9's old 1M +/// default made a reader allocate a megabyte to unpack thirty bytes. +#[test] +fn the_writer_fits_the_dictionary_to_the_member() { + for (content, expected) in [ + (26usize, 128 * 1024usize), + (130_048, 128 * 1024), + (196_608, 256 * 1024), + (262_144, 512 * 1024), + (705_644, 1024 * 1024), + ] { + let data = vec![b'z'; content]; + let entries = [StoredEntry { + name: b"fitted-dict.bin", + data: &data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let expected_flags = ((expected.trailing_zeros() as u16) - 16) << 5; + + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let bytes = write_stored_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(0), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30, "{target:?}"); + assert_eq!( + file.block.flags & 0x00e0, + expected_flags, + "{target:?} declared the wrong dictionary for {content} bytes" + ); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); + } + } +} + +#[test] +fn rar29_family_writer_stamps_requested_dictionary_size() { + let entries = [FileEntry { + name: b"dictionary.txt", + data: b"dictionary override payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + for (size, flags) in [(2 * 1024 * 1024, 0x00a0), (4 * 1024 * 1024, 0x00c0)] { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_compression_level(3) + .with_dictionary_size(size), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.block.flags & 0x00e0, flags); + assert_eq!(collect_extract(&archive).unwrap()[0].data, entries[0].data); + } +} + +#[test] +fn rar29_family_compressed_writer_level_zero_stores_member() { + let payload = b"level zero should store even through compressed writer\n".repeat(8); + let entries = [FileEntry { + name: b"level-zero.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(0), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30, "{target:?}"); + assert_eq!(file.pack_size, payload.len() as u64, "{target:?}"); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } +} + +#[test] +fn rar29_family_writer_stamps_requested_lz_method_levels() { + let payload = b"level method stamp payload alpha beta gamma delta\n".repeat(128); + let entries = [FileEntry { + name: b"method-level.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for level in 1..=5 { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(level), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30 + level, "{target:?} level {level}"); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } + } +} + +#[test] +fn rar29_family_writer_stores_small_incompressible_member_when_smaller() { + let data: Vec<_> = (0..41u8) + .map(|index| index.wrapping_mul(37).wrapping_add(11)) + .collect(); + let entries = [FileEntry { + name: b"fgrep", + data: &data, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar40, FeatureSet::store_only()) + .with_compression_level(5), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x30); + assert_eq!(file.pack_size, data.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); +} + +#[test] +fn rar29_family_writer_level_three_uses_lz_and_level_five_uses_auto_policy() { + let payload = b"fn alpha() { beta(gamma); }\nlet words = alpha beta gamma delta;\n".repeat(512); + let entries = [FileEntry { + name: b"rar29-level-policy.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let level_three = + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(3); + let level_five = + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(5); + + let lz = write_compressed_archive(&entries, level_three).unwrap(); + let auto = write_compressed_archive(&entries, level_five).unwrap(); + let lz_archive = Archive::parse(&lz).unwrap(); + let auto_archive = Archive::parse(&auto).unwrap(); + let lz_file = lz_archive.files().next().unwrap(); + let auto_file = auto_archive.files().next().unwrap(); + + assert_eq!(lz_file.method, 0x33, "{target:?} level 3"); + assert!( + matches!(auto_file.method, 0x33 | 0x35), + "{target:?} level 5" + ); + assert!(auto_file.pack_size <= lz_file.pack_size, "{target:?}"); + assert_eq!(collect_extract(&lz_archive).unwrap()[0].data, payload); + assert_eq!(collect_extract(&auto_archive).unwrap()[0].data, payload); + } +} + +#[test] +fn rar29_family_writer_middle_levels_use_lz_filter_policy_without_ppmd() { + let mut payload = Vec::new(); + payload.extend((0..2048).map(|index| (index * 37 + 11) as u8)); + let code_start = payload.len(); + let call_target = code_start + 0x1800; + for index in 0..512usize { + payload.extend_from_slice(&[0x55, 0x8b, 0xec, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = payload.len(); + payload.push(0xe8); + let next = call_pos + 5; + let relative = (call_target as i64 - next as i64) as i32; + payload.extend_from_slice(&relative.to_le_bytes()); + payload.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5d, 0xc3]); + } + payload.extend((0..2048).map(|index| (index * 53 + 7) as u8)); + let entries = [FileEntry { + name: b"rar29-level-four-x86.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let plain_lz_size = rars::codec::rar29::unpack29_encode_literals(&payload) + .unwrap() + .len() as u64; + + for (level, expected_method) in [(1, 0x31), (2, 0x32), (3, 0x33), (4, 0x34)] { + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(level), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, expected_method, "{target:?} level {level}"); + assert!(file.pack_size < plain_lz_size, "{target:?} level {level}"); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + } + } +} + +#[test] +fn rar29_family_level_four_stamps_method_and_round_trips_lazy_payload() { + let payload = b"abcdXbcdYYYYYYYYYYYYabcdYYYYYYYYYYYY".repeat(64); + let entries = [FileEntry { + name: b"lazy-lz.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for target in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let level_three = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(3), + ) + .unwrap(); + let level_four = write_compressed_archive( + &entries, + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(4), + ) + .unwrap(); + let level_three_archive = Archive::parse(&level_three).unwrap(); + let level_four_archive = Archive::parse(&level_four).unwrap(); + let level_three_file = level_three_archive.files().next().unwrap(); + let level_four_file = level_four_archive.files().next().unwrap(); + + assert_eq!(level_three_file.method, 0x33, "{target:?}"); + assert_eq!(level_four_file.method, 0x34, "{target:?}"); + assert_eq!( + collect_extract(&level_four_archive).unwrap()[0].data, + payload + ); + } +} + +#[test] +fn default_rar29_writer_uses_auto_policy_for_audio_shaped_data() { + let mut payload = Vec::new(); + for sample in 0..16_384i16 { + let left = sample.wrapping_mul(3).wrapping_add(200); + let right = sample.wrapping_mul(3).wrapping_sub(200); + payload.extend_from_slice(&left.to_le_bytes()); + payload.extend_from_slice(&right.to_le_bytes()); + } + let entries = [FileEntry { + name: b"default-auto-audio.wav", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let default = write_compressed_archive(&entries, options).unwrap(); + let explicit_audio = write_rar29_compressed_archive_with_filter_policy( + &entries, + options, + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::Audio { channels: 4 })), + ) + .unwrap(); + let ppmd = write_rar29_compressed_archive_with_filter_policy( + &entries, + options.with_method(Rar29Method::Ppmd), + FilterPolicy::None, + ) + .unwrap(); + let default_archive = Archive::parse(&default).unwrap(); + let audio_archive = Archive::parse(&explicit_audio).unwrap(); + let ppmd_archive = Archive::parse(&ppmd).unwrap(); + let default_file = default_archive.files().next().unwrap(); + let audio_file = audio_archive.files().next().unwrap(); + let ppmd_file = ppmd_archive.files().next().unwrap(); + + assert_eq!(default_file.method, 0x33); + assert!(default_file.pack_size <= audio_file.pack_size); + assert!(default_file.pack_size < ppmd_file.pack_size); + assert_eq!(collect_extract(&default_archive).unwrap()[0].data, payload); +} + +#[test] +fn auto_filtered_rar29_writer_spans_separated_x86_call_clusters() { + let mut payload = Vec::new(); + payload.extend((0..12_000).map(|index| (index * 37 + 11) as u8)); + let code_start = payload.len(); + let call_target = code_start + 0x5000; + for index in 0..128usize { + payload.extend_from_slice(&[0x55, 0x8b, 0xec, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = payload.len(); + payload.push(0xe8); + let next = call_pos + 5; + let relative = (call_target as i64 - next as i64) as i32; + payload.extend_from_slice(&relative.to_le_bytes()); + payload.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5d, 0xc3]); + } + payload.extend((0..8192).map(|index| (index * 17 + 5) as u8)); + for index in 0..128usize { + payload.extend_from_slice(&[0x56, 0x8b, 0xf1, 0x83, 0xec, (index & 0x7f) as u8]); + let call_pos = payload.len(); + payload.push(0xe8); + let next = call_pos + 5; + let relative = (call_target as i64 - next as i64) as i32; + payload.extend_from_slice(&relative.to_le_bytes()); + payload.extend_from_slice(&[0x83, 0xc4, 0x04, 0x5e, 0xc3]); + } + let code_end = payload.len(); + payload.extend((0..12_000).map(|index| (index * 53 + 7) as u8)); + let entries = [FileEntry { + name: b"x86-call-clusters.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let auto = + write_rar29_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::Auto) + .unwrap(); + let span = write_rar29_compressed_archive_with_filter_policy( + &entries, + options, + FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8, code_start..code_end)), + ) + .unwrap(); + let auto = Archive::parse(&auto).unwrap(); + let span = Archive::parse(&span).unwrap(); + let auto_file = auto.files().next().unwrap(); + let span_file = span.files().next().unwrap(); + + assert_eq!(auto_file.method, 0x33); + assert!( + auto_file.pack_size <= span_file.pack_size, + "auto filter should consider the whole code-section span" + ); + assert_eq!(collect_extract(&auto).unwrap()[0].data, payload); +} + +/// The method byte names the level, not the engine. It used to be stamped 0x35 +/// for every PPMd payload, which was only ever right at level 5: WinRAR 3.00 +/// writes 0x34 on the PPMd archive it produces at -m4, and RAR 2.9 signals the +/// engine inside the stream, which is why our decoder takes no method byte. +#[test] +fn ppmd_rar29_writer_stamps_the_level_and_still_reads_back_as_ppmd() { + let mut payload = b"rar29 ppmd writer text alpha beta gamma delta\n".repeat(128); + payload.extend_from_slice(&[2, 2, 2, b'p', b'p', b'm', b'd']); + let entries = [FileEntry { + name: b"rar29-ppmd.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Ppmd), + FilterPolicy::None, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + let ppmd_packed = rars::codec::rar29::unpack29_encode_ppmd(&payload, RAR29_DICTIONARY).unwrap(); + + assert_eq!(file.method, 0x33, "the default level, not the engine"); + assert_eq!( + file.pack_size, + ppmd_packed.len() as u64, + "and the payload is still PPMd" + ); + assert_eq!(file.unp_ver, 29); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + + let default_route = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Ppmd), + ) + .unwrap(); + assert_eq!(default_route, bytes); +} + +#[test] +fn ppmd_rar29_writer_uses_period_compatible_lz_escapes_for_repeated_data() { + let phrase = b"rar29 ppmd writer repeated distance phrase "; + let mut payload = b"seed " + .iter() + .copied() + .chain(std::iter::repeat_n(b'Q', 512)) + .collect::>(); + payload.extend_from_slice(phrase); + payload.extend_from_slice(b"middle gap for offset greater than one "); + payload.extend_from_slice(phrase); + payload.extend_from_slice(phrase); + let entries = [FileEntry { + name: b"rar29-ppmd-lz.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let ppmd = write_rar29_compressed_archive_with_filter_policy( + &entries, + options.with_method(Rar29Method::Ppmd), + FilterPolicy::None, + ) + .unwrap(); + let codec_packed = + rars::codec::rar29::unpack29_encode_ppmd(&payload, RAR29_DICTIONARY).unwrap(); + let archive = Archive::parse(&ppmd).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x33); + assert_eq!(file.pack_size, codec_packed.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn ppmd_rar29_writer_embeds_vm_filter_record() { + let payload = b"\xe8\0\0\0\0rar29 ppmd embedded e8 filter payload\n".repeat(16); + let entries = [FileEntry { + name: b"rar29-ppmd-e8.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Ppmd), + FilterPolicy::explicit(FilterKind::E8), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.method, 0x33); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn solid_rar29_forced_ppmd_keeps_explicit_filters_self_contained() { + let first = b"\xe8\0\0\0\0first filtered ppmd payload\n".repeat(16); + let second = b"\xe8\0\0\0\0second filtered ppmd payload\n".repeat(16); + let entries = [ + FileEntry { + name: b"first", + data: &first, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"second", + data: &second, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, features).with_method(Rar29Method::Ppmd), + FilterPolicy::explicit(FilterKind::E8), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.is_solid()); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn writes_solid_compressed_rar15_archive_that_reader_extracts() { + let entries = [ + FileEntry { + name: b"one.txt", + data: b"shared prefix shared prefix shared prefix alpha\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + FileEntry { + name: b"two.txt", + data: b"shared prefix shared prefix shared prefix beta\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }, + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_compressed_archive( + &entries, + WriterOptions::new(ArchiveVersion::Rar15, features), + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 2); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); +} + +#[test] +fn writes_encrypted_rar15_archives_that_reader_extracts_with_password() { + let features = FeatureSet::store_only(); + + let stored = [StoredEntry { + name: b"secret-store.txt", + data: b"stored secret\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let stored_bytes = + write_stored_archive(&stored, WriterOptions::new(ArchiveVersion::Rar15, features)).unwrap(); + let stored_archive = Archive::parse(&stored_bytes).unwrap(); + let stored_file = stored_archive.files().next().unwrap(); + assert!(stored_file.is_encrypted()); + assert!(matches!( + collect_extract(&stored_archive), + Err(Error::NeedPassword) + )); + let extracted = collect_extract_with_password(&stored_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored[0].data); + + let compressed = [FileEntry { + name: b"secret-compressed.txt", + data: b"compressed secret compressed secret\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let compressed_bytes = write_compressed_archive( + &compressed, + WriterOptions::new(ArchiveVersion::Rar15, features), + ) + .unwrap(); + let compressed_archive = Archive::parse(&compressed_bytes).unwrap(); + let compressed_file = compressed_archive.files().next().unwrap(); + assert!(compressed_file.is_encrypted()); + assert!(matches!( + collect_extract_with_password(&compressed_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&compressed_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed[0].data); +} + +#[test] +fn writes_aes_encrypted_rar3_and_rar4_archives_that_reader_extracts_with_password() { + let features = FeatureSet::store_only(); + + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + let stored = [StoredEntry { + name: b"aes-store.txt", + data: b"stored aes secret\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let stored_bytes = write_stored_archive(&stored, WriterOptions::new(target, features)) + .unwrap_or_else(|error| panic!("{target:?} stored AES writer failed: {error}")); + let stored_archive = Archive::parse(&stored_bytes).unwrap(); + let stored_file = stored_archive.files().next().unwrap(); + assert!(stored_file.is_encrypted()); + assert_eq!(stored_file.unp_ver, 29); + assert!(stored_file.salt.is_some()); + assert_eq!(stored_file.pack_size % 16, 0); + assert!(matches!( + collect_extract(&stored_archive), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&stored_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&stored_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored[0].data); + + let compressed = [FileEntry { + name: b"aes-compressed.txt", + data: b"compressed aes secret compressed aes secret\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let compressed_bytes = + write_compressed_archive(&compressed, WriterOptions::new(target, features)) + .unwrap_or_else(|error| panic!("{target:?} compressed AES writer failed: {error}")); + let compressed_archive = Archive::parse(&compressed_bytes).unwrap(); + let compressed_file = compressed_archive.files().next().unwrap(); + assert!(compressed_file.is_encrypted()); + assert_eq!(compressed_file.unp_ver, 29); + assert!(compressed_file.salt.is_some()); + assert_eq!(compressed_file.pack_size % 16, 0); + assert!(matches!( + collect_extract(&compressed_archive), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&compressed_archive, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_with_password(&compressed_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed[0].data); + } +} + +#[test] +fn writes_header_encrypted_rar3_and_rar4_archives_that_reader_extracts_with_password() { + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let entries = [FileEntry { + name: b"header-secret.txt", + data: b"RAR 3.x header encrypted writer payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let bytes = + write_compressed_archive(&entries, WriterOptions::new(target, features)).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + assert!(archive.main.has_encrypted_headers()); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"header-secret.txt"); + assert!(files[0].is_encrypted()); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, entries[0].data); + } +} + +/// A reader that finds the headers encrypted expects every one of them to be +/// introduced by its own salt, so when the last member's payload ends it reads +/// eight more bytes looking for the next group. Ending the file there is a short +/// read rather than a clean end, and 7-Zip treats it as a failed decryption and +/// refuses to open the archive at all. unrar stops at the last member and never +/// looks, which is why every header-encrypted archive we wrote was unreadable in +/// 7-Zip and nothing said so. +#[test] +fn a_header_encrypted_archive_ends_with_an_end_of_archive_block() { + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let entries = [FileEntry { + name: b"header-secret.txt", + data: b"RAR 3.x header encrypted writer payload\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + let bytes = + write_compressed_archive(&entries, WriterOptions::new(target, features)).unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + + assert!( + matches!(archive.blocks.last(), Some(Block::End(_))), + "{target:?} archive does not end with an end-of-archive block" + ); + } +} + +#[test] +fn writes_solid_header_encrypted_rar3_and_rar4_archives_that_reader_extracts_with_password() { + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let entries = [ + FileEntry { + name: b"solid-header-one.txt", + data: b"solid header encrypted common prefix one one one\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + FileEntry { + name: b"solid-header-two.txt", + data: b"solid header encrypted common prefix two two two\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }, + ]; + let bytes = + write_compressed_archive(&entries, WriterOptions::new(target, features)).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + assert!(archive.main.has_encrypted_headers()); + assert!(archive.main.is_solid()); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + assert!(files.iter().all(|file| file.is_encrypted())); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, entries[0].data); + assert_eq!(extracted[1].data, entries[1].data); + } +} + +#[test] +fn rar3_and_rar4_aes_writer_uses_fresh_salts() { + let features = FeatureSet::store_only(); + let entry = [FileEntry { + name: b"aes-salt.txt", + data: b"same plaintext same password\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }]; + + let first = + write_compressed_archive(&entry, WriterOptions::new(ArchiveVersion::Rar30, features)) + .unwrap(); + let second = + write_compressed_archive(&entry, WriterOptions::new(ArchiveVersion::Rar30, features)) + .unwrap(); + let first_archive = Archive::parse(&first).unwrap(); + let second_archive = Archive::parse(&second).unwrap(); + let first_file = first_archive.files().next().unwrap(); + let second_file = second_archive.files().next().unwrap(); + + assert_ne!(first, second); + assert_ne!(first_file.salt, second_file.salt); + assert_eq!( + collect_extract_with_password(&first_archive, Some(b"password")).unwrap()[0].data, + entry[0].data + ); + assert_eq!( + collect_extract_with_password(&second_archive, Some(b"password")).unwrap()[0].data, + entry[0].data + ); +} + +#[test] +fn writes_stored_rar15_volume_set_that_reader_reassembles() { + let entry = StoredEntry { + name: b"split-store.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + + let parts = write_stored_volumes(entry, WriterOptions::default(), 10).unwrap(); + assert_eq!(parts.len(), 4); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives[0].main.is_volume()); + assert!(archives[0].main.is_first_volume()); + assert!(archives[0].files().next().unwrap().is_split_after()); + assert!(archives[1].files().next().unwrap().is_split_before()); + assert!(archives[1].files().next().unwrap().is_split_after()); + assert!(archives[3].files().next().unwrap().is_split_before()); + assert!(!archives[3].files().next().unwrap().is_split_after()); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, entry.name); + assert_eq!(extracted[0].data, entry.data); +} + +/// Compressing well enough to need only one volume used to be an error, so a +/// script with a fixed `--volume-size` could start failing because compression +/// improved. RAR 5 always wrote the single volume; the legacy writers refused. +#[test] +fn a_volume_set_that_fits_in_one_volume_is_written_not_refused() { + let entry = StoredEntry { + name: b"fits-in-one.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + + let parts = write_stored_volumes(entry, WriterOptions::default(), 4096).unwrap(); + assert_eq!(parts.len(), 1); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let file = archives[0].files().next().unwrap(); + // It is a volume set, of one, so the member is whole: nothing continues + // into a part that was never written. + assert!(archives[0].main.is_volume()); + assert!(archives[0].main.is_first_volume()); + assert!(!file.is_split_before()); + assert!(!file.is_split_after()); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, entry.data); +} + +#[test] +fn writes_compressed_rar15_volume_set_that_reader_reassembles() { + let entry = FileEntry { + name: b"split-compressed.txt", + data: b"abcabcabcabcabcabcabcabcabcabcabcabcabcabc", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + + let parts = write_compressed_volumes(entry, WriterOptions::default(), 8).unwrap(); + assert!(parts.len() >= 2); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives[0].main.is_volume()); + assert!(archives[0].main.is_first_volume()); + assert!(archives[0].files().next().unwrap().is_split_after()); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, entry.name); + assert_eq!(extracted[0].data, entry.data); +} + +#[test] +fn solid_compressed_volume_sets_keep_the_solid_main_flag() { + let data = b"solid split payload solid split payload ".repeat(32); + for (target, encrypted_headers) in [ + (ArchiveVersion::Rar20, false), + (ArchiveVersion::Rar29, false), + (ArchiveVersion::Rar30, true), + ] { + let mut features = FeatureSet::store_only(); + features.solid = true; + features.header_encryption = encrypted_headers; + let entry = FileEntry { + name: b"split-solid.txt", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: encrypted_headers.then_some(b"password".as_slice()), + file_comment: None, + }; + let parts = + write_compressed_volumes(entry, WriterOptions::new(target, features), 24).unwrap(); + assert!(parts.len() > 1, "{target:?}"); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, entry.password).unwrap()) + .collect(); + assert!( + archives.iter().all(|archive| archive.main.is_solid()), + "{target:?}" + ); + let extracted = collect_extract_volumes_with_password(&archives, entry.password).unwrap(); + assert_eq!(extracted[0].data, data, "{target:?}"); + } +} + +#[test] +fn compressed_rar29_volumes_keep_requested_dictionary_size() { + let data = b"dictionary override across split volumes ".repeat(16); + let entry = FileEntry { + name: b"dictionary.txt", + data: &data, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_dictionary_size(2 * 1024 * 1024); + let parts = write_compressed_volumes(entry, options, 24).unwrap(); + assert!(parts.len() > 1); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives + .iter() + .all(|archive| archive.files().next().unwrap().block.flags & 0x00e0 == 0x00a0)); + assert_eq!(collect_extract_volumes(&archives).unwrap()[0].data, data); +} + +#[test] +fn writes_compressed_rar20_volume_set_that_reader_reassembles() { + let entry = FileEntry { + name: b"split-rar20-compressed.txt", + data: &b"rar20 split compressed phrase alpha beta gamma "[..].repeat(32), + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let options = WriterOptions::new(ArchiveVersion::Rar20, FeatureSet::store_only()); + + let parts = write_compressed_volumes(entry, options, 8).unwrap(); + assert!(parts.len() >= 2); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives[0].main.is_volume()); + assert!(archives[0].main.is_first_volume()); + let first_file = archives[0].files().next().unwrap(); + assert_eq!(first_file.unp_ver, 20); + assert!(first_file.is_split_after()); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, entry.name); + assert_eq!(extracted[0].data, entry.data); +} + +#[test] +fn writes_compressed_rar29_volume_set_that_reader_reassembles() { + let entry = FileEntry { + name: b"split-rar29-compressed.txt", + data: &b"rar29 split compressed phrase alpha beta gamma "[..].repeat(32), + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + let parts = write_compressed_volumes(entry, options, 8).unwrap(); + assert!(parts.len() >= 2); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives[0].main.is_volume()); + assert!(archives[0].main.is_first_volume()); + let first_file = archives[0].files().next().unwrap(); + assert_eq!(first_file.unp_ver, 29); + assert!(first_file.is_split_after()); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, entry.name); + assert_eq!(extracted[0].data, entry.data); +} + +#[test] +fn writes_encrypted_rar15_volume_sets_that_reader_reassembles_with_password() { + let features = FeatureSet::store_only(); + let options = WriterOptions::new(ArchiveVersion::Rar15, features); + + let stored = StoredEntry { + name: b"split-secret-store.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let stored_parts = write_stored_volumes(stored, options, 24).unwrap(); + let stored_archives: Vec<_> = stored_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(stored_archives[0].files().next().unwrap().is_encrypted()); + assert!(matches!( + collect_extract_volumes(&stored_archives), + Err(Error::NeedPassword) + )); + let extracted = + collect_extract_volumes_with_password(&stored_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored.data); + + let compressed = FileEntry { + name: b"split-secret-compressed.txt", + data: b"secret compressed secret compressed secret compressed\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let compressed_parts = write_compressed_volumes(compressed, options, 24).unwrap(); + let compressed_archives: Vec<_> = compressed_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(compressed_archives[0] + .files() + .next() + .unwrap() + .is_encrypted()); + assert!(matches!( + collect_extract_volumes_with_password(&compressed_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&compressed_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed.data); +} + +#[test] +fn writes_encrypted_rar20_volume_sets_that_reader_reassembles_with_password() { + let features = FeatureSet::store_only(); + let options = WriterOptions::new(ArchiveVersion::Rar20, features); + + let stored = StoredEntry { + name: b"split-rar20-secret-store.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let stored_parts = write_stored_volumes(stored, options, 24).unwrap(); + let stored_archives: Vec<_> = stored_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first_stored = stored_archives[0].files().next().unwrap(); + assert_eq!(first_stored.unp_ver, 20); + assert!(first_stored.is_encrypted()); + assert!(matches!( + collect_extract_volumes(&stored_archives), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_volumes_with_password(&stored_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&stored_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored.data); + + let compressed = FileEntry { + name: b"split-rar20-secret-compressed.txt", + data: b"secret rar20 compressed secret rar20 compressed secret rar20 compressed\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let compressed_parts = write_compressed_volumes(compressed, options, 24).unwrap(); + let compressed_archives: Vec<_> = compressed_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first_compressed = compressed_archives[0].files().next().unwrap(); + assert_eq!(first_compressed.unp_ver, 20); + assert!(first_compressed.is_encrypted()); + assert!(matches!( + collect_extract_volumes_with_password(&compressed_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&compressed_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed.data); +} + +#[test] +fn writes_encrypted_rar29_volume_sets_that_reader_reassembles_with_password() { + let features = FeatureSet::store_only(); + let options = WriterOptions::new(ArchiveVersion::Rar29, features); + + let stored = StoredEntry { + name: b"split-rar29-secret-store.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let stored_parts = write_stored_volumes(stored, options, 24).unwrap(); + let stored_archives: Vec<_> = stored_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first_stored = stored_archives[0].files().next().unwrap(); + assert_eq!(first_stored.unp_ver, 29); + assert!(first_stored.is_encrypted()); + assert!(first_stored.salt.is_some()); + for archive in &stored_archives[1..] { + assert_eq!(archive.files().next().unwrap().salt, first_stored.salt); + } + assert!(matches!( + collect_extract_volumes(&stored_archives), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_volumes_with_password(&stored_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&stored_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored.data); + + let compressed = FileEntry { + name: b"split-rar29-secret-compressed.txt", + data: b"secret rar29 compressed secret rar29 compressed secret rar29 compressed\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let compressed_parts = write_compressed_volumes(compressed, options, 24).unwrap(); + let compressed_archives: Vec<_> = compressed_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first_compressed = compressed_archives[0].files().next().unwrap(); + assert_eq!(first_compressed.unp_ver, 29); + assert!(first_compressed.is_encrypted()); + assert!(first_compressed.salt.is_some()); + for archive in &compressed_archives[1..] { + assert_eq!(archive.files().next().unwrap().salt, first_compressed.salt); + } + assert!(matches!( + collect_extract_volumes_with_password(&compressed_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&compressed_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed.data); +} + +#[test] +fn writes_encrypted_rar3_and_rar4_volume_sets_that_reader_reassembles_with_password() { + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + assert_encrypted_rar3_volume_sets_round_trip(target); + } +} + +#[test] +fn writes_header_encrypted_rar3_and_rar4_volume_sets_that_reader_reassembles_with_password() { + for target in [ArchiveVersion::Rar30, ArchiveVersion::Rar40] { + assert_header_encrypted_rar3_volume_sets_round_trip(target); + } +} + +fn assert_encrypted_rar3_volume_sets_round_trip(target: ArchiveVersion) { + let features = FeatureSet::store_only(); + let options = WriterOptions::new(target, features); + + let stored = StoredEntry { + name: b"rar30-split-secret-store.bin", + data: b"abcdefghijklmnopqrstuvwxyz0123456789", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let stored_parts = write_stored_volumes(stored, options, 10).unwrap(); + let stored_archives: Vec<_> = stored_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first_stored = stored_archives[0].files().next().unwrap(); + assert!(first_stored.is_encrypted()); + assert!(first_stored.salt.is_some()); + assert_eq!( + first_stored.file_crc, + crc32(&first_stored.packed_data(&stored_archives[0]).unwrap()) + ); + for archive in &stored_archives[1..] { + assert_eq!(archive.files().next().unwrap().salt, first_stored.salt); + } + assert!(matches!( + collect_extract_volumes(&stored_archives), + Err(Error::NeedPassword) + )); + assert!(matches!( + collect_extract_volumes_with_password(&stored_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&stored_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored.data); + + let compressed = FileEntry { + name: b"rar30-split-secret-compressed.txt", + data: b"secret compressed secret compressed secret compressed\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let compressed_parts = write_compressed_volumes(compressed, options, 8).unwrap(); + let compressed_archives: Vec<_> = compressed_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first_compressed = compressed_archives[0].files().next().unwrap(); + assert!(first_compressed.is_encrypted()); + assert!(first_compressed.salt.is_some()); + assert_eq!( + first_compressed.file_crc, + crc32( + &first_compressed + .packed_data(&compressed_archives[0]) + .unwrap() + ) + ); + for archive in &compressed_archives[1..] { + assert_eq!(archive.files().next().unwrap().salt, first_compressed.salt); + } + assert!(matches!( + collect_extract_volumes_with_password(&compressed_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&compressed_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed.data); +} + +fn assert_header_encrypted_rar3_volume_sets_round_trip(target: ArchiveVersion) { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let options = WriterOptions::new(target, features); + + let stored = StoredEntry { + name: b"rar30-header-split-secret-store.bin", + data: b"header encrypted stored split data repeats repeats", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let stored_parts = write_stored_volumes(stored, options, 24).unwrap(); + assert!(matches!( + Archive::parse(&stored_parts[0]), + Err(Error::NeedPassword) + )); + let stored_archives: Vec<_> = stored_parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(stored_archives[0].main.has_encrypted_headers()); + assert!(stored_archives[0].files().next().unwrap().is_encrypted()); + assert!(matches!( + collect_extract_volumes_with_password(&stored_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&stored_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, stored.data); + + let compressed = FileEntry { + name: b"rar30-header-split-secret-compressed.txt", + data: b"header encrypted compressed split data repeats repeats repeats\n", + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: Some(b"password"), + file_comment: None, + }; + let compressed_parts = write_compressed_volumes(compressed, options, 24).unwrap(); + assert!(matches!( + Archive::parse(&compressed_parts[0]), + Err(Error::NeedPassword) + )); + let compressed_archives: Vec<_> = compressed_parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(compressed_archives[0].main.has_encrypted_headers()); + assert!(compressed_archives[0] + .files() + .next() + .unwrap() + .is_encrypted()); + assert!(matches!( + collect_extract_volumes_with_password(&compressed_archives, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let extracted = + collect_extract_volumes_with_password(&compressed_archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, compressed.data); +} + +#[test] +fn pins_rars_generated_rar15_writer_oracle_bytes() { + for (name, expected_len, expected_crc) in RARS_GENERATED_FIXTURE_BYTES { + let bytes = std::fs::read(fixture(&format!("rars_generated/{name}"))).unwrap(); + + assert_eq!(bytes.len(), *expected_len, "{name} length"); + assert_eq!(crc32(&bytes), *expected_crc, "{name} crc32"); + } +} + +#[test] +fn extracts_rars_generated_rar15_writer_oracles() { + for fixture_name in [ + "rars_generated/stored.rar", + "rars_generated/compressed.rar", + "rars_generated/solid.rar", + ] { + let bytes = std::fs::read(fixture(fixture_name)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted[0].name, b"payload.txt"); + assert_eq!(extracted[0].data, RARS_GENERATED_PAYLOAD); + if fixture_name.ends_with("solid.rar") { + assert_eq!(extracted[1].name, b"second.txt"); + assert_eq!(extracted[1].data, RARS_GENERATED_SECOND); + } + } +} + +/// Rebuilds `rars_generated/comments.rar` from the writer. +/// +/// The other oracles in that directory are pinned by size and CRC alone. This +/// one is pinned byte for byte, because the file comment inside it is the one +/// piece of the RAR 1.5 header layout the writer got wrong for six releases. +fn rars_generated_comments_archive() -> Vec { + let entries = [FileEntry { + name: b"payload.txt", + data: RARS_GENERATED_PAYLOAD, + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: Some(b"file-note"), + }]; + let mut options = WriterOptions::default(); + options.target = ArchiveVersion::Rar15; + write_compressed_archive_with_comment(&entries, options, Some(b"oracle-note")).unwrap() +} + +#[test] +fn extracts_rars_generated_rar15_comments_and_encryption_oracles() { + let comment_bytes = std::fs::read(fixture("rars_generated/comments.rar")).unwrap(); + let comment_archive = Archive::parse(&comment_bytes).unwrap(); + assert_eq!( + comment_archive.archive_comment().unwrap().as_deref(), + Some(&b"oracle-note"[..]) + ); + let comment_file = comment_archive.files().next().unwrap(); + assert_eq!( + comment_file.file_comment().unwrap().as_deref(), + Some(&b"file-note"[..]) + ); + assert_eq!( + collect_extract(&comment_archive).unwrap()[0].data, + RARS_GENERATED_PAYLOAD + ); + assert_eq!(comment_bytes, rars_generated_comments_archive()); + + let encrypted_bytes = std::fs::read(fixture("rars_generated/encrypted.rar")).unwrap(); + let encrypted_archive = Archive::parse(&encrypted_bytes).unwrap(); + assert!(matches!( + collect_extract(&encrypted_archive), + Err(Error::NeedPassword) + )); + assert_eq!( + collect_extract_with_password(&encrypted_archive, Some(b"pass")).unwrap()[0].data, + RARS_GENERATED_PAYLOAD + ); +} + +#[test] +fn extracts_rars_generated_rar15_writer_volume_oracles() { + let stored_archives: Vec<_> = [ + "rars_generated/split-store.rar", + "rars_generated/split-store.r00", + "rars_generated/split-store.r01", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + let stored = collect_extract_volumes(&stored_archives).unwrap(); + assert_eq!(stored[0].name, b"payload.txt"); + assert_eq!(stored[0].data, RARS_GENERATED_PAYLOAD); + + let encrypted_archives: Vec<_> = [ + "rars_generated/split-encrypted.rar", + "rars_generated/split-encrypted.r00", + "rars_generated/split-encrypted.r01", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + assert!(matches!( + collect_extract_volumes(&encrypted_archives), + Err(Error::NeedPassword) + )); + let encrypted = + collect_extract_volumes_with_password(&encrypted_archives, Some(b"pass")).unwrap(); + assert_eq!(encrypted[0].name, b"payload.txt"); + assert_eq!(encrypted[0].data, RARS_GENERATED_PAYLOAD); +} + +/// RAR 1.5 has no header encryption, and used to be told so only by a message +/// naming neither the option nor a format that does support it. +#[test] +fn rar15_writer_rejects_header_encryption_by_name() { + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let options = WriterOptions::new(ArchiveVersion::Rar15, features); + let entry = StoredEntry { + name: b"hello.txt", + data: b"hello", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }; + + let error = write_stored_archive(&[entry], options).unwrap_err(); + assert_eq!( + error.to_string(), + "header encryption is not supported by rar15" + ); + assert_eq!( + rars::formats_supporting( + rars::WriterOption::Feature(rars::Feature::HeaderEncryption), + rars::PlanShape::new(), + ), + vec![ + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] + ); +} + +#[test] +fn parses_rar300_comment_subblock_and_stored_file() { + let bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.sfx_offset, 0); + assert_eq!(archive.main.flags, 0); + + let subblocks: Vec<_> = archive.new_subs().collect(); + assert_eq!(subblocks.len(), 1); + assert_eq!(subblocks[0].kind, NewSubKind::ArchiveComment); + assert_eq!(subblocks[0].file.name, b"CMT"); + assert_eq!(subblocks[0].file.method, 0x33); + assert_eq!(subblocks[0].file.unp_ver, 29); + assert_eq!(subblocks[0].file.pack_size, 42); + assert_eq!(subblocks[0].file.unp_size, 29); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert!(files[0].is_stored()); + assert_eq!(files[0].method, 0x30); + assert_eq!(files[0].unp_ver, 29); + assert_eq!(files[0].pack_size, 30); + assert_eq!(files[0].unp_size, 30); + assert_eq!(files[0].file_crc, 0xa538535e); + assert_eq!( + files[0].packed_data(&archive).unwrap(), + b"Hello, RAR 3.x fixture world.\n" + ); +} + +#[test] +fn parses_rar202_main_header_with_embedded_comment_subblock() { + let bytes = std::fs::read(fixture("rar202/comment_nopsw.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.main.head_crc, 0x01bd); + assert_eq!(archive.main.head_size, 51); + assert!(archive.main.flags & 0x0002 != 0); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"FILE1.TXT"); + assert_eq!(files[1].name, b"FILE2.TXT"); + assert_eq!(files[0].unp_ver, 20); + assert_eq!(files[1].unp_ver, 20); + assert!(files.iter().all(|file| file.block.flags & 0x0008 != 0)); +} + +#[test] +fn reads_rar202_comments_written_by_winrar() { + let bytes = std::fs::read(fixture("rar202/comment_nopsw.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + // Nested in the main header and compressed with the RAR 2.0 codec, which + // is two reasons this one went unread until 0.7. + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(&b"RARcomment"[..]) + ); + + let files: Vec<_> = archive.files().collect(); + assert_eq!( + files[0].file_comment().unwrap().as_deref(), + Some(&b"file1comment"[..]) + ); + assert_eq!( + files[1].file_comment().unwrap().as_deref(), + Some(&b"file2comment"[..]) + ); +} + +#[test] +fn decodes_rar15_compressed_old_style_archive_comment() { + // Reuse a WinRAR 1.54 compressed member as comment data. The comment block + // carries the same unpack version and method but has a 16-bit output size. + let source = std::fs::read(fixture("rar154/readme_154_normal.rar")).unwrap(); + let source_archive = Archive::parse(&source).unwrap(); + let file = source_archive.files().next().unwrap(); + let packed = file.packed_data(&source_archive).unwrap(); + let expected = std::fs::read(fixture("rar154/expected/README.md")).unwrap(); + assert_eq!(file.unp_ver, 15); + assert_eq!(file.method, 0x33); + assert_eq!(file.unp_size as usize, expected.len()); + + let comment_head_size = 13 + packed.len(); + let main_head_size = 13 + comment_head_size; + let mut bytes = source[..7].to_vec(); // RAR 1.5 signature + let main_start = bytes.len(); + bytes.extend_from_slice(&[0, 0, 0x73]); // MAIN_HEAD + bytes.extend_from_slice(&0x0002u16.to_le_bytes()); // MHD_COMMENT + bytes.extend_from_slice(&(main_head_size as u16).to_le_bytes()); + bytes.extend_from_slice(&[0; 6]); // reserved main-header fields + let comment_start = bytes.len(); + bytes.extend_from_slice(&[0, 0, 0x75, 0, 0]); // COMM_HEAD, no flags + bytes.extend_from_slice(&(comment_head_size as u16).to_le_bytes()); + bytes.extend_from_slice(&(expected.len() as u16).to_le_bytes()); + bytes.extend_from_slice(&[file.unp_ver, file.method]); + bytes.extend_from_slice(&((crc32(&expected) & 0xffff) as u16).to_le_bytes()); + bytes.extend_from_slice(&packed); + let main_crc = (crc32(&bytes[main_start + 2..main_start + 13]) & 0xffff) as u16; + bytes[main_start..main_start + 2].copy_from_slice(&main_crc.to_le_bytes()); + let comment_crc = (crc32(&bytes[comment_start + 2..comment_start + 13]) & 0xffff) as u16; + bytes[comment_start..comment_start + 2].copy_from_slice(&comment_crc.to_le_bytes()); + + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.archive_comment().unwrap().unwrap(), expected); +} + +#[test] +fn a_written_file_comment_matches_the_block_winrar_writes() { + let winrar = std::fs::read(fixture("rar202/comment_nopsw.rar")).unwrap(); + let entries = [StoredEntry { + name: b"FILE1.TXT", + data: b"file1\r\n", + file_time: 0, + file_attr: 0x20, + host_os: 0, + password: None, + file_comment: Some(b"file1comment"), + }]; + let mut options = WriterOptions::default(); + options.target = ArchiveVersion::Rar20; + let bytes = write_stored_archive(&entries, options).unwrap(); + + // WinRAR 2.02 puts the block last in the header, after a 9 byte name. + let expected = &winrar[0x3a + 41..0x3a + 66]; + assert!( + bytes + .windows(expected.len()) + .any(|window| window == expected), + "written header has no comment block matching {expected:02x?}" + ); +} + +#[test] +fn an_encrypted_member_with_a_comment_keeps_its_header_checksum() { + // The comment sits outside the header CRC, the salt inside it. Reading the + // salt as part of the comment cost eight bytes of coverage and left an + // archive this reader would not parse. + for target in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ] { + let entries = [FileEntry { + name: b"secret.txt", + data: b"salted and commented\n", + file_time: 0, + file_attr: 0x20, + host_os: 3, + password: Some(b"pw"), + file_comment: Some(b"note"), + }]; + let mut options = WriterOptions::default(); + options.target = target; + let bytes = write_compressed_archive(&entries, options).unwrap(); + + let archive = Archive::parse(&bytes).unwrap_or_else(|err| panic!("{target:?}: {err}")); + let file = archive.files().next().unwrap(); + assert_eq!(file.file_comment().unwrap().as_deref(), Some(&b"note"[..])); + assert_eq!( + collect_extract_with_password(&archive, Some(b"pw")).unwrap()[0].data, + b"salted and commented\n" + ); + } +} + +#[test] +fn extracts_rar202_encrypted_files_with_rar20_cipher() { + let bytes = std::fs::read(fixture("rar202/comment_psw.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert!(files.iter().all(|file| file.is_encrypted())); + assert!(matches!( + collect_extract(&archive), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"FILE1.TXT"); + assert_eq!(extracted[0].data, b"file1\r\n"); + assert_eq!(extracted[1].name, b"FILE2.TXT"); + assert_eq!(extracted[1].data, b"file2\r\n"); + assert_eq!(crc32(&extracted[0].data), files[0].file_crc); + assert_eq!(crc32(&extracted[1].data), files[1].file_crc); +} + +#[test] +fn rejects_wrong_password_for_rar20_encrypted_file_as_password_error() { + let bytes = std::fs::read(fixture("rar202/comment_psw.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!( + collect_extract_with_password(&archive, Some(b"wrong-password")), + Err(Error::WrongPasswordOrCorruptData) + ); +} + +#[test] +fn header_encrypted_rar3_archive_requires_password_to_parse() { + let bytes = std::fs::read(fixture("encrypted/header_enc_1234.rar")).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); +} + +#[test] +fn header_encrypted_rar3_rejects_non_utf8_password_before_decryption() { + let bytes = std::fs::read(fixture("encrypted/header_enc_1234.rar")).unwrap(); + let expected = |result: Result| { + assert!(matches!( + result, + Err(Error::Rar30Crypto( + rars::crypto::rar30::Error::NonUtf8Password + )) + )); + }; + + expected(Archive::parse_with_password(&bytes, Some(b"\xff"))); + let root = scratch::case("non-utf8-rar3-password"); + let path = root.join("encrypted.rar"); + std::fs::write(&path, &bytes).unwrap(); + expected(Archive::parse_path_with_password(&path, Some(b"\xff"))); +} + +#[test] +fn extracts_rar300_header_encrypted_archive_with_password() { + let bytes = std::fs::read(fixture("encrypted/header_rar300_password.rar")).unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let file = archive.files().next().unwrap(); + + assert!(archive.main.has_encrypted_headers()); + assert_eq!(file.name, b"hello.txt"); + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); + assert_eq!(crc32(&extracted[0].data), 0xa538535e); +} + +#[test] +fn parses_rar300_header_encrypted_archive_from_path_with_password() { + let archive = Archive::parse_path_with_password( + fixture("encrypted/header_rar300_password.rar"), + Some(b"password"), + ) + .unwrap(); + let file = archive.files().next().unwrap(); + + assert!(archive.main.has_encrypted_headers()); + assert_eq!(file.name, b"hello.txt"); + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); +} + +#[test] +fn extracts_rar420_header_encrypted_archive_with_password() { + let bytes = std::fs::read(fixture("encrypted/header_rar420_password.rar")).unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let file = archive.files().next().unwrap(); + + assert!(archive.main.has_encrypted_headers()); + assert_eq!(file.name, b"hello.txt"); + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); + assert_eq!(crc32(&extracted[0].data), 0xa538535e); +} + +#[test] +fn parses_rar420_header_encrypted_archive_from_path_with_password() { + let archive = Archive::parse_path_with_password( + fixture("encrypted/header_rar420_password.rar"), + Some(b"password"), + ) + .unwrap(); + let file = archive.files().next().unwrap(); + + assert!(archive.main.has_encrypted_headers()); + assert_eq!(file.name, b"hello.txt"); + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); +} + +#[test] +fn extracts_rar300_aes_encrypted_file_with_password() { + let bytes = std::fs::read(fixture("encrypted/per_file_rar300_password.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"hello.txt"); + assert!(file.is_encrypted()); + assert_eq!(file.unp_ver, 29); + assert_eq!(file.method, 0x33); + assert_eq!( + file.salt, + Some([0x4a, 0x81, 0x67, 0x7d, 0xc0, 0x3d, 0x5f, 0x83]) + ); + assert!(matches!( + collect_extract(&archive), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"hello.txt"); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); + assert_eq!(crc32(&extracted[0].data), 0xa538535e); +} + +#[test] +fn rejects_wrong_password_for_rar3_encrypted_file_as_password_error() { + let bytes = std::fs::read(fixture("encrypted/per_file_rar300_password.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!( + collect_extract_with_password(&archive, Some(b"wrong-password")), + Err(Error::WrongPasswordOrCorruptData) + ); +} + +#[test] +fn extracts_rar4_aes_encrypted_compressed_member() { + let bytes = std::fs::read(fixture("encrypted/per_file_rar4_libarchive_mixed.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 4); + assert_eq!(files[1].name, b"b.txt"); + assert!(files[1].is_encrypted()); + assert_eq!(files[1].unp_ver, 29); + assert_eq!(files[1].method, 0x33); + + let data = collect_file_with_password(&archive, files[1], Some(b"password")) + .unwrap() + .data; + assert_eq!(data, b"This is from b.txt"); + assert_eq!(crc32(&data), 0xa9fa1485); +} + +#[test] +fn extracts_rar4_junrar_encrypted_member_with_correct_password() { + let bytes = std::fs::read(fixture("encrypted/rar4_junrar_password.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"file1.txt"); + assert!(file.is_encrypted()); + assert_eq!(file.method, 0x33); + assert_eq!(file.unp_ver, 29); + assert!(matches!( + collect_extract(&archive), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_with_password(&archive, Some(b"junrar")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"file1.txt"); + assert_eq!(extracted[0].data, b"file1\n"); + assert_eq!(crc32(&extracted[0].data), 0xe229f704); +} + +#[test] +fn rejects_wrong_password_for_rar4_encrypted_file_as_password_error() { + let bytes = std::fs::read(fixture("encrypted/rar4_junrar_password.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!( + collect_extract_with_password(&archive, Some(b"wrong-password")), + Err(Error::WrongPasswordOrCorruptData) + ); +} + +#[test] +fn extracts_rar4_junrar_header_encrypted_member_with_correct_password() { + let bytes = std::fs::read(fixture("encrypted/rar4_junrar_header_encrypted.rar")).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"junrar")).unwrap(); + assert!(archive.main.has_encrypted_headers()); + + let extracted = collect_extract_with_password(&archive, Some(b"junrar")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"file1.txt"); + assert_eq!(extracted[0].data, b"file1\n"); + assert_eq!(crc32(&extracted[0].data), 0xe229f704); +} + +#[test] +fn decodes_rar4_compact_unicode_name_before_extraction() { + let bytes = std::fs::read(fixture( + "encrypted/rar4_junrar_file_content_encrypted_unicode.rar", + )) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, "新建文本文档.txt".as_bytes()); + assert!(file.is_encrypted()); + + let extracted = collect_extract_with_password(&archive, Some(b"test")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, "新建文本文档.txt".as_bytes()); + assert_eq!(extracted[0].data, b"aaaaaaaaaa"); + assert_eq!(crc32(&extracted[0].data), 0x4c11cdf0); +} + +#[test] +fn extracts_rar4_sharpcompress_encrypted_files_only_archive() { + let bytes = std::fs::read(fixture("encrypted/rar4_sharpcompress_files_only.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 6); + assert_eq!(files[0].name, b"exe\\test.exe"); + assert_eq!(files[1].name, b"jpg\\test.jpg"); + assert_eq!(files[2].name, "те�т.txt".as_bytes()); + assert_eq!(files[3].name, b"Empty"); + assert_eq!(files[4].name, b"exe"); + assert_eq!(files[5].name, b"jpg"); + assert!(files[..3].iter().all(|file| file.is_encrypted())); + assert!(files[3..].iter().all(|file| file.is_directory())); + + let extracted = collect_extract_with_password(&archive, Some(b"test")).unwrap(); + assert_eq!(extracted.len(), 6); + assert_eq!(extracted[0].name, b"exe\\test.exe"); + assert_eq!(extracted[0].data.len(), 45056); + assert_eq!(crc32(&extracted[0].data), 0xcfb109c8); + assert_eq!(extracted[1].name, b"jpg\\test.jpg"); + assert_eq!(extracted[1].data.len(), 40372); + assert_eq!(crc32(&extracted[1].data), 0x088814e3); + assert_eq!(extracted[2].name, "те�т.txt".as_bytes()); + assert_eq!(extracted[2].data.len(), 15498); + assert_eq!(crc32(&extracted[2].data), 0x9bd160fa); + assert!(extracted[3..].iter().all(|entry| entry.is_directory)); +} + +#[test] +fn extracts_rar4_mixed_visible_names_known_password_fixture() { + let bytes = std::fs::read(fixture("encrypted/rar4_mixed_visible_names_password.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 3); + assert_eq!(files[0].name, b"1File.txt"); + assert_eq!(files[1].name, "2中文.txt".as_bytes()); + assert_eq!(files[2].name, b"3Sec.txt"); + assert!(!files[0].is_encrypted()); + assert!(files[1].is_encrypted()); + assert!(files[2].is_encrypted()); + + let stored = collect_file(&archive, files[0]).unwrap(); + assert_eq!(stored.data, b"1File"); + assert_eq!(crc32(&stored.data), 0x578a2019); + + assert!(matches!( + collect_extract(&archive), + Err(Error::NeedPassword) + )); + assert_eq!( + collect_extract_with_password(&archive, Some(b"wrong-password")), + Err(Error::WrongPasswordOrCorruptData) + ); + + let extracted = collect_extract_with_password(&archive, Some(b"known-pass")).unwrap(); + assert_eq!(extracted.len(), 3); + assert_eq!(extracted[0].name, b"1File.txt"); + assert_eq!(extracted[0].data, b"1File"); + assert_eq!(extracted[1].name, "2中文.txt".as_bytes()); + assert_eq!(extracted[1].data, b"known encrypted unicode payload\n"); + assert_eq!(crc32(&extracted[1].data), 0x1e180200); + assert_eq!(extracted[2].name, b"3Sec.txt"); + assert_eq!(extracted[2].data, b"known encrypted ascii payload\n"); + assert_eq!(crc32(&extracted[2].data), 0xbef64217); +} + +#[test] +fn extracts_rar300_stored_file_and_verifies_crc32() { + let bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"hello.txt"); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); + assert_eq!(extracted[0].host_os, 2); + assert!(!extracted[0].is_directory); + + let file = archive.files().next().unwrap(); + file.verify_crc32(&extracted[0].data).unwrap(); + assert_eq!(crc32(&extracted[0].data), 0xa538535e); +} + +#[test] +fn rejects_corrupt_rar15_40_header_checksum() { + let mut bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let main_flags_offset = 7 + 3; + bytes[main_flags_offset] ^= 0x01; + + assert!(matches!( + Archive::parse(&bytes), + Err(Error::CrcMismatch { .. }) + )); +} + +#[test] +fn rejects_corrupt_rar15_40_stored_payload_checksum() { + let mut bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let needle = b"Hello, RAR 3.x fixture world.\n"; + let offset = bytes + .windows(needle.len()) + .position(|window| window == needle) + .expect("stored payload in fixture"); + bytes[offset] ^= 0x01; + let archive = Archive::parse(&bytes).unwrap(); + + match collect_extract(&archive) { + Err(Error::Crc32Mismatch { .. }) => {} + Err(Error::AtEntry { source, .. }) if matches!(*source, Error::Crc32Mismatch { .. }) => {} + other => panic!("expected checksum error, got {other:?}"), + } +} + +#[test] +fn extracts_large_solid_rar300_with_reused_tables() { + let bytes = std::fs::read(fixture("rar300/solid_rar300.rar")).unwrap(); + let expected_big = std::fs::read(fixture("rar300/bigtext_64k.bin")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 3); + assert_eq!(files[0].name, b"hello.txt"); + assert_eq!(files[1].name, b"tiny.txt"); + assert_eq!(files[2].name, b"bigtext_64k.bin"); + assert_eq!(files[0].pack_size, 45); + assert_eq!(files[1].pack_size, 3); + assert_eq!(files[2].pack_size, 9_753); + assert_eq!(files[0].unp_size, 30); + assert_eq!(files[1].unp_size, 9); + assert_eq!(files[2].unp_size, 65_536); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + assert!(files[2].is_solid()); + assert!(files.iter().all(|file| file.method == 0x33)); + assert!(files.iter().all(|file| file.unp_ver == 29)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 3); + assert_eq!(extracted[0].name, b"hello.txt"); + assert_eq!(extracted[0].data, b"Hello, RAR 3.x fixture world.\n"); + assert_eq!(crc32(&extracted[0].data), 0xa538535e); + assert_eq!(extracted[1].name, b"tiny.txt"); + assert_eq!(extracted[1].data, b"AAAAAAAA\n"); + assert_eq!(crc32(&extracted[1].data), 0xd27b5891); + assert_eq!(extracted[2].name, b"bigtext_64k.bin"); + assert_eq!(extracted[2].data, expected_big); + assert_eq!(crc32(&extracted[2].data), 0xddc95682); +} + +#[test] +fn extracts_simple_solid_rar300_entries_with_codec_state() { + let bytes = std::fs::read(fixture("rar300/solid_simple_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"one.txt"); + assert_eq!( + extracted[0].data, + b"shared prefix shared prefix shared prefix alpha\n" + ); + assert_eq!(crc32(&extracted[0].data), 0x11cc9fbb); + assert_eq!(extracted[1].name, b"two.txt"); + assert_eq!( + extracted[1].data, + b"shared prefix shared prefix shared prefix beta\n" + ); + assert_eq!(crc32(&extracted[1].data), 0xf4fd09e8); +} + +#[test] +fn extracts_compressed_rar300_lz_file() { + let bytes = std::fs::read(fixture("rar300/compressed_text_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"text.txt"); + assert_eq!(extracted[0].data, expected_compressed_text_payload()); + assert_eq!(crc32(&extracted[0].data), 0x6a0d746d); +} + +#[test] +fn extracts_rar154_unp15_compressed_file() { + let bytes = std::fs::read(fixture("rar154/readme_154_normal.rar")).unwrap(); + let expected = std::fs::read(fixture("rar154/expected/README.md")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"README.md"); + assert_eq!(file.method, 0x33); + assert_eq!(file.unp_ver, 15); + assert_eq!(file.pack_size, 2068); + assert_eq!(file.unp_size, 4198); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"README.md"); + assert_eq!(extracted[0].data, expected); + assert_eq!(crc32(&extracted[0].data), 0x509e5e3c); +} + +#[test] +fn extracts_rar154_crypt15_encrypted_compressed_file() { + let bytes = std::fs::read(fixture("rar154/readme_154_password.rar")).unwrap(); + let expected = std::fs::read(fixture("rar154/expected/README.md")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"README.md"); + assert!(file.is_encrypted()); + assert_eq!(file.method, 0x33); + assert_eq!(file.unp_ver, 15); + assert!(matches!( + collect_extract(&archive), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"README.md"); + assert_eq!(extracted[0].data, expected); + assert_eq!(crc32(&extracted[0].data), 0x509e5e3c); +} + +#[test] +fn rejects_wrong_password_for_rar15_encrypted_file_as_password_error() { + let bytes = std::fs::read(fixture("rar154/readme_154_password.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!( + collect_extract_with_password(&archive, Some(b"wrong-password")), + Err(Error::WrongPasswordOrCorruptData) + ); +} + +#[test] +fn extracts_rar154_unp15_solid_flagged_file() { + let bytes = std::fs::read(fixture("rar154/readme_154_store_solid.rar")).unwrap(); + let expected = std::fs::read(fixture("rar154/expected/README.md")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert!(archive.main.is_solid()); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"README.md"); + assert_eq!(extracted[0].data, expected); +} + +#[test] +fn extracts_rar154_unp15_multi_file_archive() { + let bytes = std::fs::read(fixture("rar154/doc_154_best.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 17); + assert!(files.iter().all(|file| file.unp_ver == 15)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 17); + let expected = expected_doc_154_best_manifest(); + for (entry, (name, size, crc)) in extracted.iter().zip(expected) { + assert_eq!(entry.name, name.as_bytes()); + assert_eq!(entry.data.len(), size); + assert_eq!(crc32(&entry.data), crc, "{name}"); + } +} + +#[test] +fn extracts_rar154_unp15_audio_shaped_windows_archive() { + let bytes = std::fs::read(fixture("rar154/audio_win_names_unpack15.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"BoatModernEnglish.wav"); + assert_eq!(extracted[0].data.len(), 56_464); + assert_eq!(crc32(&extracted[0].data), 0x82d2ed89); + assert_eq!(extracted[1].name, b"LICENSE.txt"); + assert_eq!(extracted[1].data.len(), 107); + assert_eq!(crc32(&extracted[1].data), 0x8eaf20c4); +} + +#[test] +fn extracts_rar154_unp15_audio_shaped_dos_archive() { + let bytes = std::fs::read(fixture("rar154/audio_dos_names_unpack15.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"BOATMO~1.WAV"); + assert_eq!(extracted[0].data.len(), 56_464); + assert_eq!(crc32(&extracted[0].data), 0x82d2ed89); + assert_eq!(extracted[1].name, b"LICENSE.TXT"); + assert_eq!(extracted[1].data.len(), 107); + assert_eq!(crc32(&extracted[1].data), 0x8eaf20c4); +} + +#[test] +fn extracts_rar250_unp20_lz_file() { + let bytes = std::fs::read(fixture("rar250/AUTOREJ.RAR")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"PLAIN.TXT"); + assert_eq!(file.method, 0x35); + assert_eq!(file.unp_ver, 20); + assert_eq!(file.pack_size, 54); + assert_eq!(file.unp_size, 2300); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"PLAIN.TXT"); + assert_eq!(extracted[0].data.len(), 2300); + assert_eq!(crc32(&extracted[0].data), 0xafc0db74); +} + +#[test] +fn parses_rar250_protect_head_recovery_record() { + let bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let protect = archive.protect_records().next().unwrap(); + + assert_eq!(protect.version, 0x14); + assert_eq!(protect.rec_sectors, 5); + assert_eq!(protect.total_blocks, 201); + assert_eq!(protect.mark, *b"Protect!"); + assert_eq!( + protect.data_range.len(), + protect.total_blocks as usize * 2 + protect.rec_sectors as usize * 512 + ); + assert_eq!( + protect.total_blocks as usize, + protect.block.offset.div_ceil(512) + ); + assert_eq!(protect.block.offset % 512, 59); +} + +fn rewrite_recovery_header_crc(bytes: &mut [u8], offset: usize, head_size: usize) { + let crc = (crc32(&bytes[offset + 2..offset + head_size]) & 0xffff) as u16; + bytes[offset..offset + 2].copy_from_slice(&crc.to_le_bytes()); +} + +#[test] +fn rar250_recovery_rejects_invalid_mark_in_archive_bytes() { + let mut bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect = clean.protect_records().next().unwrap(); + let start = protect.block.offset; + bytes[start + 18] ^= 1; + rewrite_recovery_header_crc(&mut bytes, start, protect.block.head_size as usize); + + let malformed = Archive::parse(&bytes).unwrap(); + assert!(matches!( + malformed.repair_protect_head(), + Err(Error::InvalidHeader("RAR 2.x recovery mark is invalid")) + )); +} + +#[test] +fn rar250_recovery_rejects_zero_parity_record_in_archive_bytes() { + let mut bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect = clean.protect_records().next().unwrap(); + let start = protect.block.offset; + let tag_len = protect.total_blocks as usize * 2; + let parity_start = protect.data_range.start + tag_len; + bytes.drain(parity_start..protect.data_range.end); + bytes[start + 7..start + 11].copy_from_slice(&(tag_len as u32).to_le_bytes()); + bytes[start + 12..start + 14].copy_from_slice(&0u16.to_le_bytes()); + rewrite_recovery_header_crc(&mut bytes, start, protect.block.head_size as usize); + + let malformed = Archive::parse(&bytes).unwrap(); + assert!(matches!( + malformed.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x recovery record has no parity sectors" + )) + )); +} + +#[test] +fn rar250_recovery_rejects_protected_range_past_archive_end() { + let mut bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect = clean.protect_records().next().unwrap(); + let start = protect.block.offset; + let old_tag_len = protect.total_blocks as usize * 2; + let declared_blocks = 300u32; + let extra_tags = (declared_blocks - protect.total_blocks) as usize * 2; + let parity_start = protect.data_range.start + old_tag_len; + bytes.splice( + parity_start..parity_start, + std::iter::repeat_n(0, extra_tags), + ); + let add_size = protect.block.add_size.unwrap() as u32 + extra_tags as u32; + bytes[start + 7..start + 11].copy_from_slice(&add_size.to_le_bytes()); + bytes[start + 14..start + 18].copy_from_slice(&declared_blocks.to_le_bytes()); + rewrite_recovery_header_crc(&mut bytes, start, protect.block.head_size as usize); + + let malformed = Archive::parse(&bytes).unwrap(); + assert!((declared_blocks as usize) * 512 > bytes.len()); + assert!(matches!( + malformed.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x protected sector range is invalid" + )) + )); +} + +#[test] +fn rar250_parser_rejects_inconsistent_recovery_header_geometry() { + let original = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&original).unwrap(); + let protect = clean.protect_records().next().unwrap(); + let start = protect.block.offset; + + let mut short_header = original.clone(); + short_header[start + 5..start + 7].copy_from_slice(&25u16.to_le_bytes()); + rewrite_recovery_header_crc(&mut short_header, start, 25); + assert!(matches!( + Archive::parse(&short_header), + Err(Error::InvalidHeader( + "RAR 2.x recovery header size is invalid" + )) + )); + + let mut wrong_data_size = original; + let add_size = protect.block.add_size.unwrap() as u32 - 1; + wrong_data_size[start + 7..start + 11].copy_from_slice(&add_size.to_le_bytes()); + rewrite_recovery_header_crc(&mut wrong_data_size, start, 26); + assert!(matches!( + Archive::parse(&wrong_data_size), + Err(Error::InvalidHeader( + "RAR 2.x recovery data size does not match header" + )) + )); +} + +#[test] +fn rar250_repair_rejects_inconsistent_public_protection_metadata() { + let bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let mut archive = Archive::parse(&bytes).unwrap(); + let protect = archive + .blocks + .iter_mut() + .find_map(|block| match block { + Block::Protect(protect) => Some(protect), + _ => None, + }) + .unwrap(); + protect.data_range.end -= 1; + + assert!(matches!( + archive.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x recovery data size is invalid" + )) + )); +} + +#[test] +fn rar250_protect_head_declares_final_sector_that_overlaps_record() { + for (path, rec_sectors) in [ + ("rar250_protect_head_rr1.rar", 1), + ("rar250_protect_head_rr5.rar", 5), + ] { + let bytes = std::fs::read(fixture(path)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let protect = archive.protect_records().next().unwrap(); + + assert_eq!(protect.rec_sectors, rec_sectors); + assert_ne!(protect.block.offset % 512, 0); + assert_eq!( + protect.total_blocks as usize, + protect.block.offset.div_ceil(512) + ); + assert_eq!( + protect.total_blocks as usize, + protect.block.offset / 512 + 1 + ); + assert_eq!( + protect.data_range.len(), + protect.total_blocks as usize * 2 + protect.rec_sectors as usize * 512 + ); + } +} + +#[test] +fn parses_rar300_newsub_recovery_record() { + let bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_recovery_record()); + + let recovery = archive + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + assert_eq!(recovery.file.name, b"RR"); + assert_eq!(recovery.file.method, 0x30); + assert_eq!(recovery.file.pack_size, 5672); + assert_eq!(recovery.file.unp_size, 5672); +} + +#[test] +fn intact_rar300_recovery_record_leaves_archive_unchanged() { + let bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let repaired = archive.repair_protect_head_with_report().unwrap(); + + assert_eq!(repaired.data, bytes); + assert!(!repaired.report.changed); + assert!(!repaired.report.data_repaired); +} + +#[test] +fn preserves_unknown_rar300_service_records_without_hiding_files() { + let mut bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let original = Archive::parse(&bytes).unwrap(); + let recovery = original.new_subs().next().unwrap(); + let start = recovery.file.block.offset; + let name_start = start + 32; + assert_eq!(&bytes[name_start..name_start + 2], b"RR"); + bytes[name_start..name_start + 2].copy_from_slice(b"XY"); + rewrite_recovery_header_crc(&mut bytes, start, recovery.file.block.head_size as usize); + + let parsed = Archive::parse(&bytes).unwrap(); + assert!(matches!( + parsed.new_subs().next().unwrap().kind, + NewSubKind::Unknown(ref name) if name == b"XY" + )); + assert_eq!(parsed.files().count(), original.files().count()); + + let root = scratch::case("unknown-rar300-service-record"); + let path = root.join("unknown-service.rar"); + std::fs::write(&path, &bytes).unwrap(); + let seekable = Archive::parse_path(&path).unwrap(); + assert_eq!(seekable.blocks, parsed.blocks); +} + +#[test] +fn rar300_recovery_rejects_stored_size_mismatch_in_archive_bytes() { + let mut bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let recovery = clean + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + let start = recovery.file.block.offset; + let wrong_size = recovery.file.unp_size as u32 - 1; + bytes[start + 11..start + 15].copy_from_slice(&wrong_size.to_le_bytes()); + rewrite_recovery_header_crc(&mut bytes, start, recovery.file.block.head_size as usize); + + let malformed = Archive::parse(&bytes).unwrap(); + assert!(matches!( + malformed.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery record packed size does not match unpacked size" + )) + )); +} + +#[test] +fn rar300_repair_rejects_inconsistent_public_recovery_metadata() { + fn recovery_file(archive: &mut Archive) -> &mut rars::rar15_40::FileHeader { + archive + .blocks + .iter_mut() + .find_map(|block| match block { + Block::NewSub(sub) if sub.kind == NewSubKind::RecoveryRecord => Some(&mut sub.file), + _ => None, + }) + .unwrap() + } + + let bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let original = Archive::parse(&bytes).unwrap(); + + let mut short_payload = original.clone(); + recovery_file(&mut short_payload).packed_range.end -= 1; + assert!(matches!( + short_payload.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery data size does not match unpacked size" + )) + )); + + let mut past_source = original.clone(); + recovery_file(&mut past_source).block.offset = bytes.len() + 1; + assert!(matches!( + past_source.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery protected range is invalid" + )) + )); + + let mut no_protected_sectors = original; + recovery_file(&mut no_protected_sectors).block.offset = 0; + assert!(matches!( + no_protected_sectors.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery record has no protected sectors" + )) + )); +} + +#[test] +fn rar300_recovery_rejects_malformed_stored_body_lengths() { + let original = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let clean = Archive::parse(&original).unwrap(); + let recovery = clean + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + let tag_len = recovery.file.block.offset.div_ceil(512) * 2; + let packed = recovery.file.packed_range.clone(); + let start = recovery.file.block.offset; + + // One missing parity byte leaves an incomplete sector. Tags alone leave + // no parity sectors. Both are malformed recovery payloads. + for kept in [packed.len() - 1, tag_len] { + let mut bytes = original.clone(); + bytes.drain(packed.start + kept..packed.end); + bytes[start + 7..start + 11].copy_from_slice(&(kept as u32).to_le_bytes()); + bytes[start + 11..start + 15].copy_from_slice(&(kept as u32).to_le_bytes()); + rewrite_recovery_header_crc(&mut bytes, start, recovery.file.block.head_size as usize); + + let malformed = Archive::parse(&bytes).unwrap(); + assert!(matches!( + malformed.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery data size is invalid" + )) + )); + } +} + +#[test] +fn encrypted_rar300_newsub_recovery_is_rejected_before_decoding() { + let bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let mut archive = Archive::parse(&bytes).unwrap(); + let recovery = archive.blocks.iter_mut().find_map(|block| match block { + Block::NewSub(sub) if sub.kind == NewSubKind::RecoveryRecord => Some(sub), + _ => None, + }); + let recovery = recovery.unwrap(); + recovery.file.block.flags |= 0x0004; // FHD_PASSWORD + + assert!(matches!( + archive.repair_protect_head(), + Err(Error::UnsupportedFeature { + version: ArchiveVersion::Rar30, + feature: "encrypted RAR 3.x NEWSUB recovery record", + }) + )); +} + +#[test] +fn parses_compressed_rar300_newsub_recovery_record_fixture() { + let bytes = std::fs::read(fixture("rar300/with_compressed_recovery_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_recovery_record()); + + let recovery = archive + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + assert_eq!(recovery.file.name, b"RR"); + assert_eq!(recovery.file.method, 0x33); + assert_eq!(recovery.file.pack_size, 6444); + assert_eq!(recovery.file.unp_size, 5672); +} + +#[test] +fn rejects_corrupt_compressed_rar300_newsub_recovery_record_fixture() { + let bytes = std::fs::read(fixture( + "rar300/with_compressed_recovery_header_synthetic.rar", + )) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_recovery_record()); + + let recovery = archive + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + assert_eq!(recovery.file.name, b"RR"); + assert_eq!(recovery.file.method, 0x33); + + let err = archive.repair_protect_head().unwrap_err(); + assert!(matches!( + err, + Error::InvalidHeader(_) + | Error::Codec(_) + | Error::Crc32Mismatch { .. } + | Error::CrcMismatch { .. } + )); +} + +#[test] +fn repairs_rar250_protect_head_single_damaged_sector() { + let bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let _file = clean.files().next().unwrap(); + let damage_offset = 512 + 16; + let mut damaged = bytes.clone(); + damaged[damage_offset..damage_offset + 64].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_protect_head().unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse(&repaired).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"BIG.BIN"); + assert_eq!(crc32(&extracted[0].data), 0x9a0e0c8c); +} + +#[test] +fn rar250_recovery_rejects_damage_beyond_parity_capacity() { + let mut bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect = clean.protect_records().next().unwrap(); + assert_eq!(protect.rec_sectors, 5); + assert!(protect.block.offset > 7 * 512); + + for sector in 1..=6 { + bytes[sector * 512 + 16] ^= 0x55; + } + let damaged = Archive::parse(&bytes).unwrap(); + assert!(matches!( + damaged.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x recovery damage exceeds parity sector count" + )) + )); +} + +#[test] +fn rar250_recovery_rejects_incorrect_parity() { + let mut bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect = clean.protect_records().next().unwrap(); + let parity_start = protect.data_range.start + protect.total_blocks as usize * 2; + + bytes[512 + 16] ^= 0x55; + bytes[parity_start + 512 + 16] ^= 0x01; + let damaged = Archive::parse(&bytes).unwrap(); + let result = damaged.repair_protect_head(); + assert!( + matches!(result, Err(Error::CrcMismatch { .. })), + "{:?}", + result.as_ref().map(Vec::len) + ); +} + +#[test] +fn protect_head_does_not_repair_trailing_partial_sector_before_record() { + let bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect: &ProtectHeader = clean.protect_records().next().unwrap(); + assert_ne!(protect.block.offset % 512, 0); + + let mut damaged = bytes.clone(); + let damage_offset = protect.block.offset - 16; + damaged[damage_offset..damage_offset + 8].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_protect_head().unwrap(); + + assert_eq!(repaired, damaged); + assert_ne!(repaired, bytes); + assert!(collect_extract(&Archive::parse(&repaired).unwrap()).is_err()); +} + +#[test] +fn protect_head_repairs_last_stable_sector_before_metadata_overlap() { + let bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect: &ProtectHeader = clean.protect_records().next().unwrap(); + let stable_blocks = protect.block.offset / 512; + assert!(stable_blocks > 0); + assert!(usize::try_from(protect.total_blocks).unwrap() > stable_blocks); + assert_ne!(protect.block.offset % 512, 0); + + let damage_offset = (stable_blocks - 1) * 512 + 16; + assert!(damage_offset + 64 <= protect.block.offset); + let mut damaged = bytes.clone(); + damaged[damage_offset..damage_offset + 64].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_protect_head().unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse(&repaired).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"BIG.BIN"); + assert_eq!(crc32(&extracted[0].data), 0x9a0e0c8c); +} + +#[test] +fn repairs_rar300_newsub_recovery_single_damaged_sector() { + let bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let mut damaged = bytes.clone(); + damaged[512 + 16..512 + 80].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_protect_head().unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse(&repaired).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn rar300_recovery_rejects_incorrect_parity() { + let mut bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let recovery = clean + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + let protected_sectors = recovery.file.block.offset.div_ceil(512); + let parity_start = recovery.file.packed_range.start + protected_sectors * 2; + + bytes[512 + 16] ^= 0x55; + bytes[parity_start + 512 + 16] ^= 0x01; + let damaged = Archive::parse(&bytes).unwrap(); + let result = damaged.repair_protect_head(); + assert!( + matches!(result, Err(Error::CrcMismatch { .. })), + "{:?}", + result.as_ref().map(Vec::len) + ); +} + +#[test] +fn rar300_recovery_rejects_damage_beyond_parity_capacity() { + let mut bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let recovery = clean + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + let protected_sectors = recovery.file.block.offset.div_ceil(512); + let tag_len = protected_sectors * 2; + let parity_sectors = (recovery.file.unp_size as usize - tag_len) / 512; + assert!(parity_sectors + 1 < protected_sectors); + + for sector in 1..=parity_sectors + 1 { + bytes[sector * 512 + 16] ^= 0x55; + } + let damaged = Archive::parse(&bytes).unwrap(); + assert!(matches!( + damaged.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery damage exceeds parity sector count" + )) + )); +} + +#[test] +fn rar300_recovery_rejects_two_damaged_sectors_in_one_parity_group() { + let mut bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let recovery = clean + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + let protected_sectors = recovery.file.block.offset.div_ceil(512); + let tag_len = protected_sectors * 2; + let parity_sectors = (recovery.file.unp_size as usize - tag_len) / 512; + assert!(1 + parity_sectors < protected_sectors); + + bytes[512 + 16] ^= 0x55; + bytes[(1 + parity_sectors) * 512 + 16] ^= 0x55; + let damaged = Archive::parse(&bytes).unwrap(); + assert!(matches!( + damaged.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 3.x recovery cannot repair multiple sectors in the same parity group" + )) + )); +} + +#[test] +fn repairs_compressed_rar300_newsub_recovery_single_damaged_sector() { + let compressed = std::fs::read(fixture("rar300/with_compressed_recovery_rar300.rar")).unwrap(); + let expected = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let mut damaged = compressed.clone(); + damaged[512 + 16..512 + 80].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_protect_head().unwrap(); + + assert_eq!(repaired, compressed); + let repaired_archive = Archive::parse(&repaired).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); + + let expected_archive = Archive::parse(&expected).unwrap(); + let expected_extracted = collect_extract(&expected_archive).unwrap(); + assert_eq!(extracted[0].data, expected_extracted[0].data); +} + +#[test] +fn newsub_recovery_repairs_trailing_partial_sector_before_record() { + let bytes = std::fs::read(fixture("rar300/with_recovery_rar300.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let recovery = clean + .new_subs() + .find(|sub| sub.kind == NewSubKind::RecoveryRecord) + .unwrap(); + assert_ne!(recovery.file.block.offset % 512, 0); + + let mut damaged = bytes.clone(); + let damage_offset = recovery.file.block.offset - 16; + damaged[damage_offset..damage_offset + 8].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_protect_head().unwrap(); + + assert_eq!(repaired, bytes); + let extracted = collect_extract(&Archive::parse(&repaired).unwrap()).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn rejects_rar250_protect_head_same_group_damage() { + let bytes = std::fs::read(fixture("rar250_protect_head_rr5.rar")).unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let protect: &ProtectHeader = clean.protect_records().next().unwrap(); + assert_eq!(protect.rec_sectors, 5); + let mut damaged = bytes.clone(); + damaged[512 + 10] ^= 0x55; + damaged[512 * 6 + 10] ^= 0x55; + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(matches!( + damaged_archive.repair_protect_head(), + Err(Error::InvalidHeader( + "RAR 2.x recovery cannot repair multiple sectors in the same parity group" + )) + )); +} + +#[test] +fn repairs_rar300_old_style_recovery_volume_set() { + let part1 = std::fs::read(fixture("rar300/rev_oldstyle.part1.rar")).unwrap(); + let part2 = std::fs::read(fixture("rar300/rev_oldstyle.part2.rar")).unwrap(); + let part3 = std::fs::read(fixture("rar300/rev_oldstyle.part3.rar")).unwrap(); + let part4 = std::fs::read(fixture("rar300/rev_oldstyle.part4.rar")).unwrap(); + let rev1 = std::fs::read(fixture("rar300/rev_oldstyle.part4_2_1.rev")).unwrap(); + + let repaired = repair_rev3_volumes( + &[Some(&part1), None, Some(&part3), Some(&part4)], + 2, + &[(0, rev1.as_slice())], + ) + .unwrap(); + + assert_eq!(repaired[1], part2); + let extracted = collect_extract_volumes(&[ + Archive::parse(&repaired[0]).unwrap(), + Archive::parse(&repaired[1]).unwrap(), + Archive::parse(&repaired[2]).unwrap(), + Archive::parse(&repaired[3]).unwrap(), + ]) + .unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(crc32(&extracted[0].data), 0xf3a82e44); +} + +#[test] +fn rev3_repair_keeps_supplied_damaged_volume_when_no_shard_is_missing() { + let mut volume = std::fs::read(fixture("rar300/rev_oldstyle.part1.rar")).unwrap(); + volume[0] ^= 1; + assert!(Archive::parse(&volume).is_err()); + let parity = vec![0; volume.len()]; + + let repaired = repair_rev3_volumes(&[Some(&volume)], 1, &[(0, &parity)]).unwrap(); + + assert_eq!(repaired, [volume]); +} + +#[test] +fn repairs_rar4_new_style_recovery_volume_set_with_zeroed_rev_trailer() { + let part1 = std::fs::read(fixture("rar300/rev_newstyle.part1.rar")).unwrap(); + let part2 = std::fs::read(fixture("rar300/rev_newstyle.part2.rar")).unwrap(); + let part3 = std::fs::read(fixture("rar300/rev_newstyle.part3.rar")).unwrap(); + let part4 = std::fs::read(fixture("rar300/rev_newstyle.part4.rar")).unwrap(); + let mut rev1 = std::fs::read(fixture("rar300/rev_newstyle.part1.rev")).unwrap(); + let len = rev1.len(); + rev1[len - 7..].fill(0); + + let repaired = repair_rev3_volumes( + &[Some(&part1), None, Some(&part3), Some(&part4)], + 2, + &[(0, rev1.as_slice())], + ) + .unwrap(); + + assert_eq!(repaired[1], part2); + let extracted = collect_extract_volumes(&[ + Archive::parse(&repaired[0]).unwrap(), + Archive::parse(&repaired[1]).unwrap(), + Archive::parse(&repaired[2]).unwrap(), + Archive::parse(&repaired[3]).unwrap(), + ]) + .unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(crc32(&extracted[0].data), 0x442c5489); +} + +#[test] +fn extracts_rar250_unp20_multimedia_switch_lz_file() { + let bytes = std::fs::read(fixture("rar250/AUDIO.RAR")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"PCM_LR.WAV"); + assert_eq!(file.method, 0x35); + assert_eq!(file.unp_ver, 20); + assert_eq!(file.pack_size, 1938); + assert_eq!(file.unp_size, 32768); + assert_eq!(rar15_first_file_data_peek(&bytes), 0x0040); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"PCM_LR.WAV"); + assert_eq!(extracted[0].data, expected_rar250_multimedia_payload()); + assert_eq!(crc32(&extracted[0].data), 0x713ef34b); +} + +#[test] +fn extracts_synthetic_unp20_audio_block_archive() { + for channels in 1..=4 { + let samples = channels * 4; + let bytes = synthetic_rar20_audio_archive(channels, samples); + let peek = rar15_first_file_data_peek(&bytes); + assert_eq!(peek & 0x8000, 0x8000); + assert_eq!(((peek >> 12) & 3) + 1, channels as u16); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.name, format!("AUDIO{channels}.BIN").as_bytes()); + assert_eq!(file.method, 0x35); + assert_eq!(file.unp_ver, 20); + assert_eq!(file.unp_size, samples as u64); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, format!("AUDIO{channels}.BIN").as_bytes()); + assert_eq!(extracted[0].data, vec![0; samples]); + } +} + +#[test] +fn extracts_rar250_unp20_audio_shaped_and_text_lz_archive() { + let bytes = std::fs::read(fixture("rar250/unpack20_audio_text.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"BoatModernEnglish.wav"); + assert_eq!(files[1].name, b"LICENSE.txt"); + assert!(files.iter().all(|file| file.unp_ver == 20)); + assert_eq!(rar15_first_file_data_peek(&bytes), 0x2221); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"BoatModernEnglish.wav"); + assert_eq!(crc32(&extracted[0].data), files[0].file_crc); + assert_eq!(extracted[1].name, b"LICENSE.txt"); + assert_eq!(crc32(&extracted[1].data), files[1].file_crc); +} + +#[test] +fn extracts_rar250_unp20_solid_members() { + let bytes = std::fs::read(fixture("rar250/SOLID.RAR")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"SOLID1.TXT"); + assert_eq!(files[1].name, b"SOLID2.TXT"); + assert_eq!(files[0].unp_ver, 20); + assert_eq!(files[1].unp_ver, 20); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + assert_eq!(rar15_file_data_peeks(&bytes), [0x0dcd, 0xdfbe]); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, expected_rar250_solid1_payload()); + assert_eq!(extracted[1].data, expected_rar250_solid2_payload()); + assert_eq!(crc32(&extracted[0].data), 0x97668cf2); + assert_eq!(crc32(&extracted[1].data), 0x28833332); +} + +#[test] +fn extracts_rar250_unp20_large_lz_file() { + let bytes = std::fs::read(fixture("rar250/BIGLZ.RAR")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"BIGLZ.BIN"); + assert_eq!(file.method, 0x35); + assert_eq!(file.unp_ver, 20); + assert_eq!(file.unp_size, 167_936); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"BIGLZ.BIN"); + assert_eq!(extracted[0].data, expected_rar250_big_lz_payload()); + assert_eq!(crc32(&extracted[0].data), 0x46ce9077); +} + +#[test] +fn extracts_rar250_unp20_keep_tables_archive() { + let bytes = std::fs::read(fixture("rar250/unpack20_keep_tables.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"unrar"); + assert_eq!(files[0].method, 0x33); + assert_eq!(files[0].unp_ver, 20); + assert_eq!(files[0].pack_size, 25_077); + assert_eq!(files[0].unp_size, 54_212); + assert_eq!(files[0].file_crc, 0xbf94ba22); + assert_eq!(files[1].name, b"file_id.diz"); + assert_eq!(files[1].method, 0x33); + assert_eq!(files[1].unp_ver, 20); + assert_eq!(files[1].pack_size, 85); + assert_eq!(files[1].unp_size, 76); + assert_eq!(files[1].file_crc, 0x497a718f); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(crc32(&extracted[0].data), 0xbf94ba22); + assert_eq!(crc32(&extracted[1].data), 0x497a718f); +} + +#[test] +fn extracts_rar250_unp20_explicit_multiblock_archive() { + let bytes = std::fs::read(fixture("rar250/unpack20_multiblock.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"multiblock.bin"); + assert_eq!(file.unp_ver, 20); + assert_eq!(file.method, 0x35); + assert_eq!(file.pack_size, 4_761); + assert_eq!(file.unp_size, 16_384); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data.len(), 16_384); + assert_eq!(crc32(&extracted[0].data), 0xa24d_a8f8); +} + +#[test] +fn extracts_rar300_standard_rarvm_filter_fixtures() { + for (name, entry_name, size, expected_crc) in [ + ( + "rar300/rarvm_x86_e8_rar300.rar", + b"x86_e8_stream.bin".as_slice(), + 196_608, + 0xe0f3971f, + ), + ( + "rar300/rarvm_x86_e8e9_rar300.rar", + b"x86_e8e9_stream.bin".as_slice(), + 196_608, + 0xdc573e1b, + ), + ( + "rar300/rarvm_delta_4ch_rar300.rar", + b"delta_4ch_ramp.bin".as_slice(), + 262_144, + 0xa303b91f, + ), + ( + "rar300/rarvm_itanium_synthetic_rar300.rar", + b"itanium_synthetic_bundles.bin".as_slice(), + 1_048_576, + 0x39086451, + ), + ( + "rar300/rarvm_rgb_gradient_rar300.rar", + b"rgb_gradient_24bit.bmp".as_slice(), + 196_662, + 0xbf03aa49, + ), + ( + "rar300/rarvm_audio_stereo_rar300.rar", + b"audio_stereo_pcm.wav".as_slice(), + 705_644, + 0x8ad44141, + ), + ] { + let bytes = std::fs::read(fixture(name)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1, "{name}"); + assert_eq!(extracted[0].name, entry_name, "{name}"); + assert_eq!(extracted[0].data.len(), size, "{name}"); + assert_eq!(crc32(&extracted[0].data), expected_crc, "{name}"); + } +} + +#[test] +fn extracts_rar300_ppmd_text_file() { + let bytes = std::fs::read(fixture("ppmd/ppmd_lorem_rar300.rar")).unwrap(); + let expected = std::fs::read(fixture("ppmd/lorem_127k.txt")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"lorem_127k.txt"); + assert_eq!(file.method, 0x35); + assert_eq!(file.unp_ver, 29); + assert_eq!(file.pack_size, 13_276); + assert_eq!(file.unp_size, 130_048); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"lorem_127k.txt"); + assert_eq!(extracted[0].data, expected); + assert_eq!(crc32(&extracted[0].data), 0xc119b4e5); +} + +#[test] +fn extracts_rar300_mct_escape_input_that_selected_lz() { + let bytes = std::fs::read(fixture("ppmd/ppmd_escape_rar300.rar")).unwrap(); + let expected = std::fs::read(fixture("ppmd/escape_64k.bin")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"escape_64k.bin"); + assert_eq!(file.method, 0x35); + assert_eq!(file.unp_ver, 29); + assert_eq!(file.unp_size, 65_536); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"escape_64k.bin"); + assert_eq!(extracted[0].data, expected); + assert_eq!(crc32(&extracted[0].data), 0x9a945756); +} + +#[test] +fn extracts_rar300_ppmd_mixed_archive() { + let bytes = std::fs::read(fixture("ppmd/ppmd_mixed_rar300.rar")).unwrap(); + let expected_text = std::fs::read(fixture("ppmd/lorem_127k.txt")).unwrap(); + let expected_binary = std::fs::read(fixture("ppmd/binary_64k.bin")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"lorem_127k.txt"); + assert_eq!(files[1].name, b"binary_64k.bin"); + assert_eq!(files[0].unp_ver, 29); + assert_eq!(files[1].unp_ver, 29); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, expected_text); + assert_eq!(extracted[1].data, expected_binary); + assert_eq!(crc32(&extracted[0].data), 0xc119b4e5); + assert_eq!(crc32(&extracted[1].data), 0x9d672acd); +} + +#[test] +fn extracts_rar300_solid_ppmd_archive() { + let bytes = std::fs::read(fixture("ppmd/ppmd_solid_rar300.rar")).unwrap(); + let expected_a = std::fs::read(fixture("ppmd/solid_lorem_a.txt")).unwrap(); + let expected_b = std::fs::read(fixture("ppmd/solid_lorem_b.txt")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"solid_lorem_a.txt"); + assert_eq!(files[1].name, b"solid_lorem_b.txt"); + assert!(!files[0].is_solid()); + assert!(files[1].is_solid()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, expected_a); + assert_eq!(extracted[1].data, expected_b); + assert_eq!(crc32(&extracted[0].data), 0x14284201); + assert_eq!(crc32(&extracted[1].data), 0xca4cac47); +} + +#[test] +fn extracts_wild_solid_ppmd_farmanager_archive() { + let bytes = std::fs::read(fixture("ppmd/farmanager170.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let opened = Rc::new(RefCell::new(Vec::>::new())); + + assert!(archive.main.is_solid()); + assert!(archive + .files() + .any(|file| file.unp_ver >= 29 && file.method == 0x35)); + + let result = archive.extract_to(ArchiveReadOptions::default(), { + let opened = Rc::clone(&opened); + move |meta| { + opened.borrow_mut().push(meta.name.clone()); + if meta.name == b"Far.exe" { + return Err(Error::InvalidHeader("stopped after PPMd regression target")); + } + Ok(Box::new(std::io::sink())) + } + }); + + assert!(matches!( + result, + Err(Error::InvalidHeader("stopped after PPMd regression target")) + )); + let opened = opened.borrow(); + assert!(opened + .iter() + .any(|name| name == b"Addons\\Shell\\FARHere.inf")); + assert!(opened.iter().any(|name| name == b"Far.exe")); +} + +#[test] +fn decodes_rar300_compressed_archive_comment() { + let bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!( + archive.archive_comment().unwrap().as_deref(), + Some(b"This is the archive comment.\n".as_slice()) + ); +} + +#[test] +fn decodes_node_unrar_js_utf16_archive_comment() { + let bytes = std::fs::read(fixture("node_unrar_js/with_comment.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let comment = archive.archive_comment().unwrap().unwrap(); + let expected: Vec = "Test Comments for rar files.\r\n\r\n测试一下中文注释。\r\n日本語�コメントもテスト�����。" + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + + assert_eq!(comment, expected); + assert_eq!(crc32(&comment), 0xe96e8fcf); + assert_eq!(comment.len(), 122); +} + +#[test] +fn rejects_split_rar15_40_entries_until_volume_reassembly_exists() { + for (path, split_before) in [ + ("rar300/multivol_oldnaming_rar300.rar", false), + ("rar300/multivol_oldnaming_rar300.r00", true), + ] { + let bytes = std::fs::read(fixture(path)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!( + archive.files().next().unwrap().is_split_before(), + split_before + ); + assert!(matches!( + collect_extract(&archive), + Err(Error::InvalidHeader( + "RAR 1.5 split entry requires multivolume extraction" + )) + )); + assert!(matches!( + archive.extract_to_parallel_buffered(ArchiveReadOptions::new(), |_| { + Ok(Box::new(std::io::sink())) + }), + Err(Error::InvalidHeader( + "RAR 1.5 split entry requires multivolume extraction" + )) + )); + } +} + +#[test] +fn extracts_stored_rar300_old_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/stored_multivol_rar300.rar", + "rar300/stored_multivol_rar300.r00", + "rar300/stored_multivol_rar300.r01", + "rar300/stored_multivol_rar300.r02", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"stored-volume.txt"); + assert_eq!(extracted[0].data, expected_stored_volume_payload()); + assert_eq!(crc32(&extracted[0].data), 0x4a832ebd); +} + +#[test] +fn rejects_incomplete_rar300_stored_volume_set() { + let archives: Vec<_> = [ + "rar300/stored_multivol_rar300.rar", + "rar300/stored_multivol_rar300.r00", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + + assert!(matches!( + collect_extract_volumes(&archives), + Err(Error::InvalidHeader("RAR 1.5 split entry is incomplete")) + )); +} + +#[test] +fn extracts_compressed_rar300_old_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/compressed_multivol_prng_rar300.rar", + "rar300/compressed_multivol_prng_rar300.r00", + "rar300/compressed_multivol_prng_rar300.r01", + "rar300/compressed_multivol_prng_rar300.r02", + "rar300/compressed_multivol_prng_rar300.r03", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"cvolume.bin"); + assert_eq!(extracted[0].data.len(), 4096); + assert_eq!(crc32(&extracted[0].data), 0x96de2bef); +} + +#[test] +fn extracts_compressed_rar300_new_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/multivol_newnaming_rar300.part01.rar", + "rar300/multivol_newnaming_rar300.part02.rar", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives + .iter() + .all(|archive| archive.main.uses_new_numbering())); + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(extracted[0].data.len(), 65_536); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn extracts_encrypted_rar300_old_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/encrypted_multivol_rar300.rar", + "rar300/encrypted_multivol_rar300.r00", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + let first = archives[0].files().next().unwrap(); + let second = archives[1].files().next().unwrap(); + + assert!(first.is_encrypted()); + assert!(second.is_encrypted()); + assert_eq!(first.salt, second.salt); + assert!(matches!( + collect_extract_volumes(&archives), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(extracted[0].data.len(), 65_536); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn extracts_encrypted_rar300_new_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/encrypted_newnaming_rar300.part01.rar", + "rar300/encrypted_newnaming_rar300.part02.rar", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + + assert!(archives + .iter() + .all(|archive| archive.main.uses_new_numbering())); + assert!(archives + .iter() + .all(|archive| archive.files().next().unwrap().is_encrypted())); + assert!(matches!( + collect_extract_volumes(&archives), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(extracted[0].data.len(), 65_536); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn extracts_header_encrypted_rar300_old_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/header_encrypted_multivol_rar300.rar", + "rar300/header_encrypted_multivol_rar300.r00", + ] + .into_iter() + .map(|name| { + Archive::parse_with_password(&std::fs::read(fixture(name)).unwrap(), Some(b"password")) + .unwrap() + }) + .collect(); + let first = archives[0].files().next().unwrap(); + let second = archives[1].files().next().unwrap(); + + assert!(archives + .iter() + .all(|archive| archive.main.has_encrypted_headers())); + assert!(first.is_encrypted()); + assert!(second.is_encrypted()); + assert!(matches!( + collect_extract_volumes(&archives), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(extracted[0].data.len(), 65_536); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn extracts_header_encrypted_rar300_new_numbered_volume_set() { + let archives: Vec<_> = [ + "rar300/header_encrypted_newnaming_rar300.part01.rar", + "rar300/header_encrypted_newnaming_rar300.part02.rar", + ] + .into_iter() + .map(|name| { + Archive::parse_with_password(&std::fs::read(fixture(name)).unwrap(), Some(b"password")) + .unwrap() + }) + .collect(); + + assert!(archives + .iter() + .all(|archive| archive.main.has_encrypted_headers())); + assert!(archives + .iter() + .all(|archive| archive.main.uses_new_numbering())); + assert!(matches!( + collect_extract_volumes(&archives), + Err(Error::NeedPassword) + )); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bigtext_64k.bin"); + assert_eq!(extracted[0].data.len(), 65_536); + assert_eq!(crc32(&extracted[0].data), 0xddc95682); +} + +#[test] +fn extracts_rar154_unp15_old_numbered_volume_set() { + let archives: Vec<_> = [ + "rar154/random.rar", + "rar154/random.r00", + "rar154/random.r01", + ] + .into_iter() + .map(|name| Archive::parse(&std::fs::read(fixture(name)).unwrap()).unwrap()) + .collect(); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"random.bin"); + assert_eq!(extracted[0].data.len(), 2_097_152); + assert_eq!(crc32(&extracted[0].data), 0x1c9e_b697); +} + +#[test] +fn parses_rar300_solid_flags() { + let bytes = std::fs::read(fixture("rar300/solid_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert!(archive.main.is_solid()); + let files: Vec<_> = archive.files().collect(); + assert!(files.len() >= 2); + assert!(!files[0].is_solid()); + assert!(files[1..].iter().all(|file| file.is_solid())); +} + +#[test] +fn parses_old_and_new_volume_numbering_flags() { + let old_bytes = std::fs::read(fixture("rar300/multivol_oldnaming_rar300.rar")).unwrap(); + let old = Archive::parse(&old_bytes).unwrap(); + assert!(old.main.is_volume()); + assert!(old.main.is_first_volume()); + assert!(!old.main.uses_new_numbering()); + assert!(old.files().any(|file| file.is_split_after())); + + let new_bytes = std::fs::read(fixture("rar300/multivol_newnaming_rar300.part01.rar")).unwrap(); + let new = Archive::parse(&new_bytes).unwrap(); + assert!(new.main.is_volume()); + assert!(new.main.is_first_volume()); + assert!(new.main.uses_new_numbering()); + assert!(new.files().any(|file| file.is_split_after())); +} + +#[test] +fn parses_rar420_extended_time_header_bytes() { + let bytes = std::fs::read(fixture("rar420/ext_time_rar420.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 1); + assert!(files.iter().all(|file| file.has_ext_time())); + assert!(files.iter().all(|file| !file.ext_time.is_empty())); + assert_eq!(files[0].unp_ver, 29); +} + +#[test] +fn parses_end_of_archive_block() { + let bytes = std::fs::read(fixture("rar300/with_comment_rar300.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert!(matches!(archive.blocks.last(), Some(Block::End(_)))); +} + +#[test] +fn crc32_matches_standard_check_value() { + assert_eq!(crc32(b""), 0x00000000); + assert_eq!(crc32(b"123456789"), 0xcbf43926); +} + +#[test] +fn rejects_file_header_salt_that_extends_beyond_head_size_without_panicking() { + let mut archive = b"Rar!\x1a\x07\x00".to_vec(); + archive.extend_from_slice(&rar15_header(0x73, 0, &[0; 6])); + + let mut file_body = Vec::new(); + push_u32(&mut file_body, 0); + push_u32(&mut file_body, 0); + file_body.push(2); + push_u32(&mut file_body, 0); + push_u32(&mut file_body, 0); + file_body.push(29); + file_body.push(0x30); + push_u16(&mut file_body, 0); + push_u32(&mut file_body, 0x20); + + archive.extend_from_slice(&rar15_header(0x74, 0x0400 | 0x1000, &file_body)); + archive.extend_from_slice(&[0xaa; 8]); + + assert!(Archive::parse(&archive).is_err()); +} + +fn expected_stored_volume_payload() -> Vec { + "RAR 3.00 stored multivolume fixture line.\n" + .repeat(80) + .into_bytes() +} + +fn expected_compressed_text_payload() -> Vec { + "Hello, RAR 3.x fixture world.\n".repeat(80).into_bytes() +} + +fn expected_doc_154_best_manifest() -> [(&'static str, usize, u32); 17] { + [ + ("ARCH~Y3X.MD", 53_262, 0x5ab9_a7da), + ("CRC3~F4U.MD", 5_271, 0xad7e_2a11), + ("ENCR~BXO.MD", 30_796, 0xc5d3_da4f), + ("FILT~XX0.MD", 27_069, 0x89e6_3874), + ("HUFF~BID.MD", 11_958, 0xc4b2_3356), + ("IMPL~KS0.MD", 8_846, 0x0def_58b4), + ("INTE~BSL.MD", 32_722, 0xcb56_7947), + ("LZ_M~HYW.MD", 14_181, 0xf5e6_4896), + ("PATH~EJS.MD", 13_819, 0x3c0d_6e22), + ("PPMD~D4Q.MD", 38_140, 0xffd9_b31f), + ("RAR1~FHU.MD", 40_371, 0xaac1_91a8), + ("RAR1~OEK.MD", 101_788, 0x292f_35d1), + ("RAR5~YP0.MD", 71_276, 0xe52c_f5ec), + ("RARV~0F3.MD", 12_429, 0xab07_a4a6), + ("README.md", 4_198, 0x509e_5e3c), + ("READ~0WB.MD", 22_024, 0xd987_5535), + ("TEST~FAD.MD", 14_811, 0xb55b_a84a), + ] +} + +fn expected_rar250_multimedia_payload() -> Vec { + let mut pcm = Vec::with_capacity(32 * 1024); + for i in 0..8192 { + let left = (20000.0 * ((i as f64) * 2.0 * std::f64::consts::PI / 256.0).sin()) as i32; + let right = + (15000.0 * ((i as f64) * 2.0 * std::f64::consts::PI / 384.0 + 1.0).sin()) as i32; + pcm.extend_from_slice(&(left as u16).to_le_bytes()); + pcm.extend_from_slice(&(right as u16).to_le_bytes()); + } + pcm +} + +fn rar15_first_file_data_peek(bytes: &[u8]) -> u16 { + rar15_file_data_peeks(bytes)[0] +} + +fn rar15_file_data_peeks(bytes: &[u8]) -> Vec { + let mut pos = 7 + 13; + let mut peeks = Vec::new(); + while pos + 7 <= bytes.len() { + let head_type = bytes[pos + 2]; + let flags = u16::from_le_bytes([bytes[pos + 3], bytes[pos + 4]]); + let head_size = u16::from_le_bytes([bytes[pos + 5], bytes[pos + 6]]) as usize; + if head_size < 7 || pos + head_size > bytes.len() { + break; + } + let add_size = if flags & 0x8000 != 0 { + u32::from_le_bytes([ + bytes[pos + 7], + bytes[pos + 8], + bytes[pos + 9], + bytes[pos + 10], + ]) as usize + } else { + 0 + }; + if head_type == 0x74 { + let data = pos + head_size; + peeks.push(u16::from_be_bytes([bytes[data], bytes[data + 1]])); + } + pos += head_size + add_size; + } + peeks +} + +fn synthetic_rar20_audio_archive(channels: usize, samples: usize) -> Vec { + let packed = synthetic_rar20_audio_block(channels, samples); + let unpacked = vec![0; samples]; + let name = format!("AUDIO{channels}.BIN").into_bytes(); + + let mut archive = b"Rar!\x1a\x07\x00".to_vec(); + archive.extend_from_slice(&rar15_header(0x73, 0, &[0; 6])); + + let mut file_body = Vec::new(); + push_u32(&mut file_body, packed.len() as u32); + push_u32(&mut file_body, samples as u32); + file_body.push(2); // host OS: Win32. + push_u32(&mut file_body, crc32(&unpacked)); + push_u32(&mut file_body, 0x4a83_a11d); + file_body.push(20); + file_body.push(0x35); + push_u16(&mut file_body, name.len() as u16); + push_u32(&mut file_body, 0x20); + file_body.extend_from_slice(&name); + + archive.extend_from_slice(&rar15_header(0x74, 0x8000, &file_body)); + archive.extend_from_slice(&packed); + archive +} + +fn rar15_header(head_type: u8, flags: u16, body: &[u8]) -> Vec { + let mut header = Vec::new(); + push_u16(&mut header, 0); + header.push(head_type); + push_u16(&mut header, flags); + push_u16(&mut header, (7 + body.len()) as u16); + header.extend_from_slice(body); + + let crc = (crc32(&header[2..]) & 0xffff) as u16; + header[0..2].copy_from_slice(&crc.to_le_bytes()); + header +} + +fn synthetic_rar20_audio_block(channels: usize, samples: usize) -> Vec { + let mut bits = TestBitWriter::default(); + + bits.write_bits(0b10, 2); // audio block, do not keep previous tables. + bits.write_bits((channels - 1) as u32, 2); + + for symbol in 0..19 { + let len = if symbol == 1 || symbol == 18 { 1 } else { 0 }; + bits.write_bits(len, 4); + } + + for _ in 0..channels { + bits.write_bit(false); // level symbol 1: audio delta 0 has code length 1. + bits.write_bit(true); // level symbol 18: 138 zeros. + bits.write_bits(127, 7); + bits.write_bit(true); // level symbol 18: 118 zeros. + bits.write_bits(107, 7); + } + + for _ in 0..samples { + bits.write_bit(false); // audio delta 0. + } + + bits.finish() +} + +fn push_u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_le_bytes()); +} + +fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_le_bytes()); +} + +#[derive(Default)] +struct TestBitWriter { + bytes: Vec, + bit_pos: usize, +} + +impl TestBitWriter { + fn write_bit(&mut self, bit: bool) { + if self.bit_pos.is_multiple_of(8) { + self.bytes.push(0); + } + if bit { + let shift = 7 - (self.bit_pos % 8); + *self.bytes.last_mut().unwrap() |= 1 << shift; + } + self.bit_pos += 1; + } + + fn write_bits(&mut self, value: u32, count: u8) { + for shift in (0..count).rev() { + self.write_bit(((value >> shift) & 1) != 0); + } + } + + fn finish(self) -> Vec { + self.bytes + } +} + +fn expected_rar250_solid1_payload() -> Vec { + rar250_solid_shared_line().repeat(180).into_bytes() +} + +fn expected_rar250_solid2_payload() -> Vec { + let mut data = rar250_solid_shared_line().repeat(90).into_bytes(); + data.extend_from_slice( + "second member unique tail after shared history.\r\n" + .repeat(120) + .as_bytes(), + ); + data +} + +fn rar250_solid_shared_line() -> &'static str { + "RAR 2.50 solid dictionary carry-over line with repeated tokens alpha beta gamma delta.\r\n" +} + +fn expected_rar250_big_lz_payload() -> Vec { + let mut data = Vec::with_capacity(167_936); + for i in 0..4096 { + data.extend_from_slice(format!("{i:04x}: unpack20 block refresh fixture ").as_bytes()); + data.extend_from_slice(&[(i * 17) as u8, (i * 31) as u8, b'\r', b'\n']); + } + data +} + +/// The engine and the filter used to be one five-variant enum, so some pairings +/// could not be asked for at all. These are the ones that were missing. +#[test] +fn rar29_chooses_engine_and_filter_independently() { + // Text, so the PPMd trial is worth taking; the delta filter below is asked + // for explicitly rather than because it suits the content. + let payload = b"rar29 two axis payload alpha beta gamma delta epsilon\n".repeat(400); + let entries = [FileEntry { + name: b"two-axis.txt", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + let options = WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()); + + // The method byte names the level, so it cannot say which engine ran. The + // pack size against a known PPMd encode of the same payload can. + let ppmd_packed = rars::codec::rar29::unpack29_encode_ppmd(&payload, RAR29_DICTIONARY).unwrap(); + let is_ppmd = |options: WriterOptions, policy: FilterPolicy| { + let bytes = + write_rar29_compressed_archive_with_filter_policy(&entries, options, policy).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let pack_size = file.pack_size; + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload, "round trip failed"); + pack_size == ppmd_packed.len() as u64 + }; + + // Searching for a filter with PPMd off: new, and the search must not reach + // for PPMd behind the caller's back. + assert!( + !is_ppmd(options.with_method(Rar29Method::Lz), FilterPolicy::Auto), + "forcing LZ must not produce a PPMd member" + ); + // Weighing an explicitly named filter against PPMd: also new. + assert!( + is_ppmd( + options.with_method(Rar29Method::Auto), + FilterPolicy::explicit(FilterKind::Delta { channels: 2 }) + ), + "text with an explicit filter should still be allowed to pick PPMd" + ); + // Forcing PPMd leaves the filter search nothing to measure against. + let refused = write_rar29_compressed_archive_with_filter_policy( + &entries, + options.with_method(Rar29Method::Ppmd), + FilterPolicy::Auto, + ); + assert!( + refused.is_err(), + "PPMd plus a filter search must be refused" + ); +} + +/// The automatic search now emits one filter per merged code region, which is a +/// shape the RarVM program emitter never saw while the two copies of the range +/// merger disagreed and this one dropped overlaps instead of merging them. +#[test] +fn rar29_auto_filter_round_trips_a_member_with_two_distant_code_regions() { + let mut payload = vec![0x41u8; 900_000]; + for region_start in [40_000, 700_000] { + for index in 0..2_000u32 { + let pos = region_start + index as usize * 32; + payload[pos] = 0xe8; + payload[pos + 1..pos + 5].copy_from_slice(&(0x4000u32 + index).to_le_bytes()); + } + } + let entries = [FileEntry { + name: b"two-regions.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + FilterPolicy::Auto, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +/// A filter whose chunking left a remainder shorter than its own scanline width +/// used to fail the whole write. Those bytes are simply left unfiltered now. +#[test] +fn rar29_rgb_filter_handles_a_length_that_leaves_a_short_trailing_chunk() { + // 131_072 splits into one 131_064-byte chunk and an 8-byte remainder, which + // is shorter than the 24-byte scanline. + let payload = vec![0x40u8; 131_072]; + let entries = [FileEntry { + name: b"short-tail.rgb", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Lz), + FilterPolicy::Explicit(FilterSpec::whole(FilterKind::Rgb { + width: 24, + pos_r: 0, + })), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +/// A filter record's start is read relative to the head of the block that +/// declares it, and the decoder masks that offset against its window. Declaring +/// every record at the head of the member wrote offsets the window could not +/// express, so a filter aimed past the first window landed somewhere else and +/// the member came out corrupt. Our own decoder made the same mistake, so it +/// agreed with the writer and only an external tool disagreed. +#[test] +fn rar29_applies_a_filter_that_starts_beyond_the_dictionary() { + // Calls to one address, written as the relative displacements a compiler + // would emit. The filter turns them back into the same absolute address, so + // it pays off only if it reaches them. The default RAR 2.9 dictionary is + // 1 MiB, and the calls start past it. + let mut payload: Vec = (0..3_000_000u32).map(|index| (index % 251) as u8).collect(); + const TARGET: u32 = 0x0020_1000; + for pos in (1_500_000..2_800_000).step_by(16) { + payload[pos] = 0xe8; + let displacement = TARGET.wrapping_sub(pos as u32); + payload[pos + 1..pos + 5].copy_from_slice(&displacement.to_le_bytes()); + } + let entries = [FileEntry { + name: b"late-filter.bin", + data: &payload, + file_time: 0x5a21_0000, + file_attr: 0x20, + host_os: 3, + password: None, + file_comment: None, + }]; + + for policy in [ + FilterPolicy::Auto, + FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8E9, 1_500_000..2_800_000)), + ] { + let bytes = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Lz), + policy.clone(), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); + + // A round trip alone would not have caught this, because the decoder + // read the offsets back the same way the writer wrote them. The size + // is what shows the filter reached the bytes it was aimed at, so the + // named one has to pay off. The automatic policy is free to decline. + if matches!(policy, FilterPolicy::Explicit(_)) { + let unfiltered = write_rar29_compressed_archive_with_filter_policy( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()) + .with_method(Rar29Method::Lz), + FilterPolicy::None, + ) + .unwrap(); + assert!( + bytes.len() < unfiltered.len(), + "the filter did not reach its bytes: {} against {} unfiltered", + bytes.len(), + unfiltered.len() + ); + } + } +} + +/// The streaming writer and the in-memory one are the same writer with a +/// different way of getting at the bytes, so the archives they produce have to +/// be the same archives. Anything else means the streaming path has quietly +/// grown its own behaviour. +#[test] +fn streaming_and_buffered_writers_agree_byte_for_byte() { + use rars::rar15_40::{write_streaming_archive_to, StreamingEntry}; + use rars::{EntrySource, MemberCoding, WriterResources}; + + let text = b"the quick brown fox jumps over the lazy dog\n".repeat(400); + let binary = level_sensitive_payload(); + let members: Vec<(&[u8], &[u8])> = vec![ + (b"text.txt", &text), + (b"binary.bin", &binary), + (b"empty.dat", b""), + ]; + + for target in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for solid in [false, true] { + for coding in [ + MemberCoding::Stored, + MemberCoding::Compressed, + MemberCoding::Filtered(FilterPolicy::Auto), + MemberCoding::Filtered(FilterPolicy::None), + ] { + let mut features = FeatureSet::store_only(); + features.solid = solid && coding.compresses(); + let options = WriterOptions::new(target, features); + let comment: Option<&[u8]> = Some(b"shared comment"); + + let buffered = match &coding { + MemberCoding::Stored => { + let entries: Vec<_> = members + .iter() + .map(|(name, data)| StoredEntry { + name, + data, + file_time: 0, + file_attr: 0, + host_os: 3, + password: None, + file_comment: None, + }) + .collect(); + write_stored_archive_with_comment(&entries, options, comment) + } + coding => { + let entries: Vec<_> = members + .iter() + .map(|(name, data)| FileEntry { + name, + data, + file_time: 0, + file_attr: 0, + host_os: 3, + password: None, + file_comment: None, + }) + .collect(); + match coding { + MemberCoding::Filtered(policy) => { + write_rar29_compressed_archive_with_filter_policy( + &entries, + options, + policy.clone(), + ) + } + _ => write_compressed_archive_with_comment(&entries, options, comment), + } + } + }; + + let streamed_entries: Vec<_> = members + .iter() + .map(|(name, data)| { + StreamingEntry::new(name.to_vec(), EntrySource::from_bytes(data.to_vec())) + .with_host_os(3) + }) + .collect(); + // The filtered entry point takes no comment, so neither does + // the comparison. + let streamed_comment = if coding.is_filtered() { None } else { comment }; + let mut streamed = Vec::new(); + let streamed_result = write_streaming_archive_to( + &streamed_entries, + options, + coding.clone(), + streamed_comment, + &WriterResources::default(), + None, + &mut streamed, + ); + + let label = format!("{target} solid={solid} {coding:?}"); + match buffered { + Ok(buffered) => { + assert!(streamed_result.is_ok(), "{label}: streaming refused what the buffered writer accepted: {streamed_result:?}"); + assert_eq!(streamed, buffered, "{label}: archives differ"); + let archive = Archive::parse(&streamed).unwrap(); + let extracted = collect_extract(&archive) + .unwrap_or_else(|error| panic!("{label}: {error:?}")); + assert_eq!(extracted.len(), members.len(), "{label}"); + for (got, (_, want)) in extracted.iter().zip(&members) { + assert_eq!(&got.data, want, "{label}"); + } + } + Err(buffered) => { + let streamed_error = streamed_result.expect_err(&format!( + "{label}: streaming accepted what the buffered writer refused" + )); + assert_eq!( + streamed_error.to_string(), + buffered.to_string(), + "{label}: the two paths refused it differently" + ); + } + } + } + } + } +} + +/// A stored member is copied straight from its source rather than read into +/// memory, which is a different code path from the one that holds the bytes. +#[test] +fn a_stored_member_round_trips_from_a_file_on_disk() { + use rars::rar15_40::{write_streaming_archive_to, StreamingEntry}; + use rars::{EntrySource, MemberCoding, WriterResources}; + + let directory = scratch::case("rars-stream"); + let path = directory.join("payload.bin"); + // Larger than the walk chunk, so the checksum really is taken in pieces. + let payload: Vec = (0..700_000u32).map(|index| index as u8).collect(); + std::fs::write(&path, &payload).unwrap(); + + let entries = vec![StreamingEntry::new( + b"payload.bin".to_vec(), + EntrySource::from_path(path.clone()), + )]; + let mut bytes = Vec::new(); + write_streaming_archive_to( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + MemberCoding::Stored, + None, + &WriterResources::default(), + None, + &mut bytes, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, payload); + assert_eq!(crc32(&extracted[0].data), crc32(&payload)); +} + +/// A member larger than the whole budget still gets written: the legacy codecs +/// have no smaller unit to fall back to, so the budget serialises them rather +/// than refusing the job. +#[test] +fn a_member_larger_than_the_budget_is_written_anyway() { + use rars::rar15_40::{write_streaming_archive_to, StreamingEntry}; + use rars::{EntrySource, MemberCoding, WriterResources}; + + let payload = b"a budget this small cannot hold one member\n".repeat(500); + let entries: Vec<_> = (0..4) + .map(|index| { + StreamingEntry::new( + format!("member{index}.txt").into_bytes(), + EntrySource::from_bytes(payload.clone()), + ) + }) + .collect(); + + let mut bytes = Vec::new(); + write_streaming_archive_to( + &entries, + WriterOptions::new(ArchiveVersion::Rar29, FeatureSet::store_only()), + MemberCoding::Compressed, + None, + &WriterResources::new(4096), + None, + &mut bytes, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 4); + for entry in &extracted { + assert_eq!(entry.data, payload); + } +} + +/// Tests an archive with a locally installed reference tool, returning `None` +/// when there is not one. +fn local_reference_test(label: &str, archive: &[u8]) -> Option { + let dir = scratch::case(&format!("rars-{label}")); + let path = dir.join("archive.rar"); + std::fs::write(&path, archive).unwrap(); + let mut result = None; + for tool in ["unrar", "rar"] { + if let Ok(output) = Command::new(tool).arg("t").arg(&path).output() { + result = Some(output); + break; + } + } + result +} + +fn solid_options(target: ArchiveVersion) -> WriterOptions { + let mut features = FeatureSet::store_only(); + features.solid = true; + WriterOptions::new(target, features) +} + +fn file_entries<'a>(members: &'a [(&'a [u8], Vec)]) -> Vec> { + members + .iter() + .map(|(name, data)| FileEntry { + name, + data, + file_time: 0, + file_attr: 0, + host_os: 3, + password: None, + file_comment: None, + }) + .collect() +} + +/// A member compression cannot shrink is stored instead, which rebuilds the +/// encoder and so starts a fresh solid chain. Only RAR 2.0 onwards can say so: +/// its file headers carry a solid bit. RAR 1.5 has none, and readers take every +/// member of a solid RAR 1.5 archive as a continuation whatever the header +/// says, so breaking the chain there wrote an archive nothing could read. +#[test] +fn a_solid_rar15_run_survives_a_member_that_does_not_compress() { + let compressible = b"solid chain payload that repeats and repeats\n".repeat(400); + // Incompressible, and past the 1 KiB floor, so the store fallback wants it. + let incompressible: Vec = (0..90_000u32) + .map(|index| { + let mixed = index.wrapping_mul(2_654_435_761).rotate_left(index % 17); + (mixed >> 13) as u8 + }) + .collect(); + let tail = b"and a short tail afterwards\n".repeat(50); + let members: Vec<(&[u8], Vec)> = vec![ + (b"first.txt", compressible), + (b"middle.bin", incompressible), + (b"last.txt", tail), + ]; + let entries = file_entries(&members); + + let bytes = write_compressed_archive(&entries, solid_options(ArchiveVersion::Rar15)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), members.len()); + for (got, (_, want)) in extracted.iter().zip(&members) { + assert_eq!(&got.data, want); + } + + if let Some(output) = local_reference_test("rar15-solid-store", &bytes) { + assert!( + output.status.success(), + "the reference tool rejected a solid RAR 1.5 archive\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +/// An empty member neither carries the solid chain nor breaks it: it feeds the +/// encoder nothing, and a reader passes over an empty payload without advancing +/// its decoder. Counting one as a member left the member after it flagged as +/// continuing a chain that a stored member two places back had already broken. +#[test] +fn an_empty_member_does_not_restart_a_broken_solid_chain() { + let opener = b"a short opening member\n".repeat(3); + let incompressible: Vec = (0..40_000u32) + .map(|index| { + let mixed = index.wrapping_mul(0x9e37_79b9).rotate_left(index % 23); + (mixed >> 11) as u8 + }) + .collect(); + let members: Vec<(&[u8], Vec)> = vec![ + (b"opener.txt", opener), + (b"incompressible.bin", incompressible), + (b"empty.dat", Vec::new()), + (b"after.txt", b"the member after the empty one\n".repeat(4)), + ]; + let entries = file_entries(&members); + + for target in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let bytes = write_compressed_archive(&entries, solid_options(target)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let extracted = + collect_extract(&archive).unwrap_or_else(|error| panic!("{target}: {error:?}")); + assert_eq!(extracted.len(), members.len(), "{target}"); + for (got, (_, want)) in extracted.iter().zip(&members) { + assert_eq!(&got.data, want, "{target}"); + } + + if let Some(output) = local_reference_test("solid-empty-member", &bytes) { + assert!( + output.status.success(), + "{target}: the reference tool rejected it\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + } +} + +#[test] +fn extracts_legacy_matches_that_reach_past_the_start_of_the_window() { + let mut expected = vec![0u8; 8]; + for _ in 0..8 { + expected.extend_from_slice(b"zero-fill regression payload "); + } + + for name in ["zero_fill/rar20.rar", "zero_fill/rar29.rar"] { + let bytes = std::fs::read(fixture(name)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + let entry = collect_file(&archive, file).unwrap(); + + assert_eq!(entry.name, b"zerofill.bin", "{name}"); + assert_eq!(entry.data, expected, "{name}"); + } +} diff --git a/arca-rar/vendor/rars/tests/rar50_fixtures.rs b/arca-rar/vendor/rars/tests/rar50_fixtures.rs new file mode 100644 index 00000000..76934475 --- /dev/null +++ b/arca-rar/vendor/rars/tests/rar50_fixtures.rs @@ -0,0 +1,6699 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::codec::rar50::{ + decode_lz, encode_lz_member, encode_table_lengths, parse_compressed_block, read_table_lengths, + DecodeTables, +}; +use rars::crc32::crc32; +use rars::crypto::rar50::{Rar50Cipher, Rar50Keys}; +use rars::rar50::{ + extract_volumes_to, repair_inline_recovery_bytes, repair_rev5_volumes_to, Archive, + ArchiveMetadataEntry, Block, FilterKind, FilterPolicy, Rev5Volume, Rev5VolumeMeta, + ServiceEntry, +}; +use rars::recovery::rar5::crc64_xz; +use rars::{ + detect_archive_family, rar50, ArchiveFamily, ArchiveReadOptions, ArchiveVersion, Error, + FeatureSet, +}; +use std::cell::RefCell; +use std::fs; +use std::io::{Read, Result as IoResult, Write}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::rc::Rc; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar50") + .join(name) +} + +#[test] +fn public_rar50_table_encoder_checks_each_table_shape() { + let packed = encode_lz_member(b"ABABABAB", 0).unwrap(); + let block = parse_compressed_block(&packed).unwrap(); + let (lengths, _) = read_table_lengths(&packed[block.payload], 0).unwrap(); + assert!(encode_table_lengths(&lengths, 0).is_ok()); + + for field in 0..4 { + let mut malformed = lengths.clone(); + match field { + 0 => malformed.main.pop(), + 1 => malformed.distance.pop(), + 2 => malformed.align.pop(), + 3 => malformed.length.pop(), + _ => unreachable!(), + }; + assert_eq!( + encode_table_lengths(&malformed, 0), + Err(rars::codec::Error::InvalidData( + "RAR 5 table length count mismatch" + )), + "malformed table field {field}" + ); + } +} + +#[test] +fn rar50_member_encoder_rejects_invalid_filter_ranges_in_both_size_paths() { + for data in [vec![b'A'; 16], vec![b'A'; 0x40000 + 1]] { + for range in [0..0, data.len()..data.len() - 1, data.len()..data.len() + 1] { + let invalid = rars::FilterSpec::range(rars::FilterKind::E8, range.clone()); + assert_eq!( + rars::codec::rar50::Unpack50Encoder::new() + .encode_member_with_filter(&data, 0, invalid), + Err(rars::codec::Error::InvalidData( + "RAR 5 filter range is invalid" + )), + "input length {}, range {range:?}", + data.len() + ); + } + } +} + +#[test] +fn large_rar50_member_routes_reject_unknown_version_without_committing_history() { + let data = vec![b'A'; 64 * 1024 + 1]; + let expected_error = + rars::codec::Error::InvalidData("RAR 5 unknown compression algorithm version"); + let retry = b"ABABABAB"; + let expected_retry = rars::codec::rar50::Unpack50Encoder::new() + .encode_member(retry, 0) + .unwrap(); + + for filtered in [false, true] { + let mut encoder = rars::codec::rar50::Unpack50Encoder::new(); + let result = if filtered { + encoder.encode_member_with_filter(&data, 2, rars::FilterSpec::whole(FilterKind::E8)) + } else { + encoder.encode_member(&data, 2) + }; + assert_eq!(result, Err(expected_error.clone()), "filtered={filtered}"); + assert_eq!(encoder.encode_member(retry, 0).unwrap(), expected_retry); + } +} + +#[test] +fn large_rar50_delta_filter_rejects_invalid_channels_without_committing_history() { + let data = vec![b'A'; 64 * 1024 + 1]; + let retry = b"ABABABAB"; + let expected_retry = rars::codec::rar50::Unpack50Encoder::new() + .encode_member(retry, 0) + .unwrap(); + + for channels in [0, 33] { + let mut encoder = rars::codec::rar50::Unpack50Encoder::new(); + assert_eq!( + encoder.encode_member_with_filter( + &data, + 0, + rars::FilterSpec::whole(FilterKind::Delta { channels }), + ), + Err(rars::codec::Error::InvalidData( + "RAR 5 DELTA filter channel count is invalid" + )), + "channels={channels}" + ); + assert_eq!(encoder.encode_member(retry, 0).unwrap(), expected_retry); + } +} + +fn service_names(archive: &Archive) -> Vec { + archive + .services() + .map(|service| service.name_lossy()) + .collect() +} + +struct CollectWriter { + data: Rc>>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +// Historical numeric snapshots; timestamp presence has dedicated regression tests. +struct CollectedEntry { + name: Vec, + data: Vec, + file_time: u32, + attr: u64, + host_os: u64, + is_directory: bool, +} + +impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.data.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +fn collect_extract(archive: &Archive) -> Result, Error> { + collect_extract_with_password(archive, None) +} + +fn assert_rar50_end_header_has_flags_vint(archive: &Archive) { + let end = archive + .blocks + .iter() + .find_map(|block| match block { + Block::End(header) => Some(header), + _ => None, + }) + .expect("archive has end header"); + assert_eq!(end.block.header_type, 5); + assert_eq!( + end.block.header_size, 3, + "RAR 5 end header must include End of Archive Flags vint" + ); +} + +/// Every volume but the last has to say another one follows. unrar guesses +/// from the file names and does not care, but 7-Zip believes the flag: with it +/// clear it stops at the volume it was handed, and a member continuing into the +/// next one comes out truncated as a data error. +fn assert_volume_set_links_its_parts(parts: &[Vec], password: Option<&[u8]>) { + for (index, part) in parts.iter().enumerate() { + let archive = match password { + Some(password) => { + Archive::parse_with_options(part, ArchiveReadOptions::with_password(password)) + .unwrap() + } + None => Archive::parse(part).unwrap(), + }; + let end = archive + .blocks + .iter() + .find_map(|block| match block { + Block::End(header) => Some(header), + _ => None, + }) + .expect("a volume ends with an end header"); + let last = index + 1 == parts.len(); + assert_eq!( + end.has_next_volume(), + !last, + "volume {} of {} has the wrong next-volume flag", + index + 1, + parts.len() + ); + } +} + +fn collect_extract_with_password( + archive: &Archive, + password: Option<&[u8]>, +) -> Result, Error> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(read_options(password), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time.unwrap_or(0), + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) + .collect()) +} + +fn collect_file(archive: &Archive, file: &rar50::FileHeader) -> Result { + let meta = file.metadata(); + let data = Rc::new(RefCell::new(Vec::new())); + file.write_to( + archive, + None, + &mut CollectWriter { + data: Rc::clone(&data), + }, + )?; + let data = data.borrow().clone(); + Ok(CollectedEntry { + name: meta.name, + data, + file_time: meta.file_time.unwrap_or(0), + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) +} + +fn collect_extract_volumes(archives: &[Archive]) -> Result, Error> { + collect_extract_volumes_with_password(archives, None) +} + +fn collect_extract_volumes_with_password( + archives: &[Archive], + password: Option<&[u8]>, +) -> Result, Error> { + let entries = RefCell::new(Vec::new()); + extract_volumes_to(archives, read_options(password), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + file_time: meta.file_time.unwrap_or(0), + attr: meta.attr, + host_os: meta.host_os, + is_directory: meta.is_directory, + }) + .collect()) +} + +fn read_options(password: Option<&[u8]>) -> ArchiveReadOptions<'_> { + match password { + Some(password) => ArchiveReadOptions::with_password(password), + None => ArchiveReadOptions::default(), + } +} + +fn deterministic_noise(len: usize) -> Vec { + let mut state = 0x1234_5678u32; + (0..len) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect() +} + +fn corrupt_encrypted_stored_padding(bytes: &mut [u8], password: &[u8]) { + let archive = Archive::parse(bytes).unwrap(); + let file = archive.files().next().unwrap(); + let encryption = file.encryption.as_ref().unwrap(); + let keys = Rar50Keys::derive(password, encryption.salt, encryption.kdf_count).unwrap(); + let range = file.block.data_range.clone(); + let mut plaintext = bytes[range.clone()].to_vec(); + Rar50Cipher::new(keys.key, encryption.iv) + .decrypt_in_place(&mut plaintext) + .unwrap(); + assert!(file.unpacked_size < plaintext.len() as u64); + plaintext[file.unpacked_size as usize] = 1; + Rar50Cipher::new(keys.key, encryption.iv) + .encrypt_in_place(&mut plaintext) + .unwrap(); + bytes[range].copy_from_slice(&plaintext); +} + +fn corrupt_encrypted_stored_split_padding(volumes: &mut [Vec], password: &[u8]) { + let archives = volumes + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect::>(); + let first_file = archives[0].files().next().unwrap(); + let final_file = archives.last().unwrap().files().next().unwrap(); + let encryption = first_file.encryption.as_ref().unwrap(); + let keys = Rar50Keys::derive(password, encryption.salt, encryption.kdf_count).unwrap(); + let mut ranges = Vec::with_capacity(archives.len()); + let mut encrypted = Vec::new(); + for (volume_index, archive) in archives.iter().enumerate() { + let file = archive.files().next().unwrap(); + let range = file.block.data_range.clone(); + encrypted.extend_from_slice(&volumes[volume_index][range.clone()]); + ranges.push(range); + } + let mut plaintext = encrypted; + Rar50Cipher::new(keys.key, encryption.iv) + .decrypt_in_place(&mut plaintext) + .unwrap(); + assert!(final_file.unpacked_size < plaintext.len() as u64); + plaintext[final_file.unpacked_size as usize] = 1; + Rar50Cipher::new(keys.key, encryption.iv) + .encrypt_in_place(&mut plaintext) + .unwrap(); + + let mut offset = 0; + for (volume, range) in volumes.iter_mut().zip(ranges) { + let len = range.len(); + volume[range].copy_from_slice(&plaintext[offset..offset + len]); + offset += len; + } +} + +fn level_sensitive_payload() -> Vec { + let long_match = [b"abc".as_slice(), &[b'Z'; 256]].concat(); + let mut data = long_match.clone(); + for index in 0..32u8 { + data.extend_from_slice(b"abc"); + data.push(index); + data.extend_from_slice(&deterministic_noise(24)); + } + data.extend_from_slice(&long_match); + data +} + +fn repair_rev5_volumes( + data_volumes: &[Option<&[u8]>], + recovery_volumes: &[Rev5Volume], +) -> Result>, Error> { + let mut repaired = Vec::new(); + repair_rev5_volumes_to(data_volumes, recovery_volumes, |_, bytes| { + repaired.push(bytes.to_vec()); + Ok(()) + })?; + Ok(repaired) +} + +/// Reads a member's bytes back out, for tests that assert on what they wrote. +fn entry_data(entry: &rar50::ArchiveEntry) -> Vec { + let mut data = Vec::new(); + entry.source.open().unwrap().read_to_end(&mut data).unwrap(); + data +} + +/// Builds a member from bytes the test already holds. +fn entry(name: &[u8], data: &[u8]) -> rar50::ArchiveEntry { + rar50::ArchiveEntry::new( + name.to_vec(), + rars::EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data.to_vec())), + ) +} + +/// Storing is a compression level, not a kind of member. +fn stored(options: rar50::WriterOptions) -> rar50::WriterOptions { + options.with_compression_level(0) +} + +fn write_stored_archive( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, +) -> Result, Error> { + rar50::Rar50Writer::new(stored(options)) + .entries(entries.to_vec()) + .finish() +} + +fn write_stored_archive_with_comment( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + archive_comment: Option<&[u8]>, +) -> Result, Error> { + rar50::Rar50Writer::new(stored(options)) + .entries(entries.to_vec()) + .archive_comment(archive_comment) + .finish() +} + +fn write_stored_archive_with_comment_and_metadata( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + archive_comment: Option<&[u8]>, + archive_metadata: Option>, +) -> Result, Error> { + rar50::Rar50Writer::new(stored(options)) + .entries(entries.to_vec()) + .archive_comment(archive_comment) + .archive_metadata(archive_metadata) + .finish() +} + +fn write_stored_archive_with_recovery( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + recovery_percent: u64, +) -> Result, Error> { + rar50::Rar50Writer::new(stored(options)) + .entries(entries.to_vec()) + .recovery_percent(Some(recovery_percent)) + .finish() +} + +fn write_compressed_archive( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, +) -> Result, Error> { + rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .finish() +} + +fn write_compressed_archive_with_metadata( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + archive_metadata: Option>, +) -> Result, Error> { + rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .archive_metadata(archive_metadata) + .finish() +} + +fn write_compressed_archive_with_comment( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + archive_comment: Option<&[u8]>, +) -> Result, Error> { + rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .archive_comment(archive_comment) + .finish() +} + +fn write_compressed_archive_with_recovery( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + recovery_percent: u64, +) -> Result, Error> { + rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .recovery_percent(Some(recovery_percent)) + .finish() +} + +fn write_compressed_archive_with_filter_policy( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + policy: FilterPolicy, +) -> Result, Error> { + rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .filter_policy(policy) + .finish() +} + +fn write_encrypted_archive_with_comment( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + archive_comment: Option<(&[u8], &[u8])>, + archive_metadata: Option>, +) -> Result, Error> { + let writer = rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .archive_metadata(archive_metadata); + let writer = match archive_comment { + Some((comment, password)) => writer.encrypted_archive_comment(comment, password), + None => writer, + }; + writer.finish() +} + +/// Collects a volume set in memory. The writer still holds one volume at a +/// time; only the test keeps them all. +fn write_volumes( + entries: &[rar50::ArchiveEntry], + options: rar50::WriterOptions, + max_payload_per_volume: u64, + recovery_percent: Option, +) -> Result>, Error> { + let mut sink = rar50::CollectedVolumes::new(); + rar50::write_streaming_volumes_to( + entries, + options, + rar50::ArchiveExtras::default().with_recovery_percent(recovery_percent), + max_payload_per_volume, + &mut sink, + &rars::WriterResources::default(), + )?; + let volumes = sink.take(); + Ok(volumes) +} + +fn assert_rar5_inline_recovery_chunks(data: &[u8]) { + let mut offset = 0; + let mut chunks = 0; + let mut expected_recovery_shards = None; + while offset < data.len() { + assert!(data[offset..].starts_with(b"{RB}")); + let total_size = + u32::from_le_bytes(data[offset + 0x0c..offset + 0x10].try_into().unwrap()) as usize; + let header_size = + u32::from_le_bytes(data[offset + 0x10..offset + 0x14].try_into().unwrap()) as usize; + assert!(total_size >= header_size); + assert!(header_size >= 0x48); + assert!(offset + total_size <= data.len()); + + let chunk = &data[offset..offset + total_size]; + assert_eq!( + u64::from_le_bytes(chunk[0x04..0x0c].try_into().unwrap()), + crc64_xz(&chunk[0x0c..]) + ); + assert_eq!(chunk[0x14], 1); + assert_eq!(chunk[0x15], 1); + + let data_shards = u16::from_le_bytes(chunk[0x3a..0x3c].try_into().unwrap()) as usize; + let recovery_shards = u16::from_le_bytes(chunk[0x3c..0x3e].try_into().unwrap()) as usize; + let shard_index = u16::from_le_bytes(chunk[0x3e..0x40].try_into().unwrap()) as usize; + assert_eq!(header_size, data_shards * 8 + 0x48); + assert_eq!(shard_index, chunks); + assert_eq!( + expected_recovery_shards.get_or_insert(recovery_shards), + &recovery_shards + ); + + chunks += 1; + offset += total_size; + } + assert_eq!(Some(chunks), expected_recovery_shards); +} + +fn read_test_vint(input: &[u8], offset: &mut usize) -> u64 { + let mut value = 0; + for shift in (0..70).step_by(7) { + let byte = input[*offset]; + *offset += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + } + panic!("overlong test vint"); +} + +fn count_verified_quick_open_wrappers(data: &[u8]) -> usize { + let mut offset = 0; + let mut count = 0; + while offset < data.len() { + let expected = u32::from_le_bytes(data[offset..offset + 4].try_into().unwrap()); + offset += 4; + let framed_start = offset; + let body_size = read_test_vint(data, &mut offset) as usize; + let body = &data[offset..offset + body_size]; + // The checksum spans the BlockSize vint as well as the body. + assert_eq!(crc32(&data[framed_start..offset + body_size]), expected); + let mut body_offset = 0; + assert_eq!(read_test_vint(body, &mut body_offset), 0); + assert_ne!(read_test_vint(body, &mut body_offset), 0); + let header_size = read_test_vint(body, &mut body_offset) as usize; + assert_eq!(body.len() - body_offset, header_size); + offset += body_size; + count += 1; + } + count +} + +fn header_encryption_salt_and_first_header_iv(data: &[u8]) -> ([u8; 16], [u8; 16]) { + let mut offset = b"Rar!\x1a\x07\x01\x00".len() + 4; + let header_size = read_test_vint(data, &mut offset) as usize; + let body_start = offset; + let body_end = body_start + header_size; + + assert_eq!(read_test_vint(data, &mut offset), 4); + assert_eq!(read_test_vint(data, &mut offset), 0); + assert_eq!(read_test_vint(data, &mut offset), 0); + assert_eq!(read_test_vint(data, &mut offset), 0x0001); + offset += 1; + + let salt = data[offset..offset + 16].try_into().unwrap(); + let first_header_iv = data[body_end..body_end + 16].try_into().unwrap(); + (salt, first_header_iv) +} + +#[test] +fn detects_rar50_signature_family() { + let bytes = std::fs::read(fixture("stored.rar")).unwrap(); + let sig = detect_archive_family(&bytes).unwrap(); + + assert_eq!(sig.family, ArchiveFamily::Rar50Plus); + assert_eq!(sig.offset, 0); + assert_eq!(sig.length, 8); +} + +#[test] +fn parses_rar7_archive_metadata_main_extra_record() { + let archive = Archive::parse_path(fixture("ams_archive_name_rar721.rar")).unwrap(); + + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!(metadata.flags, 0x0003); + assert_eq!( + metadata.name.as_deref(), + Some(b"ams_archive_name_rar721.rar".as_slice()) + ); + assert_eq!(metadata.creation_time, Some(0x01dcd60e_662d7a32)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"hello7.txt"); + assert_eq!(extracted[0].data, b"Hello, RAR 7.21 fixture world.\n"); +} + +#[test] +fn parses_and_extracts_rar50_stored_file() { + let bytes = std::fs::read(fixture("stored.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.sfx_offset, 0); + assert_eq!(archive.main.archive_flags, 0); + assert_eq!(archive.main.extras.len(), 1); + let locator = archive.main.locator().unwrap(); + assert_eq!(locator.flags, 0x0001); + assert_eq!(locator.quick_open_offset, Some(0)); + assert_eq!(locator.recovery_record_offset, None); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert!(files[0].is_stored()); + assert_eq!(files[0].packed_size(), 30); + assert_eq!(files[0].unpacked_size, 30); + assert_eq!(files[0].data_crc32, Some(0x83b2_7227)); + assert_eq!(files[0].attributes, 0x20); + assert_eq!(files[0].host_os, 0); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"hello.txt"); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn writes_store_only_rar50_archive_that_reader_extracts() { + let entries = [ + entry(b"hello5.txt", b"hello from rars rar5 writer\n") + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3), + entry(b"empty.bin", b"") + .with_attributes(0x20) + .with_host_os(3), + ]; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + assert_eq!(&bytes[..8], b"Rar!\x1a\x07\x01\0"); + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!(archive.main.archive_flags, 0); + assert_rar50_end_header_has_flags_vint(&archive); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert!(files.iter().all(|file| file.is_stored())); + assert!(files.iter().all(|file| file.hash.is_some())); + assert_eq!(files[0].data_crc32, Some(crc32(&entry_data(&entries[0])))); + assert_eq!(files[1].data_crc32, Some(crc32(&entry_data(&entries[1])))); + assert_eq!(files[0].hash.as_ref().unwrap().hash_type, 0); + assert_eq!(files[0].hash.as_ref().unwrap().data.len(), 32); + files[0].verify_hash(&entry_data(&entries[0])).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, entries[0].name); + assert_eq!(extracted[0].data, entry_data(&entries[0])); + assert_eq!(extracted[0].file_time, 0x5a21_0000); + assert_eq!(extracted[1].name, entries[1].name); + assert_eq!(extracted[1].data, entry_data(&entries[1])); +} + +#[test] +fn rar50_writer_builder_writes_stored_archive_with_comment_and_metadata() { + let entries = [entry( + b"builder-stored.txt", + b"stored through the resolved writer builder\n", + ) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = rar50::Rar50Writer::new(rar50::WriterOptions::new(ArchiveVersion::Rar70, features)) + .entries(entries.to_vec()) + .archive_comment(Some(b"builder archive comment")) + .archive_metadata(Some(ArchiveMetadataEntry { + name: Some(b"builder-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + })) + .finish() + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert_eq!( + collect_file(&archive, services[0]).unwrap().data, + b"builder archive comment" + ); + assert_eq!( + archive.main.archive_metadata().unwrap().name.as_deref(), + Some(b"builder-metadata.rar".as_slice()) + ); + assert_eq!( + collect_extract(&archive).unwrap()[0].data, + entry_data(&entries[0]) + ); +} + +#[test] +fn writes_compressed_rar50_archive_that_reader_extracts() { + let entries = [ + entry( + b"compressed.txt", + b"hello from rars rar5 compressed writer\nhello again\n", + ) + .with_mtime(Some(0x5a21_0001)) + .with_attributes(0x20) + .with_host_os(3), + entry(b"empty.bin", b"") + .with_attributes(0x20) + .with_host_os(3), + ]; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert!(files.iter().all(|file| file.hash.is_some())); + assert_eq!(files[0].data_crc32, Some(crc32(&entry_data(&entries[0])))); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, entries[0].name); + assert_eq!(extracted[0].data, entry_data(&entries[0])); + assert_eq!(extracted[0].file_time, 0x5a21_0001); + assert_eq!(extracted[1].name, entries[1].name); + assert_eq!(extracted[1].data, entry_data(&entries[1])); +} + +#[test] +fn compressed_rar50_writer_stores_member_when_lz_payload_would_grow() { + let data = deterministic_noise(8192); + assert!(encode_lz_member(&data, 0).unwrap().len() >= data.len()); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let entries = [entry(b"incompressible.bin", &data) + .with_mtime(Some(0x5a21_00a0)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(target, FeatureSet::store_only()), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.is_stored(), "{target:?}"); + assert_eq!(file.packed_size(), data.len() as u64); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"incompressible.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_00a0); + } +} + +#[test] +fn compressed_rar50_writer_level_zero_stores_member() { + let data = b"level zero stores through compressed writer\n".repeat(64); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let entries = [entry(b"level-zero.txt", &data) + .with_mtime(Some(0x5a21_00a2)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(0), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.is_stored(), "{target:?}"); + assert_eq!(file.packed_size(), data.len() as u64); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); + } +} + +#[test] +fn compressed_rar50_writer_uses_compression_level_for_match_effort() { + let data = level_sensitive_payload(); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let entries = [entry(b"level-sensitive.bin", &data) + .with_mtime(Some(0x5a21_00a4)) + .with_attributes(0x20) + .with_host_os(3)]; + let low = write_compressed_archive( + &entries, + rar50::WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(1), + ) + .unwrap(); + let high = write_compressed_archive( + &entries, + rar50::WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(5), + ) + .unwrap(); + + let low_archive = Archive::parse(&low).unwrap(); + let high_archive = Archive::parse(&high).unwrap(); + let low_file = low_archive.files().next().unwrap(); + let high_file = high_archive.files().next().unwrap(); + assert!( + high_file.packed_size() < low_file.packed_size(), + "{target:?}" + ); + assert_eq!(collect_extract(&low_archive).unwrap()[0].data, data); + assert_eq!(collect_extract(&high_archive).unwrap()[0].data, data); + } +} + +#[test] +fn compressed_rar50_writer_stamps_requested_method_levels() { + let data = level_sensitive_payload(); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for level in 1..=5 { + let entries = [entry(b"level-method.bin", &data) + .with_mtime(Some(0x5a21_00a4)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(target, FeatureSet::store_only()) + .with_compression_level(level), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + assert_eq!(info.algorithm_version, 0, "{target:?} level {level}"); + assert_eq!(info.method, level, "{target:?} level {level}"); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); + } + } +} + +#[test] +fn rar50_writer_builder_writes_filtered_compressed_archive() { + let payload = b"\xe8\0\0\0\0builder filtered payload\n".repeat(8); + let entries = [entry(b"builder-filtered.bin", &payload) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = rar50::Rar50Writer::new(rar50::WriterOptions::new( + ArchiveVersion::Rar50, + FeatureSet::store_only(), + )) + .entries(entries.to_vec()) + .filter_policy(FilterPolicy::explicit(rar50::FilterKind::E8)) + .finish() + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert_eq!( + collect_extract(&archive).unwrap()[0].data, + entry_data(&entries[0]) + ); +} + +#[test] +fn writes_solid_compressed_rar50_archive_that_reader_extracts() { + let first = b"rar50 solid shared phrase alpha beta gamma\n".repeat(32); + let second = b"rar50 solid shared phrase alpha beta gamma\nsecond\n".repeat(8); + let entries = [ + entry(b"solid-one.txt", &first) + .with_mtime(Some(0x5a21_0021)) + .with_attributes(0x20) + .with_host_os(3), + entry(b"solid-two.txt", &second) + .with_mtime(Some(0x5a21_0022)) + .with_attributes(0x20) + .with_host_os(3), + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let solid = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let standalone_second = write_compressed_archive( + std::slice::from_ref(&entries[1]), + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + let archive = Archive::parse(&solid).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert!(archive.main.is_solid()); + assert!(!files[0].decoded_compression_info().unwrap().solid); + assert!(files[1].decoded_compression_info().unwrap().solid); + assert!( + files[1].packed_size() + < Archive::parse(&standalone_second) + .unwrap() + .files() + .next() + .unwrap() + .packed_size() + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn writes_delta_filtered_compressed_rar50_archive_that_reader_extracts() { + let data: Vec = (0..180) + .map(|index| (index * 5 + index / 2) as u8) + .collect(); + let entries = [entry(b"delta-filtered.bin", &data) + .with_mtime(Some(0x5a21_0023)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive_with_filter_policy( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + FilterPolicy::explicit(FilterKind::Delta { channels: 3 }), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.decoded_compression_info().unwrap().method, 3); + let packed = file.packed_data(&archive).unwrap(); + let block = parse_compressed_block(&packed).unwrap(); + let (lengths, _) = read_table_lengths(&packed[block.payload], 0).unwrap(); + assert_ne!(lengths.main[256], 0); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"delta-filtered.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_0023); +} + +#[test] +fn writes_e8_filtered_compressed_rar50_archive_that_reader_extracts() { + let data = b"\xe8\0\0\0\0rar5 e8 filtered call payload".to_vec(); + let entries = [entry(b"e8-filtered.bin", &data) + .with_mtime(Some(0x5a21_0024)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive_with_filter_policy( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + FilterPolicy::explicit(FilterKind::E8), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let packed = file.packed_data(&archive).unwrap(); + let block = parse_compressed_block(&packed).unwrap(); + let (lengths, _) = read_table_lengths(&packed[block.payload], 0).unwrap(); + assert_ne!(lengths.main[256], 0); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"e8-filtered.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_0024); +} + +#[test] +fn writes_e8e9_filtered_compressed_rar50_archive_that_reader_extracts() { + let data = b"\xe9\0\0\0\0rar5 e8e9 filtered jump payload".to_vec(); + let entries = [entry(b"e8e9-filtered.bin", &data) + .with_mtime(Some(0x5a21_0025)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive_with_filter_policy( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + FilterPolicy::explicit(FilterKind::E8E9), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"e8e9-filtered.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_0025); +} + +#[test] +fn writes_arm_filtered_compressed_rar50_archive_that_reader_extracts() { + let data = [0x04, 0x00, 0x00, 0xeb, b'A', b'R', b'M', b'!']; + let entries = [entry(b"arm-filtered.bin", &data) + .with_mtime(Some(0x5a21_0026)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive_with_filter_policy( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + FilterPolicy::explicit(FilterKind::Arm), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + let packed = file.packed_data(&archive).unwrap(); + let block = parse_compressed_block(&packed).unwrap(); + let (lengths, _) = read_table_lengths(&packed[block.payload], 0).unwrap(); + assert_ne!(lengths.main[256], 0); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"arm-filtered.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_0026); +} + +#[test] +fn writes_auto_filtered_compressed_rar50_archive_that_reader_extracts() { + let mut data = Vec::new(); + for _ in 0..48 { + data.extend_from_slice(b"\xe8\0\0\0\0rar5 auto filter policy payload\n"); + } + let entries = [entry(b"auto-filtered.bin", &data) + .with_mtime(Some(0x5a21_0027)) + .with_attributes(0x20) + .with_host_os(3)]; + let options = rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()); + let plain = write_compressed_archive(&entries, options).unwrap(); + let auto = + write_compressed_archive_with_filter_policy(&entries, options, FilterPolicy::Auto).unwrap(); + + let plain_archive = Archive::parse(&plain).unwrap(); + let auto_archive = Archive::parse(&auto).unwrap(); + assert!( + auto_archive.files().next().unwrap().packed_size() + <= plain_archive.files().next().unwrap().packed_size() + ); + let extracted = collect_extract(&auto_archive).unwrap(); + assert_eq!(extracted[0].name, b"auto-filtered.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_0027); +} + +#[test] +fn auto_filtered_compressed_rar50_writer_accepts_empty_member() { + let entries = [entry(b"afile.txt", b"") + .with_mtime(Some(0x5a21_0028)) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive_with_filter_policy( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + FilterPolicy::Auto, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.name, b"afile.txt"); + assert!(file.is_stored()); + assert_eq!(file.unpacked_size, 0); + assert_eq!(file.packed_size(), 0); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"afile.txt"); + assert!(extracted[0].data.is_empty()); +} + +#[test] +fn writes_compressed_rar50_volume_set_that_reader_reassembles() { + let payload = b"rar5 compressed split payload from rars\n".repeat(24); + let entry = entry(b"compressed-split.txt", &payload) + .with_mtime(Some(0x5a21_0002)) + .with_attributes(0x20) + .with_host_os(3); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + for archive in &archives { + assert_rar50_end_header_has_flags_vint(archive); + } + assert!(archives.iter().all(|archive| archive.main.is_volume())); + let first = archives[0].files().next().unwrap(); + let last = archives.last().unwrap().files().next().unwrap(); + assert!(first.is_split_after()); + assert!(last.is_split_before()); + assert!(!last.is_split_after()); + assert_eq!(last.decoded_compression_info().unwrap().method, 3); + assert!(last.hash.is_some()); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"compressed-split.txt"); + assert_eq!(extracted[0].data, payload); + assert_eq!(extracted[0].file_time, 0x5a21_0002); +} + +/// A fragment that is not the last one has no whole-member checksum to report, +/// so it reports the bytes it stores instead, the way WinRAR does. Both a CRC32 +/// and a hash record go on every fragment: a reader takes the hash record as +/// the member's checksum type, and unrar reads that type from the first +/// fragment and compares it against the last, so a first fragment offering only +/// a CRC32 fails a member that is perfectly intact. +#[test] +fn every_rar50_volume_fragment_checksums_the_bytes_it_stores() { + let payload = deterministic_noise(4096); + let entry = entry(b"fragment-checksums.bin", &payload).with_mtime(Some(0x5a21_0031)); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 1024, + None, + ) + .unwrap(); + assert!(parts.len() >= 3); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let member_crc = crc32(&payload); + for (index, archive) in archives.iter().enumerate() { + let file = archive.files().next().unwrap(); + assert!( + file.hash.is_some(), + "volume {} has no hash record", + index + 1 + ); + let stored = &parts[index][file.block.data_range.clone()]; + match file.is_split_after() { + true => { + assert_eq!( + file.data_crc32, + Some(crc32(stored)), + "volume {} does not checksum its own bytes", + index + 1 + ); + assert_ne!(file.data_crc32, Some(member_crc)); + } + false => assert_eq!(file.data_crc32, Some(member_crc)), + } + } +} + +/// unrar and RAR 7.12 both read a set with a damaged middle volume to the end +/// and then report one checksum error for the member, which says nothing about +/// where the damage is. The fragment checksums are enough to say, so say it. +#[test] +fn a_damaged_rar50_volume_is_reported_by_its_number() { + // Compressible enough that the member is packed rather than stored, and + // large enough that the packed bytes still need several volumes. + let payload = deterministic_noise(2048).repeat(8); + let entry = entry(b"damaged-volume.bin", &payload).with_mtime(Some(0x5a21_0032)); + let mut parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 512, + None, + ) + .unwrap(); + assert!(parts.len() >= 3); + let last = Archive::parse(parts.last().unwrap()).unwrap(); + assert_eq!( + last.files() + .next() + .unwrap() + .decoded_compression_info() + .unwrap() + .method, + 3 + ); + + let damaged = 1; + let range = Archive::parse(&parts[damaged]) + .unwrap() + .files() + .next() + .unwrap() + .block + .data_range + .clone(); + parts[damaged][range.start] ^= 0xff; + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let error = collect_extract_volumes(&archives).unwrap_err(); + let mut source = &error; + while let Error::AtEntry { source: inner, .. } | Error::AtArchiveOffset { source: inner, .. } = + source + { + source = inner; + } + assert!( + matches!(source, Error::InVolume { number, source } + if *number == damaged + 1 && matches!(**source, Error::Crc32Mismatch { .. })), + "expected volume {} to be named, got {error}", + damaged + 1 + ); +} + +#[test] +fn compressed_rar50_volume_writer_stores_member_when_lz_payload_would_grow() { + let data = deterministic_noise(8192); + assert!(encode_lz_member(&data, 0).unwrap().len() >= data.len()); + let entry = entry(b"incompressible-split.bin", &data) + .with_mtime(Some(0x5a21_00a2)) + .with_attributes(0x20) + .with_host_os(3); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 1024, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives + .iter() + .all(|archive| archive.files().next().unwrap().is_stored())); + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"incompressible-split.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_00a2); +} + +#[test] +fn writes_compressed_rar50_volume_set_with_recovery_records() { + let payload = b"rar5 compressed recovery split payload from rars\n".repeat(24); + let entry = entry(b"compressed-split-rr.txt", &payload) + .with_mtime(Some(0x5a21_0002)) + .with_attributes(0x20) + .with_host_os(3); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + Some(8), + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives + .iter() + .all(|archive| archive.main.has_recovery_record())); + for archive in &archives { + let service = archive.services().next().unwrap(); + assert_eq!(service.name_lossy(), "RR"); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 8); + assert_rar5_inline_recovery_chunks(&service.packed_data(archive).unwrap()); + } + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"compressed-split-rr.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_solid_compressed_rar50_volume_set_that_reader_reassembles() { + let payload = b"rar5 solid compressed split payload from rars\n".repeat(24); + let entry = entry(b"solid-compressed-split.txt", &payload) + .with_mtime(Some(0x5a21_0003)) + .with_attributes(0x20) + .with_host_os(3); + let mut features = FeatureSet::store_only(); + features.solid = true; + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let first = archives[0].files().next().unwrap(); + let last = archives.last().unwrap().files().next().unwrap(); + assert!(first.is_split_after()); + assert!(last.is_split_before()); + assert!(!last.is_split_after()); + assert!(!last.decoded_compression_info().unwrap().solid); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"solid-compressed-split.txt"); + assert_eq!(extracted[0].data, payload); + assert_eq!(extracted[0].file_time, 0x5a21_0003); +} + +#[test] +fn writes_multi_file_solid_compressed_rar50_volume_set_that_reader_reassembles() { + let first = b"rar5 multi-file solid split shared phrase alpha beta gamma\n".repeat(20); + let second = b"rar5 multi-file solid split shared phrase alpha beta gamma\nsecond\n".repeat(16); + let entries = [ + entry(b"solid-split-one.txt", &first) + .with_mtime(Some(0x5a21_0011)) + .with_attributes(0x20) + .with_host_os(3), + entry(b"solid-split-two.txt", &second) + .with_mtime(Some(0x5a21_0012)) + .with_attributes(0x20) + .with_host_os(3), + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let parts = write_volumes( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let files: Vec<_> = archives + .iter() + .flat_map(|archive| archive.files()) + .collect(); + assert!(files.iter().any(|file| file.name == b"solid-split-two.txt" + && file.decoded_compression_info().unwrap().solid)); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"solid-split-one.txt"); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[0].file_time, 0x5a21_0011); + assert_eq!(extracted[1].name, b"solid-split-two.txt"); + assert_eq!(extracted[1].data, second); + assert_eq!(extracted[1].file_time, 0x5a21_0012); +} + +#[test] +fn writes_rar50_archive_comment_service_record() { + let entries = [entry(b"payload.txt", b"payload with comment service\n") + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some(b"RAR5 comment from rars\n"), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert!(services[0].is_stored()); + assert_eq!(services[0].service_data, Some(Vec::new())); + let comment = collect_file(&archive, services[0]).unwrap(); + assert_eq!(comment.name, b"CMT"); + assert_eq!(comment.data, b"RAR5 comment from rars\n"); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, entries[0].name); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_compressed_rar50_archive_comment_service_record() { + let payload = b"compressed payload with archive comment\n".repeat(8); + let entries = [entry(b"compressed-comment.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_compressed_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some(b"compressed RAR5 comment from rars\n"), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["CMT"]); + assert_eq!( + collect_file(&archive, services[0]).unwrap().data, + b"compressed RAR5 comment from rars\n" + ); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].name, b"compressed-comment.txt"); + assert_eq!(extracted[0].data, payload); +} + +/// What `supports` promises for the quick-open index is what each writer does +/// with it, for every shape RAR 5 has. +/// +/// The capability conformance test in `write_plan.rs` proves the table and +/// `validate_plan` agree, which is not the same as proving the writer agrees +/// with either. Quick open was said in two places, `FeatureSet::quick_open` and +/// an `ArchiveExtras` field, and validation only ever read the first: an archive +/// asking through the feature set passed validation and then had its index +/// dropped, and a volume set asking through the extras got past validation +/// entirely and was dropped too. Both looked like success. +/// +/// So this asks each writer for the index and goes looking for the block, +/// rather than asking the table what it thinks. +#[test] +fn every_rar50_writer_agrees_with_the_table_about_quick_open() { + fn wrote_the_index(bytes: &[u8]) -> bool { + let archive = Archive::parse(bytes).unwrap(); + let located = archive + .main + .locator() + .and_then(|locator| locator.quick_open_offset) + .is_some_and(|offset| offset > 0); + let stored = archive.services().any(|service| service.name == b"QO"); + assert_eq!( + located, stored, + "a locator pointing at no QO record, or a QO record nothing points at" + ); + located + } + + let mut features = FeatureSet::store_only(); + features.quick_open = true; + let options = rar50::WriterOptions::new(ArchiveVersion::Rar50, features); + let entries = [entry(b"qo.txt", b"quick-open conformance payload\n") + .with_attributes(0x20) + .with_host_os(3)]; + + let one_archive = rars::PlanShape::new().compressed(true); + assert!(rars::supports( + ArchiveVersion::Rar50, + rars::WriterOption::Feature(rars::Feature::QuickOpen), + one_archive, + )); + let bytes = write_stored_archive(&entries, options).unwrap(); + assert!(wrote_the_index(&bytes), "the buffered writer dropped it"); + + let mut streamed = Vec::new(); + rar50::write_streaming_archive_to( + &entries, + stored(options), + rar50::ArchiveExtras::default(), + &rars::WriterResources::default(), + &mut streamed, + ) + .unwrap(); + assert!( + wrote_the_index(&streamed), + "the streaming writer accepted the feature and wrote no index" + ); + + let volumes = rars::PlanShape::new().compressed(true).volumes(true); + assert!(!rars::supports( + ArchiveVersion::Rar50, + rars::WriterOption::Feature(rars::Feature::QuickOpen), + volumes, + )); + let mut sink = rar50::CollectedVolumes::new(); + let refused = rar50::write_streaming_volumes_to( + &entries, + stored(options), + rar50::ArchiveExtras::default(), + 4096, + &mut sink, + &rars::WriterResources::default(), + ) + .unwrap_err(); + assert!( + refused.to_string().contains("quick-open"), + "a volume set has to say which option it cannot carry: {refused}" + ); +} + +#[test] +fn writes_rar50_quick_open_service_record() { + let entries = [ + entry(b"first.txt", b"first quick-open payload\n") + .with_attributes(0x20) + .with_host_os(3), + entry(b"second.txt", b"second quick-open payload\n") + .with_attributes(0x20) + .with_host_os(3), + ]; + let mut features = FeatureSet::store_only(); + features.quick_open = true; + let bytes = write_stored_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some(b"QO comment from rars\n"), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let locator = archive.main.locator().unwrap(); + assert!(locator.quick_open_offset.unwrap() > 0); + assert_eq!(service_names(&archive), ["CMT", "QO"]); + let quick_open = archive + .services() + .find(|service| service.name == b"QO") + .unwrap(); + assert!(quick_open.is_stored()); + let quick_open = collect_file(&archive, quick_open).unwrap(); + assert_eq!(count_verified_quick_open_wrappers(&quick_open.data), 3); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); + assert_eq!(extracted[1].data, entry_data(&entries[1])); +} + +#[test] +fn writes_rar50_acl_and_stream_file_service_records() { + let entries = [entry(b"serviced.txt", b"payload with attached services\n") + .with_attributes(0x20) + .with_host_os(3) + .with_service(ServiceEntry::new( + b"ACL".to_vec(), + b"opaque acl descriptor".to_vec(), + )) + .with_service(ServiceEntry::new( + b"STM".to_vec(), + b"named stream bytes".to_vec(), + ))]; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["ACL", "STM"]); + assert!(services.iter().all(|service| service.is_stored())); + assert_eq!( + collect_file(&archive, services[0]).unwrap().data, + b"opaque acl descriptor" + ); + assert_eq!( + collect_file(&archive, services[1]).unwrap().data, + b"named stream bytes" + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"serviced.txt"); + assert_eq!(extracted[0].data, b"payload with attached services\n"); +} + +#[test] +fn writes_rar50_file_comment_service_record() { + let entries = [entry( + b"file-commented.txt", + b"payload with attached file comment\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_service(ServiceEntry::new( + b"CMT".to_vec(), + b"RAR5 file comment from rars\n".to_vec(), + ))]; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["CMT"]); + assert!(services[0].is_stored()); + assert_eq!( + collect_file(&archive, services[0]).unwrap().data, + b"RAR5 file comment from rars\n" + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"file-commented.txt"); + assert_eq!(extracted[0].data, b"payload with attached file comment\n"); +} + +/// When a file header carries both a `Data CRC32` and a BLAKE2sp record, the +/// hash is the authoritative check and the CRC32 is not evaluated beside it. +/// +/// Measured on RAR 7.12 and UnRAR 7.20: the fixture's CRC32 is wrong by one bit +/// while its BLAKE2sp is correct, and both readers test it clean. Corrupt the +/// digest instead and both reject it, which is what the second half pins. +#[test] +fn rar50_blake2sp_supersedes_a_wrong_crc32() { + let data = fs::read(fixture("crc32_wrong_beside_blake2sp.rar")).unwrap(); + let archive = Archive::parse(&data).unwrap(); + + let file = archive.files().next().unwrap(); + assert_eq!(file.data_crc32, Some(0x83b2_7226)); + assert!(file.hash.is_some()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); + + // The digest still has to be right. Flip a byte of it and the archive must + // fail, otherwise the fix would have turned off checking altogether. + const HEAD_CRC: usize = 0x17; + const HEAD_SIZE: usize = 0x1b; + const BODY: usize = 0x1c; + const DIGEST: usize = 0x39; + + let mut corrupted = data.clone(); + corrupted[DIGEST] ^= 0xff; + let header_end = BODY + corrupted[HEAD_SIZE] as usize; + let header_crc = crc32(&corrupted[HEAD_SIZE..header_end]); + corrupted[HEAD_CRC..HEAD_CRC + 4].copy_from_slice(&header_crc.to_le_bytes()); + let archive = Archive::parse(&corrupted).unwrap(); + assert!(matches!( + collect_extract(&archive), + Err(Error::AtEntry { source, .. }) if matches!(*source, Error::HashMismatch { .. }) + )); +} + +/// Reference builds keep 127 characters of the password and drop the rest, so +/// rars has to as well: measured against RAR 7.12, an archive rars wrote with a +/// 128-character password could not be opened with that password or with any +/// prefix of it, and rars could not open WinRAR's when handed the same long one. +/// +/// The archive here is written with a 200-character password and opened with +/// its first 127 characters. Both must derive the same key, and 126 must not. +#[test] +fn rar50_password_is_clamped_to_127_characters() { + let long = "A".repeat(197) + "ZZZ"; + let kept = &long[..127]; + let short = &long[..126]; + + let entries = [entry(b"secret.txt", b"clamped password payload\n") + .with_attributes(0x20) + .with_host_os(3) + .with_password(long.as_bytes().to_vec())]; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + for password in [long.as_str(), kept] { + let archive = Archive::parse_with_password(&bytes, Some(password.as_bytes())).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, b"clamped password payload\n"); + } + + assert!( + Archive::parse_with_password(&bytes, Some(short.as_bytes())).is_err(), + "126 characters must not open an archive written with 127 or more" + ); +} + +#[test] +fn writes_encrypted_rar50_file_comment_service_record() { + let entries = [entry( + b"encrypted-file-commented.txt", + b"encrypted payload with attached file comment\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec()) + .with_service( + ServiceEntry::new( + b"CMT".to_vec(), + b"encrypted RAR5 file comment from rars\n".to_vec(), + ) + .with_password(b"secret".to_vec()), + )]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["CMT"]); + assert!(services[0].encrypted); + assert!(matches!( + collect_file(&archive, services[0]), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"CMT" && matches!(*source, Error::NeedPassword) + )); + + let archive = Archive::parse_with_password(&bytes, Some(b"secret")).unwrap(); + let service = collect_file(&archive, archive.services().next().unwrap()).unwrap(); + assert_eq!(service.data, b"encrypted RAR5 file comment from rars\n"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"encrypted-file-commented.txt"); + assert_eq!( + extracted[0].data, + b"encrypted payload with attached file comment\n" + ); +} + +#[test] +fn writes_header_encrypted_rar50_file_comment_service_record() { + let entries = [entry( + b"header-encrypted-file-commented.txt", + b"header encrypted payload with attached file comment\n", + ) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec()) + .with_service( + ServiceEntry::new( + b"CMT".to_vec(), + b"header encrypted RAR5 file comment from rars\n".to_vec(), + ) + .with_password(b"secret".to_vec()), + )]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + // The HEAD_CRYPT block is in the clear and carries its own KDF count, + // which has to be the same 2^15 as the per-file records below. + assert_eq!(head_crypt_kdf_count(&bytes), 15); + let archive = Archive::parse_with_password(&bytes, Some(b"secret")).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["CMT"]); + assert!(services[0].encrypted); + assert_eq!( + collect_file(&archive, services[0]).unwrap().data, + b"header encrypted RAR5 file comment from rars\n" + ); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!( + extracted[0].data, + b"header encrypted payload with attached file comment\n" + ); +} + +#[test] +#[ignore = "requires local rar command; used by scripts/reference-rar5-writer.sh"] +fn reference_rar_accepts_rar50_acl_and_stream_file_service_records() { + let entries = [entry(b"serviced.txt", b"payload with attached services\n") + .with_attributes(0x20) + .with_host_os(3) + .with_service(ServiceEntry::new( + b"ACL".to_vec(), + b"opaque acl descriptor".to_vec(), + )) + .with_service(ServiceEntry::new( + b"STM".to_vec(), + b"named stream bytes".to_vec(), + ))]; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + ) + .unwrap(); + + let dir = scratch::case("rars-rar50-file-services"); + let path = dir.join("archive.rar"); + fs::write(&path, bytes).unwrap(); + let output = match Command::new("rar").arg("t").arg(&path).output() { + Ok(output) => output, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + eprintln!("skipping reference test: local `rar` command is not installed"); + return; + } + Err(error) => panic!("failed to run rar: {error}"), + }; + if std::env::var_os("RARS_KEEP_REFERENCE_ARCHIVE").is_some() { + eprintln!("kept reference archive: {}", path.display()); + std::mem::forget(dir); + } + + assert!( + output.status.success(), + "rar rejected RAR5 ACL/STM service output\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +#[ignore = "requires RARS_RAR50_LARGE_STREAM_FIXTURE pointing at an external >512 MiB sparse unfiltered RAR5 fixture"] +fn external_sparse_rar50_large_member_streams_to_sink() { + let Some(path) = std::env::var_os("RARS_RAR50_LARGE_STREAM_FIXTURE") else { + eprintln!( + "skipping reference test: set RARS_RAR50_LARGE_STREAM_FIXTURE to an external sparse RAR5 fixture" + ); + return; + }; + let archive = Archive::parse_path(path).unwrap(); + let largest = archive + .files() + .map(|file| file.unpacked_size) + .max() + .unwrap_or(0); + assert!( + largest > 512 * 1024 * 1024, + "fixture must contain a member larger than the filtered-buffer fallback" + ); + + archive + .extract_to(read_options(None), |_meta| Ok(Box::new(std::io::sink()))) + .unwrap(); +} + +#[test] +fn writes_rar50_recovery_service_record() { + let entries = [entry( + b"recoverable.txt", + b"payload with structural recovery service\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 7, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_recovery_record()); + let locator = archive.main.locator().unwrap(); + let recovery_offset = locator.recovery_record_offset.unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["RR"]); + assert_eq!( + recovery_offset, + (services[0].block.offset - b"Rar!\x1a\x07\x01\x00".len()) as u64 + ); + let recovery = services[0].recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 7); + assert_eq!(recovery.payload_size, services[0].packed_size()); + let recovery_data = services[0].packed_data(&archive).unwrap(); + assert!(recovery_data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(recovery_data[0x0c..0x10].try_into().unwrap()) as usize, + recovery_data.len() + ); + assert_eq!( + u32::from_le_bytes(recovery_data[0x10..0x14].try_into().unwrap()) as usize, + 0x48 + 8 + ); + assert_eq!(recovery_data[0x14], 1); + assert_eq!(recovery_data[0x15], 1); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"recoverable.txt"); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn repairs_rar50_inline_recovery_payload_damage() { + let payload = b"payload with structural recovery service\n".repeat(64); + let entries = [entry(b"recoverable.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let data_range = clean.files().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[data_range.start + 10..data_range.start + 80].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let mut repaired = Vec::new(); + damaged_archive.repair_recovery_to(&mut repaired).unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse(&repaired).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn rejects_rar50_inline_recovery_when_all_recovery_chunks_are_damaged() { + let bytes = write_stored_archive_with_recovery( + &[entry(b"recoverable.txt", b"payload protected by recovery")], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let recovery_range = archive.services().next().unwrap().block.data_range.clone(); + let mut damaged = bytes; + damaged[recovery_range].fill(0); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(matches!( + damaged_archive.repair_recovery(), + Err(Error::Rar5Recovery( + rars::recovery::rar5::Error::BadRecoveryChunk + )) + )); +} + +#[test] +fn rar50_recovery_rejects_a_public_service_offset_past_the_archive() { + let bytes = write_stored_archive_with_recovery( + &[entry(b"recoverable.txt", b"payload protected by recovery")], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let mut archive = Archive::parse(&bytes).unwrap(); + let service = archive + .blocks + .iter_mut() + .find_map(|block| match block { + rar50::Block::Service(service) => Some(service), + _ => None, + }) + .unwrap(); + service.block.offset = bytes.len() + 1; + + assert!(matches!( + archive.repair_recovery(), + Err(Error::InvalidHeader( + "RAR 5 recovery prefix is out of bounds" + )) + )); + + let mut archive = Archive::parse(&bytes).unwrap(); + archive.sfx_offset = bytes.len() + 1; + assert!(matches!( + archive.repair_recovery(), + Err(Error::InvalidHeader( + "RAR 5 recovery prefix range overflows archive bounds" + )) + )); +} + +#[test] +fn rejects_rar50_inline_recovery_when_damage_exceeds_available_shards() { + let bytes = write_stored_archive_with_recovery( + &[entry(b"recoverable.txt", &vec![b'A'; 32 * 1024])], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 5, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let data_range = archive.files().next().unwrap().block.data_range.clone(); + let mut damaged = bytes; + damaged[data_range].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(matches!( + damaged_archive.repair_recovery(), + Err(Error::Rar5Recovery( + rars::recovery::rar5::Error::TooManyDamagedShards + )) + )); +} + +#[test] +fn rar50_recovery_reports_output_failure_before_a_late_repaired_shard() { + struct FailingWriter; + + impl Write for FailingWriter { + fn write(&mut self, _: &[u8]) -> IoResult { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } + } + + let bytes = write_stored_archive_with_recovery( + &[entry(b"recoverable.txt", &vec![b'A'; 32 * 1024])], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let data_range = archive.files().next().unwrap().block.data_range.clone(); + let mut damaged = bytes; + damaged[data_range.end - 64..data_range.end - 32].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + let error = damaged_archive + .repair_recovery_to(&mut FailingWriter) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); +} + +#[test] +fn rar50_recovery_observes_cancellation_from_output_in_both_repair_paths() { + struct CancellingWriter<'a>(&'a rars::ReadCancellation); + impl Write for CancellingWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> IoResult { + self.0.cancel(); + Ok(bytes.len()) + } + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } + } + + let bytes = write_stored_archive_with_recovery( + &[entry(b"recoverable.txt", &vec![b'A'; 32 * 1024])], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let recovery_range = archive.services().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[recovery_range.start + 0x48] ^= 1; + + for source in [&bytes[..], &damaged[..]] { + let archive = Archive::parse(source).unwrap(); + let token = rars::ReadCancellation::new(); + let options = rars::ArchiveReadOptions::new().with_cancellation(&token); + assert!(matches!( + archive.repair_recovery_to_with_options(&mut CancellingWriter(&token), options), + Err(Error::Cancelled) + )); + assert!(token.is_cancelled()); + } +} + +#[test] +fn repairs_sfx_prefixed_rar50_data_with_a_missing_recovery_row() { + let archive_bytes = write_stored_archive_with_recovery( + &[entry(b"recoverable.txt", &vec![b'A'; 32 * 1024])], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let mut prefixed = b"MZ SFX stub".to_vec(); + prefixed.extend_from_slice(&archive_bytes); + let clean = Archive::parse(&prefixed).unwrap(); + assert_eq!(clean.sfx_offset, b"MZ SFX stub".len()); + let data_range = clean.files().next().unwrap().block.data_range.clone(); + let recovery_range = clean.services().next().unwrap().block.data_range.clone(); + let mut damaged = prefixed.clone(); + damaged[data_range.start + 128..data_range.start + 160].fill(0xa5); + damaged[recovery_range.start + 0x48] ^= 0xff; + let mut expected = prefixed; + expected[recovery_range.start + 0x48] ^= 0xff; + + let result = Archive::parse(&damaged) + .unwrap() + .repair_recovery_with_report(None) + .unwrap(); + assert!(result.report.data_repaired); + assert!(!result.report.recovery_record_rebuilt); + assert_eq!(result.data, expected); + assert_eq!( + collect_extract(&Archive::parse(&result.data).unwrap()).unwrap()[0].data, + vec![b'A'; 32 * 1024] + ); +} + +#[test] +fn repairs_rar50_inline_recovery_header_damage_without_parsing() { + let payload = b"payload protected by raw inline recovery fallback\n".repeat(64); + let entries = [entry(b"header-damaged.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let file_header_offset = clean.files().next().unwrap().block.offset; + let mut damaged = bytes.clone(); + damaged[file_header_offset] ^= 0xff; + + assert!(Archive::parse(&damaged).is_err()); + + let repaired = repair_inline_recovery_bytes(&damaged).unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse(&repaired).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn rejects_rar50_inline_recovery_service_header_damage_without_prefix_damage() { + let payload = b"payload with only the recovery service header damaged\n".repeat(64); + let entries = [entry(b"rr-header-damaged.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let service_offset = clean.services().next().unwrap().block.offset; + let mut damaged = bytes.clone(); + damaged[service_offset] ^= 0xff; + + assert!(Archive::parse(&damaged).is_err()); + assert!(repair_inline_recovery_bytes(&damaged).is_err()); +} + +#[test] +fn repairs_rar50_inline_recovery_with_damaged_recovery_chunk() { + let payload = b"payload with a damaged recovery shard still recoverable\n".repeat(512); + let entries = [entry(b"recoverable-with-bad-rr.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let clean = Archive::parse(&bytes).unwrap(); + let data_range = clean.files().next().unwrap().block.data_range.clone(); + let recovery_range = clean.services().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[data_range.start + 10..data_range.start + 80].fill(0xa5); + damaged[recovery_range.start + 0x48] ^= 0xff; + + let damaged_archive = Archive::parse(&damaged).unwrap(); + assert!(collect_extract(&damaged_archive).is_err()); + + let repaired = damaged_archive.repair_recovery_with_report(None).unwrap(); + + assert!(repaired.report.data_repaired); + assert!(repaired.report.recovery_record_rebuilt); + assert_eq!( + repaired.report.available_recovery_shards, + repaired + .report + .expected_recovery_shards + .map(|count| count - 1) + ); + assert_eq!(repaired.data, bytes); + let repaired_archive = Archive::parse(&repaired.data).unwrap(); + let extracted = collect_extract(&repaired_archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn rebuilds_damaged_recovery_chunk_without_claiming_data_repair() { + let payload = b"healthy data with one damaged recovery row\n".repeat(1024); + let entries = [entry(b"healthy-data.txt", &payload)]; + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let recovery_range = archive.services().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[recovery_range.start + 0x48] ^= 0xff; + + let result = Archive::parse(&damaged) + .unwrap() + .repair_recovery_with_report(None) + .unwrap(); + + assert!(!result.report.data_repaired); + assert!(result.report.recovery_record_rebuilt); + assert!(result.report.changed); + assert_eq!(result.data, bytes); +} + +#[test] +fn rebuilds_truncated_rar50_inline_recovery_tail() { + let payload = b"payload protected when the recovery tail is truncated\n".repeat(4096); + let entries = [entry(b"truncated-recovery.txt", &payload)]; + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let mut damaged = bytes.clone(); + damaged.truncate(damaged.len() - 4096); + + assert!(Archive::parse(&damaged).is_err()); + let repaired = repair_inline_recovery_bytes(&damaged).unwrap(); + + assert_eq!(repaired, bytes); + let archive = Archive::parse(&repaired).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, payload); +} + +#[test] +fn rebuilds_a_damaged_chunk_in_an_archive_storing_a_recovery_archive() { + // The stored inner archive carries its own {RB} chunks, which sit earlier + // in the file than the outer record's. + let inner = write_stored_archive_with_recovery( + &[entry( + b"inner-payload.txt", + &b"nested archive payload\n".repeat(2048), + )], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let bytes = write_stored_archive_with_recovery( + &[entry(b"inner.rar", &inner)], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + 20, + ) + .unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let recovery_range = archive.services().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[recovery_range.start + 0x48] ^= 0xff; + + let repaired = Archive::parse(&damaged) + .unwrap() + .repair_recovery_with_report(None) + .unwrap(); + + assert!(repaired.report.recovery_record_rebuilt); + assert_eq!(repaired.data, bytes); +} + +#[test] +fn rebuilds_a_header_encrypted_archive_end_from_a_parsed_archive() { + // Framing a replacement end-of-archive header for a header-encrypted + // archive needs the key, so the repair has to be given the password even + // though the archive itself parsed. + let payload = b"header encrypted, damaged row, missing tail\n".repeat(2048); + let entries = [entry(b"secret-recovery.txt", &payload).with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let recovery_range = archive.services().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged.truncate(recovery_range.end); + damaged[recovery_range.start + 0x48] ^= 0xff; + + let repaired = Archive::parse_with_password(&damaged, Some(b"password")) + .unwrap() + .repair_recovery_with_report(Some(b"password")) + .unwrap(); + + assert!(repaired.report.recovery_record_rebuilt); + assert!(repaired.report.end_record_rebuilt); + // Everything up to the end of the record comes back byte for byte. The + // replacement end header cannot, since each encrypted block is framed with + // a fresh initialisation vector. + assert_eq!( + repaired.data[..recovery_range.end], + bytes[..recovery_range.end] + ); + let archive = Archive::parse_with_password(&repaired.data, Some(b"password")).unwrap(); + assert_eq!( + collect_extract_with_password(&archive, Some(b"password")).unwrap()[0].data, + payload + ); +} + +#[test] +fn rebuilds_truncated_header_encrypted_rar50_recovery_tail() { + let payload = b"encrypted headers with a truncated recovery tail\n".repeat(4096); + let entries = + [entry(b"secret-truncated-recovery.txt", &payload).with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let mut damaged = bytes.clone(); + damaged.truncate(damaged.len() - 4096); + + let repaired = rars::rar50::repair_inline_recovery_bytes_with_options( + &damaged, + rars::ArchiveReadOptions::with_password(b"password"), + ) + .unwrap(); + + assert!(repaired.report.recovery_record_rebuilt); + assert!(repaired.report.end_record_rebuilt); + let archive = Archive::parse_with_password(&repaired.data, Some(b"password")).unwrap(); + assert_eq!( + collect_extract_with_password(&archive, Some(b"password")).unwrap()[0].data, + payload + ); +} + +#[test] +fn repairs_encrypted_rar50_inline_recovery_payload_damage_with_password() { + let payload = b"encrypted payload with structural recovery service\n".repeat(64); + let entries = [entry(b"secret-recoverable.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let clean = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let data_range = clean.files().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[data_range.start + 16..data_range.start + 96].fill(0xa5); + + let damaged_archive = Archive::parse_with_password(&damaged, Some(b"password")).unwrap(); + assert!(collect_extract_with_password(&damaged_archive, Some(b"password")).is_err()); + + let repaired = damaged_archive.repair_recovery().unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse_with_password(&repaired, Some(b"password")).unwrap(); + let extracted = collect_extract_with_password(&repaired_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn repairs_header_encrypted_rar50_inline_recovery_payload_damage_with_password() { + let payload = b"header encrypted payload with structural recovery service\n".repeat(64); + let entries = [entry(b"header-secret-recoverable.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 20, + ) + .unwrap(); + let clean = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let data_range = clean.files().next().unwrap().block.data_range.clone(); + let mut damaged = bytes.clone(); + damaged[data_range.start + 16..data_range.start + 96].fill(0xa5); + + assert!(matches!(Archive::parse(&damaged), Err(Error::NeedPassword))); + let damaged_archive = Archive::parse_with_password(&damaged, Some(b"password")).unwrap(); + assert!(collect_extract_with_password(&damaged_archive, Some(b"password")).is_err()); + + let repaired = damaged_archive.repair_recovery().unwrap(); + + assert_eq!(repaired, bytes); + let repaired_archive = Archive::parse_with_password(&repaired, Some(b"password")).unwrap(); + let extracted = collect_extract_with_password(&repaired_archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_compressed_rar50_recovery_service_record() { + let payload = b"compressed recovery payload with repeated phrase. ".repeat(32); + let entries = [entry(b"compressed-recoverable.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3)]; + let features = FeatureSet::store_only(); + let bytes = write_compressed_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 7, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + assert!(archive.main.has_recovery_record()); + let locator = archive.main.locator().unwrap(); + let recovery_offset = locator.recovery_record_offset.unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(service_names(&archive), ["RR"]); + assert_eq!( + recovery_offset, + (services[0].block.offset - b"Rar!\x1a\x07\x01\x00".len()) as u64 + ); + let recovery = services[0].recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 7); + let recovery_data = services[0].packed_data(&archive).unwrap(); + assert!(recovery_data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(recovery_data[0x0c..0x10].try_into().unwrap()) as usize, + recovery_data.len() + ); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"compressed-recoverable.txt"); + assert_eq!(extracted[0].data, payload); +} + +/// A recovery record used to be asked for twice: once as a feature flag and +/// once as a percentage, and the writer had to reject the case where the two +/// disagreed. Only the percentage is asked for now, so they cannot. +#[test] +fn a_recovery_record_is_asked_for_by_percentage_alone() { + let entries = [entry(b"payload.txt", b"payload without recovery writer\n") + .with_attributes(0x20) + .with_host_os(3)]; + let options = rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()); + let without = write_stored_archive(&entries, options).unwrap(); + let with = rar50::Rar50Writer::new(options) + .entries(entries.to_vec()) + .recovery_percent(Some(5)) + .finish() + .unwrap(); + assert!( + with.len() > without.len(), + "the percentage alone decides whether a recovery record is written" + ); + + // And a format with no recovery record at all says which one to use. + assert!(!rars::supports( + ArchiveVersion::Rar29, + rars::WriterOption::RecoveryRecord, + rars::PlanShape::new(), + )); +} + +#[test] +fn writes_stored_rar50_volume_set_that_reader_reassembles() { + let payload = b"RAR5 stored volume payload split across generated parts.\n".repeat(12); + let entry = entry(b"split50.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3); + let parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new( + ArchiveVersion::Rar50, + FeatureSet::store_only(), + )), + 97, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + assert!(parts.len() > 2); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert_eq!(archives[0].main.volume_number, Some(0)); + assert_eq!(archives[1].main.volume_number, Some(1)); + assert!(archives[0].files().next().unwrap().is_split_after()); + // A fragment that is not the last one checksums the bytes it stores rather + // than the member, which it has no way to checksum yet. + assert_eq!( + archives[0].files().next().unwrap().data_crc32, + Some(crc32( + &parts[0][archives[0].files().next().unwrap().block.data_range.clone()] + )) + ); + assert!(archives[1].files().next().unwrap().is_split_before()); + assert!(!archives + .last() + .unwrap() + .files() + .next() + .unwrap() + .is_split_after()); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"split50.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_stored_rar50_volume_set_with_recovery_records() { + let payload = b"RAR5 stored recovery volume payload split across generated parts.\n".repeat(12); + let entry = entry(b"split50-rr.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 97, + Some(8), + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + assert!(parts.len() > 2); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + for archive in &archives { + assert!(archive.main.has_recovery_record()); + let service = archive.services().next().unwrap(); + assert_eq!(service.name_lossy(), "RR"); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 8); + let locator = archive.main.locator().unwrap(); + assert_eq!( + locator.recovery_record_offset, + Some((service.block.offset - b"Rar!\x1a\x07\x01\x00".len()) as u64) + ); + assert_rar5_inline_recovery_chunks(&service.packed_data(archive).unwrap()); + } + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"split50-rr.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_rar50_archive_metadata_main_extra_record() { + let entries = [entry(b"payload.txt", b"payload with archive metadata\n") + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_stored_archive_with_comment_and_metadata( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar70, FeatureSet::store_only()), + None, + Some(ArchiveMetadataEntry { + name: Some(b"metadata-name.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + }), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let locator = archive.main.locator().unwrap(); + assert_eq!(locator.flags, 0x0001); + assert_eq!(locator.quick_open_offset, Some(0)); + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!(metadata.flags, 0x0003); + assert_eq!( + metadata.name.as_deref(), + Some(b"metadata-name.rar".as_slice()) + ); + assert_eq!(metadata.creation_time, Some(0x01dcd60e_662d7a32)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_compressed_rar50_archive_metadata_main_extra_record() { + let payload = b"compressed payload with archive metadata\n".repeat(8); + let entries = [entry(b"compressed-metadata.txt", &payload) + .with_attributes(0x20) + .with_host_os(3)]; + let bytes = write_compressed_archive_with_metadata( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar70, FeatureSet::store_only()), + Some(ArchiveMetadataEntry { + name: Some(b"compressed-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + }), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"compressed-metadata.rar".as_slice()) + ); + assert_eq!(metadata.creation_time, Some(0x01dcd60e_662d7a32)); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_stored_rar50_archive_that_reader_extracts_with_password() { + let entries = [ + entry(b"secret.txt", b"encrypted stored RAR5 payload from rars\n") + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ]; + let features = FeatureSet::store_only(); + let first = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let second = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + assert_ne!(first, second); + let archive = Archive::parse(&first).unwrap(); + let second_archive = Archive::parse(&second).unwrap(); + let file = archive.files().next().unwrap(); + let second_file = second_archive.files().next().unwrap(); + assert!(file.encrypted); + let encryption = file.encryption.as_ref().unwrap(); + let second_encryption = second_file.encryption.as_ref().unwrap(); + assert!(encryption.check_value.is_some()); + // 2^15 PBKDF2 iterations, the count WinRAR writes. Anything smaller is a + // free speedup for an offline password guess against our archives. + assert_eq!(encryption.kdf_count, 15); + assert_ne!(encryption.salt, second_encryption.salt); + assert_ne!(encryption.iv, second_encryption.iv); + assert_eq!(file.packed_size() % 16, 0); + assert!(matches!( + collect_extract(&archive), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"secret.txt" && matches!(*source, Error::NeedPassword) + )); + assert!(matches!( + collect_extract_with_password(&archive, Some(b"wrong")), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"secret.txt" && matches!(*source, Error::WrongPasswordOrCorruptData) + )); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"secret.txt"); + assert_eq!(extracted[0].data, entry_data(&entries[0])); + assert!(matches!( + file.verify_crc32(&extracted[0].data), + Err(Error::InvalidHeader( + "RAR 5 encrypted CRC32 verification needs encryption keys" + )) + )); + assert!(matches!( + file.verify_integrity(&extracted[0].data), + Err(Error::InvalidHeader( + "RAR 5 encrypted hash MAC needs encryption keys" + )) + )); + assert!(matches!( + file.verify_hash(&extracted[0].data), + Err(Error::InvalidHeader( + "RAR 5 encrypted hash verification needs encryption keys" + )) + )); +} + +#[test] +fn writes_encrypted_stored_rar50_archive_metadata_record() { + let entries = [entry( + b"metadata-secret.txt", + b"encrypted stored payload with archive metadata\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar70, features), + None, + Some(ArchiveMetadataEntry { + name: Some(b"encrypted-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + }), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"encrypted-metadata.rar".as_slice()) + ); + assert_eq!(metadata.creation_time, Some(0x01dcd60e_662d7a32)); + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_encrypted_compressed_rar50_archive_that_reader_extracts_with_password() { + let entries = [entry(b"secret-compressed.txt", b"encrypted compressed RAR5 payload from rars\nencrypted compressed RAR5 payload from rars\n").with_mtime(Some(0x5a21_0055)).with_attributes(0x20).with_host_os(3).with_password(b"secret".to_vec())]; + let features = FeatureSet::store_only(); + let first = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let second = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + let archive = Archive::parse(&first).unwrap(); + let second_archive = Archive::parse(&second).unwrap(); + let file = archive.files().next().unwrap(); + let second_file = second_archive.files().next().unwrap(); + assert!(file.encrypted); + assert_eq!(file.decoded_compression_info().unwrap().method, 3); + let encryption = file.encryption.as_ref().unwrap(); + let second_encryption = second_file.encryption.as_ref().unwrap(); + assert_ne!(encryption.salt, second_encryption.salt); + assert_ne!(encryption.iv, second_encryption.iv); + assert_eq!(file.packed_size() % 16, 0); + + assert!(matches!( + collect_extract_with_password(&archive, Some(b"wrong")), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"secret-compressed.txt" + && matches!(*source, Error::WrongPasswordOrCorruptData) + )); + let extracted = collect_extract_with_password(&archive, Some(b"secret")).unwrap(); + assert_eq!(extracted[0].name, b"secret-compressed.txt"); + assert_eq!(extracted[0].data, entry_data(&entries[0])); + assert_eq!(extracted[0].file_time, 0x5a21_0055); +} + +#[test] +fn encrypted_compressed_rar50_writer_stores_member_when_lz_payload_would_grow() { + let data = deterministic_noise(8192); + assert!(encode_lz_member(&data, 0).unwrap().len() >= data.len()); + let entries = [entry(b"secret-incompressible.bin", &data) + .with_mtime(Some(0x5a21_00a1)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.encrypted); + assert!(file.is_stored()); + let extracted = collect_extract_with_password(&archive, Some(b"secret")).unwrap(); + assert_eq!(extracted[0].name, b"secret-incompressible.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_00a1); +} + +#[test] +fn writes_encrypted_compressed_rar50_archive_metadata_record() { + let payload = b"encrypted compressed metadata payload repeated repeated\n".repeat(8); + let entries = [entry(b"secret-compressed-metadata.txt", &payload) + .with_mtime(Some(0x5a21_0055)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_compressed_archive_with_metadata( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar70, features), + Some(ArchiveMetadataEntry { + name: Some(b"encrypted-compressed-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + }), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"encrypted-compressed-metadata.rar".as_slice()) + ); + let extracted = collect_extract_with_password(&archive, Some(b"secret")).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_solid_compressed_rar50_archive_that_reader_extracts_with_password() { + let first = b"encrypted rar50 solid shared phrase alpha beta gamma\n".repeat(16); + let second = b"encrypted rar50 solid shared phrase alpha beta gamma\nsecond\n".repeat(8); + let entries = [ + entry(b"encrypted-solid-one.txt", &first) + .with_mtime(Some(0x5a21_0061)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec()), + entry(b"encrypted-solid-two.txt", &second) + .with_mtime(Some(0x5a21_0062)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec()), + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert!(archive.main.is_solid()); + assert!(files.iter().all(|file| file.encrypted)); + assert!(!files[0].decoded_compression_info().unwrap().solid); + assert!(files[1].decoded_compression_info().unwrap().solid); + let extracted = collect_extract_with_password(&archive, Some(b"secret")).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn writes_encrypted_rar50_archive_comment_service_that_reader_extracts_with_password() { + let entries = [entry( + b"secret.txt", + b"encrypted stored payload with encrypted comment\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some((b"encrypted CMT from rars\n", b"password")), + None, + ) + .unwrap(); + let second = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some((b"encrypted CMT from rars\n", b"password")), + None, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let second_archive = Archive::parse(&second).unwrap(); + let services: Vec<_> = archive.services().collect(); + let second_services: Vec<_> = second_archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert!(services[0].encrypted); + let service_encryption = services[0].encryption.as_ref().unwrap(); + let second_service_encryption = second_services[0].encryption.as_ref().unwrap(); + assert_ne!(service_encryption.salt, second_service_encryption.salt); + assert_ne!(service_encryption.iv, second_service_encryption.iv); + assert!(matches!( + collect_file(&archive, services[0]), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"CMT" && matches!(*source, Error::NeedPassword) + )); + + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let comment = collect_file(&archive, archive.services().next().unwrap()).unwrap(); + assert_eq!(comment.data, b"encrypted CMT from rars\n"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_encrypted_compressed_rar50_archive_comment_service_with_password() { + let payload = b"encrypted compressed payload with encrypted comment\n".repeat(8); + let entries = [entry(b"encrypted-compressed-comment.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"secret".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some((b"encrypted compressed CMT from rars\n", b"secret")), + None, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert!(services[0].encrypted); + assert!(matches!( + collect_file(&archive, services[0]), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"CMT" && matches!(*source, Error::NeedPassword) + )); + + let archive = Archive::parse_with_password(&bytes, Some(b"secret")).unwrap(); + let comment = collect_file(&archive, archive.services().next().unwrap()).unwrap(); + assert_eq!(comment.data, b"encrypted compressed CMT from rars\n"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_rar50_recovery_service_that_reader_extracts_with_password() { + let entries = [entry( + b"secret-recovery.txt", + b"encrypted stored payload with encrypted recovery\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 6, + ) + .unwrap(); + let second = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 6, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let second_archive = Archive::parse(&second).unwrap(); + assert!(archive.main.has_recovery_record()); + let file_encryption = archive.files().next().unwrap().encryption.as_ref().unwrap(); + let second_file_encryption = second_archive + .files() + .next() + .unwrap() + .encryption + .as_ref() + .unwrap(); + assert_ne!(file_encryption.salt, second_file_encryption.salt); + assert_ne!(file_encryption.iv, second_file_encryption.iv); + let service = archive.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert!(!service.encrypted); + let recovery = service.recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 6); + + let recovery_data = collect_file(&archive, service).unwrap().data; + assert!(recovery_data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(recovery_data[0x0c..0x10].try_into().unwrap()) as usize, + recovery_data.len() + ); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_encrypted_compressed_rar50_recovery_service_that_reader_extracts_with_password() { + let payload = b"encrypted compressed recovery payload repeated repeated. ".repeat(24); + let entries = [entry(b"secret-compressed-recovery.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let features = FeatureSet::store_only(); + let bytes = write_compressed_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 6, + ) + .unwrap(); + let second = write_compressed_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 6, + ) + .unwrap(); + + let archive = Archive::parse(&bytes).unwrap(); + let second_archive = Archive::parse(&second).unwrap(); + assert!(archive.main.has_recovery_record()); + let file_encryption = archive.files().next().unwrap().encryption.as_ref().unwrap(); + let second_file_encryption = second_archive + .files() + .next() + .unwrap() + .encryption + .as_ref() + .unwrap(); + assert_ne!(file_encryption.salt, second_file_encryption.salt); + assert_ne!(file_encryption.iv, second_file_encryption.iv); + let service = archive.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert!(!service.encrypted); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 6); + let recovery_data = collect_file(&archive, service).unwrap().data; + assert!(recovery_data.starts_with(b"{RB}")); + + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_header_encrypted_rar50_archive_that_reader_extracts_with_password() { + let entries = [entry( + b"header-secret.txt", + b"RAR5 header encrypted stored payload from rars\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let second = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let (salt, first_header_iv) = header_encryption_salt_and_first_header_iv(&bytes); + let (second_salt, second_first_header_iv) = header_encryption_salt_and_first_header_iv(&second); + assert_ne!(salt, second_salt); + assert_ne!(first_header_iv, second_first_header_iv); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + assert!(matches!( + Archive::parse_with_password(&bytes, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + assert_eq!(archive.files().next().unwrap().name, b"header-secret.txt"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_header_encrypted_compressed_rar50_archive_that_reader_extracts_with_password() { + let entries = [entry(b"header-compressed-secret.txt", b"RAR5 header encrypted compressed payload from rars\nRAR5 header encrypted compressed payload from rars\n").with_mtime(Some(0x5a21_0056)).with_attributes(0x20).with_host_os(3).with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let second = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + let (salt, first_header_iv) = header_encryption_salt_and_first_header_iv(&bytes); + let (second_salt, second_first_header_iv) = header_encryption_salt_and_first_header_iv(&second); + assert_ne!(salt, second_salt); + assert_ne!(first_header_iv, second_first_header_iv); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + assert!(matches!( + Archive::parse_with_password(&bytes, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.name, b"header-compressed-secret.txt"); + assert_eq!(file.decoded_compression_info().unwrap().method, 3); + assert!(file.encrypted); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); + assert_eq!(extracted[0].file_time, 0x5a21_0056); +} + +#[test] +fn writes_header_encrypted_solid_compressed_rar50_archive_that_reader_extracts_with_password() { + let first = b"header encrypted rar50 solid shared phrase alpha beta gamma\n".repeat(16); + let second = b"header encrypted rar50 solid shared phrase alpha beta gamma\nsecond\n".repeat(8); + let entries = [ + entry(b"header-solid-one.txt", &first) + .with_mtime(Some(0x5a21_0063)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + entry(b"header-solid-two.txt", &second) + .with_mtime(Some(0x5a21_0064)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let bytes = write_compressed_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert!(archive.main.is_solid()); + assert!(files.iter().all(|file| file.encrypted)); + assert!(!files[0].decoded_compression_info().unwrap().solid); + assert!(files[1].decoded_compression_info().unwrap().solid); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn writes_header_encrypted_rar50_archive_comment_service_that_reader_extracts_with_password() { + let entries = [entry( + b"header-comment-secret.txt", + b"RAR5 header encrypted archive comment payload from rars\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some((b"header encrypted CMT from rars\n", b"password")), + None, + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert!(services[0].encrypted); + let comment = collect_file(&archive, services[0]).unwrap(); + assert_eq!(comment.data, b"header encrypted CMT from rars\n"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_header_encrypted_compressed_rar50_archive_comment_service_with_password() { + let payload = b"header encrypted compressed payload with archive comment\n".repeat(8); + let entries = [entry(b"header-compressed-comment-secret.txt", &payload) + .with_mtime(Some(0x5a21_0067)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + Some((b"header encrypted compressed CMT from rars\n", b"password")), + None, + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert!(services[0].encrypted); + let comment = collect_file(&archive, services[0]).unwrap(); + assert_eq!(comment.data, b"header encrypted compressed CMT from rars\n"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_header_encrypted_rar50_archive_metadata_record() { + let entries = [entry( + b"header-metadata-secret.txt", + b"header encrypted stored payload with archive metadata\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_encrypted_archive_with_comment( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar70, features), + None, + Some(ArchiveMetadataEntry { + name: Some(b"header-encrypted-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + }), + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"header-encrypted-metadata.rar".as_slice()) + ); + assert_eq!(metadata.creation_time, Some(0x01dcd60e_662d7a32)); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_header_encrypted_rar50_recovery_service_that_reader_extracts_with_password() { + let entries = [entry( + b"header-recovery-secret.txt", + b"RAR5 header encrypted recovery payload from rars\n", + ) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_stored_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 4, + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + assert!(archive.main.has_recovery_record()); + let service = archive.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert!(!service.encrypted); + let recovery = service.recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 4); + let recovery_data = collect_file(&archive, service).unwrap().data; + assert!(recovery_data.starts_with(b"{RB}")); + assert_eq!( + u32::from_le_bytes(recovery_data[0x0c..0x10].try_into().unwrap()) as usize, + recovery_data.len() + ); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, entry_data(&entries[0])); +} + +#[test] +fn writes_header_encrypted_compressed_rar50_recovery_service_that_reader_extracts_with_password() { + let payload = b"header encrypted compressed recovery payload repeated repeated. ".repeat(24); + let entries = [entry(b"header-secret-compressed-recovery.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_compressed_archive_with_recovery( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 4, + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + assert!(archive.main.has_recovery_record()); + let service = archive.services().next().unwrap(); + assert_eq!(service.name, b"RR"); + assert!(!service.encrypted); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 4); + let recovery_data = collect_file(&archive, service).unwrap().data; + assert!(recovery_data.starts_with(b"{RB}")); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_header_encrypted_compressed_rar50_archive_metadata_record() { + let payload = b"header encrypted compressed metadata payload repeated repeated\n".repeat(8); + let entries = [entry(b"header-secret-compressed-metadata.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let bytes = write_compressed_archive_with_metadata( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar70, features), + Some(ArchiveMetadataEntry { + name: Some(b"header-encrypted-compressed-metadata.rar"), + creation_time: Some(0x01dcd60e_662d7a32), + }), + ) + .unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let metadata = archive.main.archive_metadata().unwrap(); + assert_eq!( + metadata.name.as_deref(), + Some(b"header-encrypted-compressed-metadata.rar".as_slice()) + ); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_stored_rar50_volume_set_that_reader_reassembles_with_password() { + let payload = b"RAR5 encrypted stored split payload from rars.\n".repeat(16); + let entry = entry(b"split-secret.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 97, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + let second_parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 97, + None, + ) + .unwrap(); + assert!(parts.len() > 2); + assert_eq!(parts.len(), second_parts.len()); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let second_archives: Vec<_> = second_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives + .iter() + .all(|archive| archive.files().next().is_some_and(|file| file.encrypted))); + let first_encryption = archives[0] + .files() + .next() + .unwrap() + .encryption + .as_ref() + .unwrap(); + let second_encryption = second_archives[0] + .files() + .next() + .unwrap() + .encryption + .as_ref() + .unwrap(); + assert_ne!(first_encryption.salt, second_encryption.salt); + assert_ne!(first_encryption.iv, second_encryption.iv); + assert!(matches!( + collect_extract_volumes(&archives), + Err(Error::NeedPassword) + )); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"split-secret.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_stored_rar50_volume_set_with_recovery_records() { + let payload = b"RAR5 encrypted stored recovery split payload from rars.\n".repeat(16); + let entry = entry(b"split-secret-rr.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 97, + Some(8), + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives + .iter() + .all(|archive| archive.main.has_recovery_record())); + for archive in &archives { + let service = archive.services().next().unwrap(); + assert_eq!(service.name_lossy(), "RR"); + assert!(!service.encrypted); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 8); + assert_rar5_inline_recovery_chunks(&collect_file(archive, service).unwrap().data); + } + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"split-secret-rr.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_header_encrypted_stored_rar50_volume_set_that_reader_reassembles_with_password() { + let payload = b"RAR5 header encrypted stored split payload from rars.\n".repeat(16); + let entry = entry(b"split-header-secret.txt", &payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 97, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, Some(b"password")); + let second_parts = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 97, + None, + ) + .unwrap(); + assert!(parts.len() > 2); + assert_eq!(parts.len(), second_parts.len()); + let (salt, first_header_iv) = header_encryption_salt_and_first_header_iv(&parts[0]); + let (second_salt, second_first_header_iv) = + header_encryption_salt_and_first_header_iv(&second_parts[0]); + assert_ne!(salt, second_salt); + assert_ne!(first_header_iv, second_first_header_iv); + assert!(parts + .iter() + .all(|part| matches!(Archive::parse(part), Err(Error::NeedPassword)))); + assert!(parts.iter().all(|part| matches!( + Archive::parse_with_password(part, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + ))); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"split-header-secret.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_compressed_rar50_volume_set_that_reader_reassembles_with_password() { + let payload = b"RAR5 encrypted compressed split payload from rars.\n".repeat(18); + let entry = entry(b"split-secret-compressed50.txt", &payload) + .with_mtime(Some(0x5a21_0057)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + let second_parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let second_archives: Vec<_> = second_parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + let first = archives[0].files().next().unwrap(); + let second_first = second_archives[0].files().next().unwrap(); + assert!(first.encrypted); + assert_eq!(first.decoded_compression_info().unwrap().method, 3); + let encryption = first.encryption.as_ref().unwrap(); + let second_encryption = second_first.encryption.as_ref().unwrap(); + assert_ne!(encryption.salt, second_encryption.salt); + assert_ne!(encryption.iv, second_encryption.iv); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-secret-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + assert_eq!(extracted[0].file_time, 0x5a21_0057); +} + +#[test] +fn encrypted_compressed_rar50_volume_writer_stores_member_when_lz_payload_would_grow() { + let data = deterministic_noise(8192); + assert!(encode_lz_member(&data, 0).unwrap().len() >= data.len()); + let entry = entry(b"secret-incompressible-split.bin", &data) + .with_mtime(Some(0x5a21_00a3)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 1024, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + assert!(parts.len() >= 2); + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| { + let file = archive.files().next().unwrap(); + file.encrypted && file.is_stored() + })); + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"secret-incompressible-split.bin"); + assert_eq!(extracted[0].data, data); + assert_eq!(extracted[0].file_time, 0x5a21_00a3); +} + +#[test] +fn writes_encrypted_compressed_rar50_volume_set_with_recovery_records() { + let payload = b"RAR5 encrypted compressed recovery split payload from rars.\n".repeat(18); + let entry = entry(b"split-secret-compressed-rr.txt", &payload) + .with_mtime(Some(0x5a21_0057)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + Some(8), + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_volume())); + assert!(archives + .iter() + .all(|archive| archive.main.has_recovery_record())); + for archive in &archives { + let service = archive.services().next().unwrap(); + assert_eq!(service.name_lossy(), "RR"); + assert!(!service.encrypted); + assert_eq!(service.recovery_record().unwrap().unwrap().percent, 8); + assert_rar5_inline_recovery_chunks(&collect_file(archive, service).unwrap().data); + } + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"split-secret-compressed-rr.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_solid_compressed_rar50_volume_set_that_reader_reassembles_with_password() { + let payload = b"RAR5 encrypted solid compressed split payload from rars.\n".repeat(18); + let entry = entry(b"split-solid-secret-compressed50.txt", &payload) + .with_mtime(Some(0x5a21_0058)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let mut features = FeatureSet::store_only(); + features.solid = true; + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let first = archives[0].files().next().unwrap(); + assert!(first.encrypted); + assert!(!first.decoded_compression_info().unwrap().solid); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"split-solid-secret-compressed50.txt"); + assert_eq!(extracted[0].data, payload); + assert_eq!(extracted[0].file_time, 0x5a21_0058); +} + +#[test] +fn writes_header_encrypted_compressed_rar50_volume_set_that_reader_reassembles_with_password() { + let payload = b"RAR5 header encrypted compressed split payload from rars.\n".repeat(18); + let entry = entry(b"split-header-secret-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, Some(b"password")); + let second_parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + let (salt, first_header_iv) = header_encryption_salt_and_first_header_iv(&parts[0]); + let (second_salt, second_first_header_iv) = + header_encryption_salt_and_first_header_iv(&second_parts[0]); + assert_ne!(salt, second_salt); + assert_ne!(first_header_iv, second_first_header_iv); + assert!(matches!( + Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + let first = archives[0].files().next().unwrap(); + assert!(first.encrypted); + assert_eq!(first.decoded_compression_info().unwrap().method, 3); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"split-header-secret-compressed50.txt"); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_header_encrypted_solid_compressed_rar50_volume_set_that_reader_reassembles_with_password() +{ + let payload = b"RAR5 header encrypted solid compressed split payload from rars.\n".repeat(18); + let entry = entry(b"split-header-solid-secret-compressed50.txt", &payload) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let parts = write_volumes( + std::slice::from_ref(&entry), + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 32, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, Some(b"password")); + assert!(matches!( + Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let first = archives[0].files().next().unwrap(); + assert!(first.encrypted); + assert!(!first.decoded_compression_info().unwrap().solid); + + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!( + extracted[0].name, + b"split-header-solid-secret-compressed50.txt" + ); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn writes_encrypted_multi_file_solid_compressed_rar50_volume_set_that_reader_reassembles() { + let first = b"RAR5 encrypted multi-file solid split shared phrase.\n".repeat(14); + let second = b"RAR5 encrypted multi-file solid split shared phrase.\nsecond\n".repeat(12); + let entries = [ + entry(b"encrypted-solid-split-one.txt", &first) + .with_mtime(Some(0x5a21_0061)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + entry(b"encrypted-solid-split-two.txt", &second) + .with_mtime(Some(0x5a21_0062)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let parts = write_volumes( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 96, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, None); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse(part).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + assert!(archives + .iter() + .flat_map(|archive| archive.files()) + .any(|file| file.name == b"encrypted-solid-split-two.txt" + && file.decoded_compression_info().unwrap().solid)); + + let extracted = collect_extract_volumes_with_password(&archives, Some(b"password")).unwrap(); + assert_eq!(extracted[0].name, b"encrypted-solid-split-one.txt"); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].name, b"encrypted-solid-split-two.txt"); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn writes_header_encrypted_multi_file_solid_compressed_rar50_volume_set_that_reader_reassembles() { + let first = b"RAR5 header encrypted multi-file solid split shared phrase.\n".repeat(14); + let second = + b"RAR5 header encrypted multi-file solid split shared phrase.\nsecond\n".repeat(12); + let entries = [ + entry(b"header-encrypted-solid-split-one.txt", &first) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + entry(b"header-encrypted-solid-split-two.txt", &second) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()), + ]; + let mut features = FeatureSet::store_only(); + features.header_encryption = true; + features.solid = true; + let parts = write_volumes( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + 96, + None, + ) + .unwrap(); + assert_volume_set_links_its_parts(&parts, Some(b"password")); + assert!(matches!( + Archive::parse(&parts[0]), + Err(Error::NeedPassword) + )); + + let archives: Vec<_> = parts + .iter() + .map(|part| Archive::parse_with_password(part, Some(b"password")).unwrap()) + .collect(); + assert!(archives.iter().all(|archive| archive.main.is_solid())); + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted[0].name, b"header-encrypted-solid-split-one.txt"); + assert_eq!(extracted[0].data, first); + assert_eq!(extracted[1].name, b"header-encrypted-solid-split-two.txt"); + assert_eq!(extracted[1].data, second); +} + +#[test] +fn decrypts_rar50_crc32_mac_file_with_password() { + let bytes = std::fs::read(fixture("password_crc32.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert!(files[0].encrypted); + assert_eq!(files[0].packed_size(), 48); + assert!(matches!( + collect_extract(&archive), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"hello.txt" && matches!(*source, Error::NeedPassword) + )); + + let extracted = collect_extract_with_password(&archive, Some(b"password")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); + + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn decrypts_rar50_blake2_mac_file_with_password() { + let bytes = std::fs::read(fixture("password_aes.rar")).unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert!(files[0].encrypted); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn rejects_wrong_password_for_rar50_encrypted_file() { + let bytes = std::fs::read(fixture("password_crc32.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + assert!(matches!( + collect_extract_with_password(&archive, Some(b"wrong")), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"hello.txt" && matches!(*source, Error::WrongPasswordOrCorruptData) + )); +} + +#[test] +fn rejects_rar50_header_encrypted_archive_without_or_with_wrong_password() { + let bytes = std::fs::read(fixture("header_encrypted.rar")).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + assert!(matches!( + Archive::parse_with_password(&bytes, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); +} + +#[test] +fn rejects_unsupported_rar50_header_encryption_version() { + let mut bytes = std::fs::read(fixture("header_encrypted.rar")).unwrap(); + let start = 8; // RAR5 signature length. + let mut pos = start + 4; // Header CRC32. + let read_vint = |pos: &mut usize| { + let mut value = 0u64; + let mut shift = 0; + loop { + let byte = bytes[*pos]; + *pos += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + shift += 7; + } + }; + let body_size = read_vint(&mut pos) as usize; + let header_end = pos + body_size; + assert_eq!(read_vint(&mut pos), 4); // HEAD_CRYPT. + read_vint(&mut pos); // Header flags. + let version_pos = pos; + assert_eq!(read_vint(&mut pos), 0); + bytes[version_pos] = 1; + let crc = crc32(&bytes[start + 4..header_end]); + bytes[start..start + 4].copy_from_slice(&crc.to_le_bytes()); + + assert!(matches!( + Archive::parse_with_password(&bytes, Some(b"password")), + Err(Error::UnsupportedFeature { + feature: "RAR 5 unknown header encryption version", + .. + }) + )); +} + +#[test] +fn parses_rar50_header_encryption_without_optional_password_check() { + let mut bytes = std::fs::read(fixture("header_encrypted.rar")).unwrap(); + // This fixture has a one-byte header size and the check value is the last + // 12 bytes of HEAD_CRYPT. RARLAB marks that field optional; reference + // UnRAR also tests the archive after removing it and fixing size and CRC. + assert_eq!(&bytes[12..18], &[33, 4, 0, 0, 1, 15]); + bytes.drain(34..46); + bytes[12] -= 12; // Header size. + bytes[16] = 0; // Encryption flags: no password check value. + let crc = crc32(&bytes[12..34]); + bytes[8..12].copy_from_slice(&crc.to_le_bytes()); + + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn parses_and_extracts_rar50_header_encrypted_archive_with_password() { + let bytes = std::fs::read(fixture("header_encrypted.rar")).unwrap(); + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert!(files[0].encrypted); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn parses_rar50_header_encrypted_archive_from_path_with_password() { + let archive = + Archive::parse_path_with_password(fixture("header_encrypted.rar"), Some(b"password")) + .unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn parses_winrar721_header_encrypted_quick_open_archive_with_password() { + let bytes = std::fs::read(fixture("winrar721_header_encrypted_quickopen.rar")).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + assert!(matches!( + Archive::parse_with_password(&bytes, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + + let archive = Archive::parse_with_password(&bytes, Some(b"Password")).unwrap(); + let locator = archive.main.locator().unwrap(); + assert!(locator.quick_open_offset.is_some_and(|offset| offset > 0)); + assert_eq!(service_names(&archive), ["QO"]); + assert!(archive.services().next().unwrap().encrypted); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"tmp/rars-threshold2-8191/random.bin"); + assert_eq!(extracted[0].data.len(), 8191); +} + +#[test] +fn extracts_rar50_header_encrypted_comment_service_with_password() { + let bytes = std::fs::read(fixture("header_encrypted_comment.rar")).unwrap(); + + assert!(matches!(Archive::parse(&bytes), Err(Error::NeedPassword))); + assert!(matches!( + Archive::parse_with_password(&bytes, Some(b"wrong")), + Err(Error::WrongPasswordOrCorruptData) + )); + + let archive = Archive::parse_with_password(&bytes, Some(b"password")).unwrap(); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services.len(), 1); + assert_eq!(services[0].name, b"CMT"); + assert!(services[0].encrypted); + + let comment = collect_file(&archive, services[0]).unwrap(); + assert_eq!(comment.name, b"CMT"); + assert_eq!(comment.data.len(), 48); + assert!(comment + .data + .starts_with(b"Encrypted archive comment fixture.\n")); + + let files = collect_extract(&archive).unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert_eq!(files[0].data, b"Hello, RAR 5 encrypted service fixture.\n"); +} + +#[test] +fn extract_to_reports_rar50_entry_context_on_write_failure() { + struct FailingWriter; + + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("sink failed")) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let bytes = std::fs::read(fixture("stored.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let error = archive + .extract_to(ArchiveReadOptions::default(), |_| { + Ok(Box::new(FailingWriter)) + }) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((&b"hello.txt"[..], "extracting")) + ); + assert!(matches!(error.root_cause(), Error::Io(source) if source.message == "sink failed")); +} + +#[test] +fn parses_and_extracts_rar50_stored_file_from_path() { + let archive = Archive::parse_path(fixture("stored.rar")).unwrap(); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"hello.txt"); + assert_eq!(files[0].packed_size(), 30); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn parses_rar50_sfx_prefixed_stored_file() { + let mut bytes = b"small fake sfx prefix".to_vec(); + bytes.extend_from_slice(&std::fs::read(fixture("stored.rar")).unwrap()); + let archive = Archive::parse(&bytes).unwrap(); + + assert_eq!(archive.sfx_offset, b"small fake sfx prefix".len()); + assert_eq!(archive.main.block.offset, archive.sfx_offset + 8); + assert_eq!(archive.main.locator().unwrap().quick_open_offset, Some(0)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn extracts_rar50_empty_file_with_blake2_hash_record() { + let bytes = std::fs::read(fixture("empty_file.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"empty.bin"); + assert!(file.is_stored()); + assert_eq!(file.packed_size(), 0); + assert_eq!(file.unpacked_size, 0); + assert_eq!(file.data_crc32, None); + assert_eq!(file.hash.as_ref().unwrap().hash_type, 0); + assert_eq!(file.hash.as_ref().unwrap().data.len(), 32); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"empty.bin"); + assert!(extracted[0].data.is_empty()); +} + +#[test] +fn verifies_rar50_blake2_hash_record_on_stored_file() { + let archive = Archive::parse_path(fixture("stored_blake2.rar")).unwrap(); + let file = archive.files().next().unwrap(); + + assert_eq!(file.name, b"hello.txt"); + assert_eq!(file.data_crc32, None); + assert_eq!(file.hash.as_ref().unwrap().hash_type, 0); + assert_eq!(file.hash.as_ref().unwrap().data.len(), 32); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +#[test] +fn rejects_corrupt_rar50_stored_payload_hash_when_blake2_is_present() { + let mut bytes = std::fs::read(fixture("stored_blake2.rar")).unwrap(); + let needle = b"Hello, RAR 5.0 fixture world.\n"; + let offset = bytes + .windows(needle.len()) + .position(|window| window == needle) + .expect("stored payload in fixture"); + bytes[offset] ^= 0x01; + let archive = Archive::parse(&bytes).unwrap(); + + assert!(matches!( + collect_extract(&archive), + Err(Error::AtEntry { + name, + operation: "verifying", + source + }) if name == b"hello.txt" && matches!(*source, Error::HashMismatch { hash_type: 0 }) + )); +} + +#[test] +fn ignores_unknown_rar50_file_hash_records_when_extracting() { + let bytes = std::fs::read(fixture("wild/invalid_hash_valid_htime_exfld.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"file.txt"); + assert_eq!( + extracted[0].data, + b"invalid HASH extra, and later a valid HTIME extra" + ); +} + +#[test] +fn skips_rar50_redirection_entries_when_extracting() { + for fixture_name in [ + "wild/hardlink.rar", + "wild/symlink.rar", + "wild/rarfile_hlink.rar", + ] { + let bytes = std::fs::read(fixture(fixture_name)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + + assert!( + archive.files().any(|file| file.redirection.is_some()), + "{fixture_name}" + ); + assert!( + extracted.iter().all(|entry| archive + .files() + .find(|file| file.name == entry.name) + .is_some_and(|file| file.redirection.is_none())), + "{fixture_name}" + ); + } +} + +#[test] +fn reports_rar50_redirection_entries_when_requested() { + for fixture_name in [ + "wild/hardlink.rar", + "wild/symlink.rar", + "wild/rarfile_hlink.rar", + ] { + let bytes = std::fs::read(fixture(fixture_name)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let redirections = RefCell::new(Vec::new()); + + archive + .extract_to_with_redirections( + read_options(None), + |_meta| { + let data = Rc::new(RefCell::new(Vec::new())); + let writer: Box = Box::new(CollectWriter { data }); + Ok(writer) + }, + |meta, redirection| { + redirections.borrow_mut().push(( + meta.name.clone(), + redirection.redirection_type, + redirection.flags, + redirection.target_name.clone(), + )); + Ok(()) + }, + ) + .unwrap(); + + let expected: Vec<_> = archive + .files() + .filter_map(|file| { + file.redirection.as_ref().map(|redirection| { + ( + file.name.clone(), + redirection.redirection_type, + redirection.flags, + redirection.target_name.clone(), + ) + }) + }) + .collect(); + assert_eq!(redirections.into_inner(), expected, "{fixture_name}"); + } +} + +#[test] +fn extracts_wild_rar50_solid_archives_with_redundant_filter_records() { + for fixture_name in [ + "wild/rarfile_solid.rar", + "wild/rarfile_solid_qo.rar", + "wild/libarchive_solid.rar", + "wild/libarchive_multiple_files_solid.rar", + ] { + let bytes = std::fs::read(fixture(fixture_name)).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + + assert!(!extracted.is_empty(), "{fixture_name}"); + assert!( + archive + .files() + .filter(|file| !file.is_directory() && file.redirection.is_none()) + .any(|file| file.decoded_compression_info().is_ok_and(|info| info.solid)), + "{fixture_name}" + ); + } +} + +#[test] +fn rejects_truncated_wild_rar50_loop_fixture_without_integrity_records() { + let bytes = std::fs::read(fixture("wild/libarchive_loop_bug.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + assert_eq!(file.unpacked_size, 196_608); + assert!(file.data_crc32.is_none()); + assert!(file.hash.is_none()); + + // UnRAR 7.20 accepts this fixture but emits zero bytes. We deliberately + // report incomplete decoding instead of presenting a declared non-empty + // member as an empty success just because it has no checksum (#44). + let error = archive + .extract_to(rars::ArchiveReadOptions::default(), |_| { + Ok(Box::new(std::io::sink())) + }) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::Codec(rars::codec::Error::NeedMoreInput) + )); + assert_eq!(error.entry_context().unwrap().0, b"a"); +} + +#[test] +fn parses_rar50_multifile_stored_archive() { + let bytes = std::fs::read(fixture("multifile.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 3); + assert_eq!(files[0].name, b"hello.txt"); + assert_eq!(files[1].name, b"tiny.txt"); + assert_eq!(files[2].name, b"random_4k.bin"); + assert!(files.iter().all(|file| file.is_stored())); + assert_eq!(files[0].packed_size(), 30); + assert_eq!(files[1].packed_size(), 9); + assert_eq!(files[2].packed_size(), 4096); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 3); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); + assert_eq!(extracted[1].data, b"AAAAAAAA\n"); + assert_eq!(extracted[2].data.len(), 4096); +} + +#[test] +fn extracts_rar50_solid_archive() { + let archive = Archive::parse_path(fixture("solid.rar")).unwrap(); + + assert!(archive.main.is_solid()); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 2); + assert!(files + .iter() + .any(|file| file.decoded_compression_info().unwrap().solid)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert!(extracted.iter().all(|entry| !entry.data.is_empty())); +} + +#[test] +fn rejects_nonzero_encrypted_stored_padding_in_streaming_extraction() { + let payload = b"encrypted stored RAR5 padding check"; + let entries = [entry(b"secret.txt", payload) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec())]; + let features = FeatureSet::store_only(); + let mut bytes = write_stored_archive( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features), + ) + .unwrap(); + corrupt_encrypted_stored_padding(&mut bytes, b"password"); + + let archive = Archive::parse(&bytes).unwrap(); + assert!(matches!( + collect_extract_with_password(&archive, Some(b"password")), + Err(Error::AtEntry { + name, + operation: "decoding", + source + }) if name == b"secret.txt" + && matches!(*source, Error::InvalidHeader( + "RAR 5 encrypted stored file has non-zero padding" + )) + )); +} + +#[test] +fn rar50_solid_extraction_uses_file_compression_info_flag() { + let mut archive = Archive::parse_path(fixture("solid.rar")).unwrap(); + archive.main.archive_flags = 0; + + assert!(!archive.main.is_solid()); + assert!(archive + .files() + .any(|file| file.decoded_compression_info().unwrap().solid)); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert!(extracted.iter().all(|entry| !entry.data.is_empty())); +} + +#[test] +fn extracts_rar50_compressed_multivolume_archive() { + let volumes = [ + Archive::parse_path(fixture("multivol.part1.rar")).unwrap(), + Archive::parse_path(fixture("multivol.part2.rar")).unwrap(), + Archive::parse_path(fixture("multivol.part3.rar")).unwrap(), + ]; + + assert!(volumes.iter().all(|archive| archive.main.is_volume())); + assert!(volumes[0].files().next().unwrap().is_split_after()); + assert!(volumes[1].files().next().unwrap().is_split_before()); + assert!(volumes[1].files().next().unwrap().is_split_after()); + assert!(volumes[2].files().next().unwrap().is_split_before()); + assert!(!volumes[2].files().next().unwrap().is_split_after()); + + let extracted = collect_extract_volumes(&volumes).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"random_4k.bin"); + assert_eq!(extracted[0].data.len(), 4096); +} + +#[test] +fn extracts_rar50_stored_multivolume_archive() { + let volumes = [ + Archive::parse_path(fixture("stored_multivol.part1.rar")).unwrap(), + Archive::parse_path(fixture("stored_multivol.part2.rar")).unwrap(), + Archive::parse_path(fixture("stored_multivol.part3.rar")).unwrap(), + ]; + + let extracted = collect_extract_volumes(&volumes).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"random_4k.bin"); + assert_eq!(extracted[0].data.len(), 4096); + assert_eq!(crc32(&extracted[0].data), 0xb9c5_4415); +} + +#[test] +fn extracts_rar50_solid_multivolume_archive() { + let volumes = [ + Archive::parse_path(fixture("solid_multivol.part01.rar")).unwrap(), + Archive::parse_path(fixture("solid_multivol.part02.rar")).unwrap(), + Archive::parse_path(fixture("solid_multivol.part03.rar")).unwrap(), + Archive::parse_path(fixture("solid_multivol.part04.rar")).unwrap(), + Archive::parse_path(fixture("solid_multivol.part05.rar")).unwrap(), + Archive::parse_path(fixture("solid_multivol.part06.rar")).unwrap(), + ]; + + let extracted = collect_extract_volumes(&volumes).unwrap(); + + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].name, b"tiny.txt"); + assert_eq!(extracted[0].data, b"AAAAAAAA\n"); + assert_eq!(extracted[1].name, b"bigtext_64k.bin"); + assert_eq!(extracted[1].data.len(), 65_536); + assert_eq!(crc32(&extracted[1].data), 0xddc9_5682); +} + +/// A volume set must agree on header encryption throughout. Splicing a +/// plaintext volume into an encrypted set is a substitution attack: header +/// encryption is what hides the file names, so an attacker who can replace one +/// volume otherwise injects entries into an extraction the user believes is +/// protected end to end. +/// +/// RAR 7.12 and UnRAR 7.20 both abort with a bad-archive error rather than +/// extract. rars refuses too, and must write nothing while doing so. +#[test] +fn rar50_refuses_a_volume_set_that_changes_encryption_partway() { + let volumes: Vec<_> = [ + "header_encrypted_stored_multivol.part1.rar", + "plaintext_stored_multivol.part2.rar", + "header_encrypted_stored_multivol.part3.rar", + ] + .iter() + .filter_map(|name| Archive::parse_path_with_password(fixture(name), Some(b"password")).ok()) + .collect(); + + assert_eq!( + volumes.len(), + 3, + "every volume should still parse on its own" + ); + assert!(matches!( + collect_extract_volumes_with_password(&volumes, Some(b"password")), + Err(Error::InvalidHeader( + "RAR 5 split entry encryption flag changed" + )) + )); +} + +/// Bits 0-5 of CompInfo hold six bits and only two values are assigned, so a +/// decoder must refuse the rest rather than falling through to Unpack50 and +/// misreading a stream it cannot decode. RAR 7.12 and UnRAR 7.20 answer +/// `Unknown method` for version 2 on a compressed member. +/// +/// A stored member is the exception and extracts on both, because nothing is +/// decompressed and the version never comes up. So the refusal belongs on the +/// decompressor, not at header parse time, and that is the half worth pinning: +/// a reader that rejects too early breaks archives the reference reads. +#[test] +fn rar50_refuses_an_unknown_algorithm_version_only_when_it_must_decompress() { + let compressed = Archive::parse_path(fixture("algorithm_version_2.rar")).unwrap(); + assert!(matches!( + collect_extract(&compressed), + Err(Error::AtEntry { source, .. }) + if matches!(*source, Error::UnsupportedFeature { .. }) + )); + + let stored = Archive::parse_path(fixture("algorithm_version_2_stored.rar")).unwrap(); + let extracted = collect_extract(&stored).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); +} + +/// A non-solid stream starts with no Huffman tables loaded, so a first block +/// that clears `table_present` has nothing to reuse. +/// +/// RAR 7.12 and UnRAR 7.20 fail the member too, but by a different route: they +/// appear to decode against whatever their table memory holds and emit wrong +/// bytes until the checksum catches it. Refusing the block outright reaches the +/// same verdict without decoding anything. +#[test] +fn rar50_refuses_a_first_block_that_reuses_absent_tables() { + let archive = Archive::parse_path(fixture("first_block_without_tables.rar")).unwrap(); + assert!(collect_extract(&archive).is_err()); +} + +/// Modern WinRAR writes the modification time into `FHEXTRA_HTIME` and leaves +/// the base-header field out, so a reader that only looks at the header field +/// restores nothing. 39 of the 40 RAR 5 fixtures here carry the record and one +/// carries the header field, which is how far from an edge case this is. +#[test] +fn rar50_reads_the_modification_time_from_the_htime_record() { + let mut with_record = 0; + let mut with_header_field = 0; + + for entry in fs::read_dir(fixture(".")).unwrap() { + let path = entry.unwrap().path(); + if path.extension().is_none_or(|ext| ext != "rar") { + continue; + } + let Ok(archive) = Archive::parse_path(&path) else { + continue; + }; + let common = rars::Archive::Rar50Plus(archive.clone()); + for (member, file) in common.members().zip(archive.files()) { + assert_eq!(member.meta.file_time, file.mtime.or(file.htime_mtime)); + } + for file in archive.files() { + if file.mtime.is_some() { + with_header_field += 1; + } + if file.htime_mtime.is_some() { + with_record += 1; + } + // Whichever carries it, the entry must report a time. + if file.mtime.is_some() || file.htime_mtime.is_some() { + assert_ne!( + file.metadata().file_time, + None, + "{} reports no time despite carrying one", + path.display() + ); + } + } + } + + assert!( + with_record > with_header_field, + "expected the extra record to dominate: record={with_record} header={with_header_field}" + ); +} + +/// WinRAR 5.21 and earlier set the password-check flag and then wrote eight +/// zero bytes into the field, so a reader that verifies it rejects the correct +/// password and the archive cannot be opened at all. RAR 7.12 treats an +/// all-zero field as no check available and lets the data checksum decide. +/// +/// A wrong password must still be refused, just later and by a different +/// mechanism, which is the second half of this test. +#[test] +fn opens_rar50_archive_whose_password_check_is_all_zeroes() { + let data = fs::read(fixture("zeroed_password_check.rar")).unwrap(); + + let archive = Archive::parse_with_password(&data, Some(b"secret")).unwrap(); + let extracted = collect_extract_with_password(&archive, Some(b"secret")).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"s.txt"); + + assert!( + Archive::parse_with_password(&data, Some(b"wrongpw")) + .and_then(|archive| collect_extract_with_password(&archive, Some(b"wrongpw"))) + .is_err(), + "a wrong password must still be refused, via the data checksum" + ); +} + +/// A stored, header-encrypted, split member reaches a verification path of its +/// own, and that path used to apply the encrypted-checksum transform to any +/// encrypted file rather than only to one whose crypt record sets the tweaked +/// checksum flag. The archive extracted the right bytes and then failed on a +/// checksum that had been transformed when the stored one had not been. +/// +/// Compressed volume sets verify elsewhere, which is why the sibling +/// `encrypted_multivol` fixtures never caught this. +#[test] +fn extracts_rar50_header_encrypted_stored_multivolume_archive() { + let volumes: Vec<_> = (1..=3) + .map(|part| { + Archive::parse_path_with_password( + fixture(&format!("header_encrypted_stored_multivol.part{part}.rar")), + Some(b"password"), + ) + .unwrap() + }) + .collect(); + + assert!(volumes.iter().all(|archive| archive.main.is_volume())); + + let extracted = collect_extract_volumes(&volumes).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"stored_4k.bin"); + assert_eq!(extracted[0].data.len(), 4096); + assert_eq!(crc32(&extracted[0].data), 0xa087_a9af); +} + +#[test] +fn extracts_rar50_encrypted_compressed_multivolume_archive() { + let volumes = [ + Archive::parse_path_with_password( + fixture("encrypted_multivol.part1.rar"), + Some(b"password"), + ) + .unwrap(), + Archive::parse_path_with_password( + fixture("encrypted_multivol.part2.rar"), + Some(b"password"), + ) + .unwrap(), + Archive::parse_path_with_password( + fixture("encrypted_multivol.part3.rar"), + Some(b"password"), + ) + .unwrap(), + ]; + + assert!(volumes.iter().all(|archive| archive.main.is_volume())); + assert!(volumes + .iter() + .all(|archive| { archive.files().next().is_some_and(|file| file.encrypted) })); + + let extracted = collect_extract_volumes(&volumes).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"random_4k.bin"); + assert_eq!(extracted[0].data.len(), 4096); + assert_eq!(crc32(&extracted[0].data), 0xb9c5_4415); +} + +#[test] +fn extracts_rar50_compressed_split_streams_and_preserves_failure_kinds() { + let mut volumes: Vec<_> = (1..=3) + .map(|part| { + Archive::parse_path_with_password( + fixture(&format!("encrypted_multivol.part{part}.rar")), + Some(b"password"), + ) + .unwrap() + }) + .collect(); + let options = ArchiveReadOptions::default().with_rar50_buffered_decode_limit(0); + let data = Rc::new(RefCell::new(Vec::new())); + extract_volumes_to(&volumes, options, |meta| { + assert_eq!(meta.name, b"random_4k.bin"); + Ok(Box::new(CollectWriter { + data: Rc::clone(&data), + })) + }) + .unwrap(); + assert_eq!(data.borrow().len(), 4096); + assert_eq!(crc32(&data.borrow()), 0xb9c5_4415); + + struct FailedSink; + impl Write for FailedSink { + fn write(&mut self, _: &[u8]) -> IoResult { + Err(std::io::Error::other("split sink failed")) + } + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } + } + let error = extract_volumes_to(&volumes, options, |_| Ok(Box::new(FailedSink))).unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert!( + matches!(error.root_cause(), Error::Io(source) if source.message == "split sink failed") + ); + assert_eq!(error.entry_context().unwrap().0, b"random_4k.bin"); + + // An integrity failure triggers the diagnostic scan of packed fragments. + // Give that scan a mismatching first-fragment CRC and force a final + // checksum failure without changing the encrypted payload. + let first = volumes[0] + .blocks + .iter_mut() + .find_map(|block| match block { + Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + first.data_crc32 = Some(0); + let last = volumes[2] + .blocks + .iter_mut() + .find_map(|block| match block { + Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + last.hash = None; + last.data_crc32 = Some(0); + let error = + extract_volumes_to(&volumes, options, |_| Ok(Box::new(std::io::sink()))).unwrap_err(); + assert!(matches!(error.root_cause(), Error::Crc32Mismatch { .. })); +} + +#[test] +fn rejects_nonzero_encrypted_stored_padding_in_split_streaming_extraction() { + let data = b"encrypted stored split RAR5 padding check".repeat(3); + let entry = entry(b"split-secret.txt", &data) + .with_mtime(Some(0x5a21_0000)) + .with_attributes(0x20) + .with_host_os(3) + .with_password(b"password".to_vec()); + let features = FeatureSet::store_only(); + let mut volumes = write_volumes( + std::slice::from_ref(&entry), + stored(rar50::WriterOptions::new(ArchiveVersion::Rar50, features)), + 32, + None, + ) + .unwrap(); + corrupt_encrypted_stored_split_padding(&mut volumes, b"password"); + let archives = volumes + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect::>(); + + assert!(matches!( + collect_extract_volumes_with_password(&archives, Some(b"password")), + Err(Error::AtEntry { + name, + operation: "extracting", + source + }) if name == b"split-secret.txt" + && matches!(*source, Error::InvalidHeader( + "RAR 5 encrypted stored split file has non-zero padding" + )) + )); +} + +#[test] +fn decodes_rar50_compression_info_bitfield() { + let stored = Archive::parse_path(fixture("stored.rar")).unwrap(); + let stored_file = stored.files().next().unwrap(); + let info = stored_file.decoded_compression_info().unwrap(); + assert_eq!(info.algorithm_version, 0); + assert_eq!(info.method, 0); + assert!(!info.solid); + assert_eq!(info.dictionary_fraction, 0); + assert!(!info.rar5_compat); + + for (name, method) in [ + ("m1_fastest.rar", 1), + ("m3_default.rar", 3), + ("m5_max.rar", 5), + ] { + let archive = Archive::parse_path(fixture(name)).unwrap(); + let file = archive.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + assert_eq!(info.algorithm_version, 0, "{name}"); + assert_eq!(info.method, method, "{name}"); + assert!(!info.solid, "{name}"); + assert_eq!(info.dictionary_fraction, 0, "{name}"); + assert!(!info.rar5_compat, "{name}"); + assert!(info.dictionary_size >= 128 * 1024, "{name}"); + } +} + +#[test] +fn parses_rar50_comment_service_archive() { + let archive = Archive::parse_path(fixture("with_comment.rar")).unwrap(); + + assert_eq!(archive.files().count(), 1); + assert_eq!(service_names(&archive), ["CMT"]); + let comment = archive.services().next().unwrap(); + assert_eq!(comment.packed_size(), 30); + assert_eq!(comment.unpacked_size, 30); + assert!(comment.is_stored()); +} + +#[test] +fn parses_rar50_quick_open_service_archive() { + let archive = Archive::parse_path(fixture("with_quickopen.rar")).unwrap(); + + assert_eq!(archive.files().count(), 2); + assert_eq!(service_names(&archive), ["QO"]); + let quick_open = archive.services().next().unwrap(); + assert_eq!(quick_open.packed_size(), 162); + assert_eq!(quick_open.unpacked_size, 162); + assert!(quick_open.is_stored()); +} + +#[test] +fn parses_rar50_recovery_service_archive() { + let archive = Archive::parse_path(fixture("with_recovery.rar")).unwrap(); + + assert!(archive.main.has_recovery_record()); + assert_eq!(archive.files().count(), 1); + assert_eq!(service_names(&archive), ["RR"]); + let recovery = archive.services().next().unwrap(); + assert_eq!(recovery.packed_size(), 210); + assert_eq!(recovery.unpacked_size, 210); + assert_rar5_inline_recovery_chunks(&recovery.packed_data(&archive).unwrap()); + let recovery = recovery.recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 10); + assert_eq!(recovery.payload_size, 210); +} + +/// Rewrites the `RR` service header's extra area in `with_recovery.rar` and +/// fixes up the two sizes and the header CRC32 that describe it. +/// +/// The header's `ExtraSize`, `HeadSize` and every extra record here fit in a +/// one-byte vint, so splicing is a byte-for-byte substitution. +fn with_recovery_extra_area(extra: &[u8]) -> Vec { + const CRC_POS: usize = 0x81; + const BODY: usize = 0x86; + const HEAD_SIZE_POS: usize = 0x85; + const EXTRA_SIZE_POS: usize = 0x88; + const EXTRA_START: usize = 0x95; + const EXTRA_END: usize = 0x98; + + let mut data = fs::read(fixture("with_recovery.rar")).unwrap(); + assert_eq!(&data[EXTRA_START..EXTRA_END], b"\x02\x07\x0a"); + assert!(extra.len() < 0x80); + + data.splice(EXTRA_START..EXTRA_END, extra.iter().copied()); + data[EXTRA_SIZE_POS] = extra.len() as u8; + data[HEAD_SIZE_POS] = + (data[HEAD_SIZE_POS] as usize + extra.len() - (EXTRA_END - EXTRA_START)) as u8; + let header_end = BODY + data[HEAD_SIZE_POS] as usize; + let header_crc = crc32(&data[CRC_POS + 4..header_end]); + data[CRC_POS..CRC_POS + 4].copy_from_slice(&header_crc.to_le_bytes()); + data +} + +/// WinRAR 5.21 and earlier stored the `FHEXTRA_SUBDATA` size one less than the +/// payload they wrote. `SUBDATA` is the last record in a service header, so the +/// shortfall leaves exactly one byte dangling, and a reader that drops it loses +/// the recovery percent. +#[test] +fn reads_rar50_service_subdata_whose_size_underflows_by_one() { + let archive = Archive::parse_path(fixture("subdata_size_underflow.rar")).unwrap(); + + assert_eq!(service_names(&archive), ["RR"]); + let recovery = archive.services().next().unwrap(); + assert_eq!(recovery.service_data, Some(vec![0x0a])); + let recovery = recovery.recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 10); + assert_eq!(recovery.payload_size, 210); +} + +/// The fold is deliberately narrow: it fires only for `SUBDATA` on a service +/// header, since that is the record WinRAR undersized. A stray byte after any +/// other record just ends the walk. +#[test] +fn rar50_one_byte_fold_is_limited_to_service_subdata() { + // 0x03 is FHEXTRA_HTIME: same shape, wrong record type, so no fold. + let archive = Archive::parse(&with_recovery_extra_area(b"\x01\x03\x0a")).unwrap(); + let service = archive.services().next().unwrap(); + assert_eq!(service.service_data, None); + assert!(service.recovery_record().is_err()); +} + +/// A record that does not fit its extra area ends the walk. RAR 7.12 and UnRAR +/// 7.20 extract from these shapes, so failing the archive over one would +/// discard file data that is intact. The older SUBDATA one-byte shortfall is +/// covered separately because it consumes the dangling byte instead. +#[test] +fn rar50_stops_on_a_malformed_extra_record_instead_of_failing_the_archive() { + let cases: [(&str, &[u8]); 3] = [ + ("two bytes dangle after the record", b"\x01\x07\x0a\x0b"), + ("first record claims 127 bytes", b"\x7f\x07\x0a\x0b"), + ("record too small for its type vint", b"\x00\x07\x0a\x0b"), + ]; + + for (name, extra) in cases { + let data = with_recovery_extra_area(extra); + let archive = Archive::parse(&data).unwrap_or_else(|error| panic!("{name}: {error}")); + let extracted = collect_extract(&archive).unwrap_or_else(|error| panic!("{name}: {error}")); + assert_eq!(extracted.len(), 1, "{name}"); + assert_eq!(extracted[0].name, b"hello.txt", "{name}"); + } +} + +#[test] +fn parses_rar50_mixed_service_archive() { + let archive = Archive::parse_path(fixture("with_all_services.rar")).unwrap(); + + assert!(archive.main.has_recovery_record()); + assert_eq!(archive.files().count(), 2); + assert_eq!(service_names(&archive), ["CMT", "QO", "RR"]); + let services: Vec<_> = archive.services().collect(); + assert_eq!(services[0].packed_size(), 30); + assert_eq!(services[1].packed_size(), 162); + assert_eq!(services[2].packed_size(), 526); + let recovery = services[2].recovery_record().unwrap().unwrap(); + assert_eq!(recovery.percent, 5); + assert_eq!(recovery.payload_size, 526); + assert_rar5_inline_recovery_chunks(&services[2].packed_data(&archive).unwrap()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, b"Hello, RAR 5.0 fixture world.\n"); + assert_eq!(extracted[1].data, b"AAAAAAAA\n"); +} + +#[test] +fn parses_rar50_rev5_recovery_volume_metadata() { + let bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + let rev = Rev5Volume::parse(&bytes).unwrap(); + + assert_eq!(rev.version, 1); + assert_eq!(rev.data_count, 5); + assert_eq!(rev.recovery_count, 2); + assert_eq!(rev.recovery_number, 5); + assert_eq!(rev.payload_size, 4096); + assert_eq!(rev.payload.len(), 4096); + assert_eq!(rev.payload_crc32, 0xfd0b_7e3f); + assert_eq!(rev.data_volumes.len(), 5); + assert_eq!(rev.data_volumes[0].file_size, 4096); + assert_eq!(rev.data_volumes[4].file_size, 1032); + + let parsed_meta = Rev5VolumeMeta::parse(&bytes).unwrap(); + assert_eq!(Rev5VolumeMeta::from(&rev), parsed_meta); + assert_eq!(Rev5VolumeMeta::from(rev), parsed_meta); +} + +#[test] +fn parses_rar50_rev5_metadata_without_validating_payload_crc() { + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 0x01; + + let meta = Rev5VolumeMeta::parse(&bytes).unwrap(); + + assert_eq!(meta.version, 1); + assert_eq!(meta.data_count, 5); + assert_eq!(meta.recovery_count, 2); + assert_eq!(meta.recovery_number, 5); + assert_eq!(meta.payload_size, 4096); + assert_eq!(meta.payload_crc32, 0xfd0b_7e3f); + assert_eq!(meta.data_volumes.len(), 5); +} + +fn update_rev5_header_crc(bytes: &mut [u8]) { + let header_size = u32::from_le_bytes(bytes[12..16].try_into().unwrap()) as usize; + let header_end = 16 + header_size; + let header_crc = crc32(&bytes[12..header_end]); + bytes[8..12].copy_from_slice(&header_crc.to_le_bytes()); +} + +#[test] +fn parses_rar50_rev5_metadata_with_forward_compatible_trailing_bytes() { + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + let header_size = u32::from_le_bytes(bytes[12..16].try_into().unwrap()) as usize; + let header_end = 16 + header_size; + bytes[12..16].copy_from_slice(&(header_size as u32 + 3).to_le_bytes()); + bytes.splice(header_end..header_end, [0xa5, 0x5a, 0x7e]); + update_rev5_header_crc(&mut bytes); + + let meta = Rev5VolumeMeta::parse(&bytes).unwrap(); + + assert_eq!(meta.data_count, 5); + assert_eq!(meta.data_volumes.len(), 5); + assert_eq!(meta.payload_size, 4096); +} + +#[test] +fn rejects_rar50_rev5_metadata_with_truncated_table() { + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + let header_size = u32::from_le_bytes(bytes[12..16].try_into().unwrap()) as usize; + let header_end = 16 + header_size; + bytes[12..16].copy_from_slice(&(header_size as u32 - 1).to_le_bytes()); + bytes.remove(header_end - 1); + update_rev5_header_crc(&mut bytes); + + assert!(matches!( + Rev5VolumeMeta::parse(&bytes), + Err(Error::InvalidHeader( + "RAR 5 REV metadata table size is invalid" + )) + )); +} + +#[test] +fn rejects_rar50_rev5_header_boundaries_before_reading_payload() { + let original = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + + let mut wrong_signature = original.clone(); + wrong_signature[0] ^= 1; + assert!(matches!( + Rev5VolumeMeta::parse(&wrong_signature), + Err(Error::UnsupportedSignature) + )); + assert!(matches!( + Rev5VolumeMeta::parse(&original[..15]), + Err(Error::TooShort) + )); + + let mut invalid_size = original.clone(); + invalid_size[12..16].copy_from_slice(&5u32.to_le_bytes()); + assert!(matches!( + Rev5VolumeMeta::parse(&invalid_size), + Err(Error::InvalidHeader("RAR 5 REV header size is invalid")) + )); + + invalid_size[12..16].copy_from_slice(&0x10_0001u32.to_le_bytes()); + assert!(matches!( + Rev5VolumeMeta::parse(&invalid_size), + Err(Error::InvalidHeader("RAR 5 REV header size is invalid")) + )); + + let mut truncated_header = original.clone(); + truncated_header[12..16].copy_from_slice(&(original.len() as u32).to_le_bytes()); + assert!(matches!( + Rev5VolumeMeta::parse(&truncated_header), + Err(Error::TooShort) + )); + + let mut wrong_crc = original.clone(); + wrong_crc[16] ^= 1; + assert!(matches!( + Rev5VolumeMeta::parse(&wrong_crc), + Err(Error::Crc32Mismatch { .. }) + )); + + let mut short_body = original.clone(); + short_body[12..16].copy_from_slice(&10u32.to_le_bytes()); + update_rev5_header_crc(&mut short_body); + assert!(matches!( + Rev5VolumeMeta::parse(&short_body), + Err(Error::TooShort) + )); + + let mut unknown_version = original; + unknown_version[16] = 2; + update_rev5_header_crc(&mut unknown_version); + assert!(matches!( + Rev5VolumeMeta::parse(&unknown_version), + Err(Error::UnsupportedFeature { + feature: "RAR 5 REV version", + .. + }) + )); +} + +#[test] +fn repairs_missing_rar50_data_volume_from_rev5_recovery_volume() { + let data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let first_rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let inputs = [ + Some(data[0].as_slice()), + None, + Some(data[2].as_slice()), + Some(data[3].as_slice()), + Some(data[4].as_slice()), + ]; + + let repaired = repair_rev5_volumes(&inputs, &[first_rev]).unwrap(); + + assert_eq!(repaired, data); +} + +#[test] +fn rar50_rev5_repair_preserves_reconstruction_and_sink_errors() { + let data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let too_many_missing = [ + None, + None, + Some(data[2].as_slice()), + Some(data[3].as_slice()), + Some(data[4].as_slice()), + ]; + assert!(matches!( + repair_rev5_volumes_to(&too_many_missing, std::slice::from_ref(&rev), |_, _| Ok(())), + Err(Error::Rar5Recovery( + rars::recovery::rar5::Error::TooManyDamagedShards + )) + )); + + let one_missing = [ + Some(data[0].as_slice()), + None, + Some(data[2].as_slice()), + Some(data[3].as_slice()), + Some(data[4].as_slice()), + ]; + let mut written = Vec::new(); + assert!(matches!( + repair_rev5_volumes_to(&one_missing, &[rev], |index, _| { + if index == 1 { + return Err(Error::Cancelled); + } + written.push(index); + Ok(()) + }), + Err(Error::Cancelled) + )); + assert_eq!(written, vec![0]); +} + +#[cfg(target_pointer_width = "32")] +#[test] +fn rar50_rev5_repair_rejects_data_size_above_native_range() { + let data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let mut rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + rev.data_volumes[0].file_size = u32::MAX as u64 + 1; + let inputs: Vec<_> = data.iter().map(|volume| Some(volume.as_slice())).collect(); + assert!(matches!( + repair_rev5_volumes_to(&inputs, &[rev], |_, _| panic!("unexpected write")), + Err(Error::InvalidHeader( + "RAR 5 REV data volume size overflows usize" + )) + )); +} + +#[test] +fn repairs_two_missing_rar50_data_volumes_from_rev5_recovery_volumes() { + let data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let revs = [ + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(), + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part2.rev")).unwrap()).unwrap(), + ]; + let inputs = [ + Some(data[0].as_slice()), + None, + Some(data[2].as_slice()), + None, + Some(data[4].as_slice()), + ]; + + let repaired = repair_rev5_volumes(&inputs, &revs).unwrap(); + + assert_eq!(repaired, data); +} + +#[test] +fn rejects_duplicate_rar50_rev5_recovery_rows() { + let data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let inputs = [ + Some(data[0].as_slice()), + None, + Some(data[2].as_slice()), + None, + Some(data[4].as_slice()), + ]; + + assert!(matches!( + repair_rev5_volumes(&inputs, &[rev.clone(), rev]), + Err(Error::InvalidHeader( + "RAR 5 REV recovery volume set contains duplicate recovery rows" + )) + )); +} + +#[test] +fn rejects_rar50_rev5_repair_inputs_with_inconsistent_metadata() { + let rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let data = [None; 5]; + assert!(matches!( + repair_rev5_volumes_to(&data[..4], std::slice::from_ref(&rev), |_, _| Ok(())), + Err(Error::InvalidHeader( + "RAR 5 REV data volume count does not match metadata" + )) + )); + + for field in 0..5 { + let mut inconsistent = rev.clone(); + match field { + 0 => inconsistent.version ^= 1, + 1 => inconsistent.data_count += 1, + 2 => inconsistent.recovery_count += 1, + 3 => inconsistent.data_volumes[0].crc32 ^= 1, + 4 => inconsistent.payload.pop().map(|_| ()).unwrap(), + _ => unreachable!(), + } + assert!( + matches!( + repair_rev5_volumes_to(&data, &[rev.clone(), inconsistent], |_, _| Ok(())), + Err(Error::InvalidHeader( + "RAR 5 REV recovery volume metadata differs across files" + )) + ), + "metadata field {field}" + ); + } + + let mut invalid_number = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + invalid_number.recovery_number = 0; + assert!(matches!( + repair_rev5_volumes_to(&data, &[invalid_number], |_, _| Ok(())), + Err(Error::InvalidHeader("RAR 5 REV recovery number is invalid")) + )); +} + +#[test] +fn rejects_non_rar50_input_to_inline_recovery() { + assert!(matches!( + rars::rar50::repair_inline_recovery_bytes(b"not a RAR archive"), + Err(Error::UnsupportedSignature) + )); +} + +#[test] +fn rejects_rar50_rev5_repaired_data_with_wrong_size_or_checksum() { + let data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let inputs = [ + None, + Some(data[1].as_slice()), + Some(data[2].as_slice()), + Some(data[3].as_slice()), + Some(data[4].as_slice()), + ]; + + let mut oversized = rev.clone(); + oversized.data_volumes[0].file_size = oversized.payload_size + 1; + assert!(matches!( + repair_rev5_volumes_to(&inputs, &[oversized], |_, _| Ok(())), + Err(Error::InvalidHeader( + "RAR 5 REV repaired shard is shorter than data volume size" + )) + )); + + let mut wrong_crc = rev; + wrong_crc.data_volumes[0].crc32 ^= 1; + let mut written = false; + assert!(matches!( + repair_rev5_volumes_to(&inputs, &[wrong_crc], |_, _| { + written = true; + Ok(()) + }), + Err(Error::Crc32Mismatch { .. }) + )); + assert!(!written, "invalid reconstructed data must not be published"); +} + +#[test] +fn repairs_corrupt_rar50_data_volume_from_rev5_recovery_volume() { + let mut data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let expected = data.clone(); + data[1][10..60].fill(0xa5); + let first_rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let inputs = [ + Some(data[0].as_slice()), + Some(data[1].as_slice()), + Some(data[2].as_slice()), + Some(data[3].as_slice()), + Some(data[4].as_slice()), + ]; + + let repaired = repair_rev5_volumes(&inputs, &[first_rev]).unwrap(); + + assert_eq!(repaired, expected); +} + +#[test] +fn repairs_truncated_rar50_data_volume_from_rev5_recovery_volume() { + let mut data: Vec<_> = (1..=5) + .map(|index| std::fs::read(fixture(&format!("multivol_rev.part{index}.rar"))).unwrap()) + .collect(); + let expected = data.clone(); + let truncated_len = data[1].len() / 2; + data[1].truncate(truncated_len); + let rev = + Rev5Volume::parse(&std::fs::read(fixture("multivol_rev.part1.rev")).unwrap()).unwrap(); + let inputs: Vec<_> = data.iter().map(|bytes| Some(bytes.as_slice())).collect(); + + let repaired = repair_rev5_volumes(&inputs, &[rev]).unwrap(); + + assert_eq!(repaired, expected); +} + +#[test] +fn rejects_corrupt_rar50_rev5_recovery_volume_checksum() { + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 0x01; + + assert!(matches!( + Rev5Volume::parse(&bytes), + Err(Error::Crc32Mismatch { .. }) + )); +} + +#[test] +fn rejects_rar50_rev5_volume_number_outside_recovery_range() { + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + bytes[21] = 0; + bytes[22] = 0; + update_rev5_header_crc(&mut bytes); + + assert!(matches!( + Rev5Volume::parse(&bytes), + Err(Error::InvalidHeader("RAR 5 REV volume number is invalid")) + )); + + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + bytes[19..21].copy_from_slice(&0u16.to_le_bytes()); + update_rev5_header_crc(&mut bytes); + assert!(matches!( + Rev5Volume::parse(&bytes), + Err(Error::InvalidHeader("RAR 5 REV volume number is invalid")) + )); + + let mut bytes = std::fs::read(fixture("multivol_rev.part1.rev")).unwrap(); + bytes[21..23].copy_from_slice(&7u16.to_le_bytes()); + update_rev5_header_crc(&mut bytes); + assert!(matches!( + Rev5Volume::parse(&bytes), + Err(Error::InvalidHeader("RAR 5 REV volume number is invalid")) + )); +} + +#[test] +fn extracts_rar50_compressed_members() { + for name in ["m1_fastest.rar", "m3_default.rar", "m5_max.rar"] { + let archive = Archive::parse_path(fixture(name)).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 1, "{name}"); + assert!(!files[0].is_stored(), "{name}"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1, "{name}"); + assert_eq!( + extracted[0].data.len(), + files[0].unpacked_size as usize, + "{name}" + ); + } +} + +#[test] +fn parses_real_rar50_compressed_block_framing() { + let archive = Archive::parse_path(fixture("m1_fastest.rar")).unwrap(); + let file = archive.files().next().unwrap(); + let packed = file.packed_data(&archive).unwrap(); + + let block = parse_compressed_block(&packed).unwrap(); + + assert_eq!(block.payload.start, block.header_len); + assert!(block.payload.end <= packed.len()); + assert!(block.header.has_tables); + assert!(block.header.payload_size > 0); + assert!(block.header.payload_bits <= block.header.payload_size * 8); +} + +#[test] +fn parses_real_rar50_compressed_block_tables() { + let archive = Archive::parse_path(fixture("m1_fastest.rar")).unwrap(); + let file = archive.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + let packed = file.packed_data(&archive).unwrap(); + let block = parse_compressed_block(&packed).unwrap(); + let payload = &packed[block.payload]; + + assert!(block.header.has_tables); + let (lengths, table_bits) = read_table_lengths(payload, info.algorithm_version).unwrap(); + let tables = DecodeTables::from_lengths(&lengths).unwrap(); + + assert!(table_bits < block.header.payload_bits); + assert!(!tables.main.is_empty()); + assert!(!tables.distance.is_empty()); + assert!(!tables.length.is_empty()); +} + +#[test] +fn decodes_rar50_m1_fastest_with_lz_codec() { + let archive = Archive::parse_path(fixture("m1_fastest.rar")).unwrap(); + let file = archive.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + let packed = file.packed_data(&archive).unwrap(); + + let decoded = decode_lz(&packed, info.algorithm_version, file.unpacked_size as usize).unwrap(); + + file.verify_integrity(&decoded).unwrap(); +} + +#[test] +fn extracts_rar50_filter_candidate_members() { + for name in [ + "filter_arm.rar", + "filter_delta.rar", + "filter_e8.rar", + "filter_e8e9.rar", + ] { + let archive = Archive::parse_path(fixture(name)).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert_eq!(files.len(), 1, "{name}"); + assert!(!files[0].is_stored(), "{name}"); + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1, "{name}"); + assert_eq!( + extracted[0].data.len(), + files[0].unpacked_size as usize, + "{name}" + ); + } +} + +#[test] +fn rejects_corrupt_rar50_header_checksum() { + let mut bytes = std::fs::read(fixture("stored.rar")).unwrap(); + bytes[13] ^= 0x01; + + assert!(matches!( + Archive::parse(&bytes), + Err(Error::AtArchiveOffset { + offset: 8, + source + }) if matches!(*source, Error::Crc32Mismatch { .. }) + )); +} + +#[test] +fn rejects_overlong_rar50_header_size_vint() { + let mut bytes = std::fs::read(fixture("stored.rar")).unwrap(); + bytes[12..22].fill(0x80); + + assert!(matches!( + Archive::parse(&bytes), + Err(Error::AtArchiveOffset { + offset: 8, + source + }) if matches!(*source, Error::InvalidHeader("RAR 5 vint is too long")) + )); +} + +#[test] +fn rejects_corrupt_rar50_stored_payload_checksum_when_crc32_is_present() { + let mut bytes = std::fs::read(fixture("stored.rar")).unwrap(); + let needle = b"Hello, RAR 5.0 fixture world.\n"; + let offset = bytes + .windows(needle.len()) + .position(|window| window == needle) + .expect("stored payload in fixture"); + bytes[offset] ^= 0x01; + let archive = Archive::parse(&bytes).unwrap(); + + assert!(matches!( + collect_extract(&archive), + Err(Error::AtEntry { + name, + operation: "verifying", + source + }) if name == b"hello.txt" && matches!(*source, Error::Crc32Mismatch { .. }) + )); +} + +#[test] +fn parse_owned_takes_input_buffer_by_value() { + let bytes = fs::read(fixture("empty_file.rar")).unwrap(); + let archive = Archive::parse_owned(bytes.clone()).unwrap(); + assert_eq!(archive.files().count(), 1); + + let with_options = + Archive::parse_owned_with_options(bytes.clone(), ArchiveReadOptions::default()).unwrap(); + assert_eq!(with_options.files().count(), 1); + + let no_password = Archive::parse_owned_with_password(bytes, None).unwrap(); + assert_eq!(no_password.files().count(), 1); +} + +#[test] +fn parse_owned_with_password_unlocks_header_encrypted_archive() { + let bytes = fs::read(fixture("header_encrypted.rar")).unwrap(); + + assert!(matches!( + Archive::parse_owned(bytes.clone()), + Err(Error::NeedPassword) + )); + + let archive = Archive::parse_owned_with_password(bytes.clone(), Some(b"password")).unwrap(); + assert_eq!(archive.files().next().unwrap().name, b"hello.txt"); + + let with_options = + Archive::parse_owned_with_options(bytes, ArchiveReadOptions::with_password(b"password")) + .unwrap(); + assert_eq!(with_options.files().next().unwrap().name, b"hello.txt"); +} + +#[test] +fn parse_with_options_accepts_borrowed_bytes_and_options() { + let bytes = fs::read(fixture("header_encrypted.rar")).unwrap(); + let archive = + Archive::parse_with_options(&bytes, ArchiveReadOptions::with_password(b"password")) + .unwrap(); + assert_eq!(archive.files().next().unwrap().name, b"hello.txt"); +} + +#[test] +fn parse_path_family_accepts_os_string_paths() { + let path: std::ffi::OsString = fixture("empty_file.rar").into_os_string(); + let archive = Archive::parse_path(path.clone()).unwrap(); + assert_eq!(archive.files().count(), 1); + + let with_options = + Archive::parse_path_with_options(path.clone(), ArchiveReadOptions::default()).unwrap(); + assert_eq!(with_options.files().count(), 1); + + let no_password = Archive::parse_path_with_password(path, None).unwrap(); + assert_eq!(no_password.files().count(), 1); +} + +#[test] +fn direct_rar50_parser_rejects_legacy_input_and_stale_signatures() { + let legacy_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar15_40/rars_generated/stored.rar"); + let legacy = fs::read(&legacy_path).unwrap(); + let legacy_signature = detect_archive_family(&legacy).unwrap(); + assert_eq!(legacy_signature.family, ArchiveFamily::Rar15To40); + assert!(matches!( + Archive::parse(&legacy), + Err(Error::UnsupportedSignature) + )); + assert!(matches!( + Archive::parse_path_with_options(&legacy_path, ArchiveReadOptions::new()), + Err(Error::UnsupportedSignature) + )); + assert!(matches!( + Archive::parse_path_with_signature( + &legacy_path, + legacy_signature, + ArchiveReadOptions::new(), + ), + Err(Error::UnsupportedSignature) + )); + + let rar50_path = fixture("empty_file.rar"); + let rar50_bytes = fs::read(&rar50_path).unwrap(); + let signature = detect_archive_family(&rar50_bytes).unwrap(); + let parsed = + Archive::parse_path_with_signature_and_password(&rar50_path, signature, None).unwrap(); + assert_eq!(parsed.files().count(), 1); + + let mut prefixed = vec![0]; + prefixed.extend_from_slice(&rar50_bytes); + let stale = rars::detect::find_archive_start(&prefixed, 1).unwrap(); + assert_eq!(stale.offset, 1); + assert!(matches!( + Archive::parse_path_with_signature(&rar50_path, stale, ArchiveReadOptions::new()), + Err(Error::UnsupportedSignature) + )); +} + +/// Members whose contents overlap heavily, and which together fit inside the +/// default dictionary, so a shared dictionary has something obvious to find. +fn solid_test_entries() -> Vec { + let base: Vec = (0..800u32) + .flat_map(|index| { + let mut bytes = b"solid dictionary sharing payload ".to_vec(); + bytes.extend_from_slice(&index.to_le_bytes()); + bytes + }) + .collect(); + + (0..3u8) + .map(|index| { + rar50::ArchiveEntry::new( + format!("member-{index}.bin").into_bytes(), + rars::EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(base.clone())), + ) + .with_mtime(Some(0x5000_0000)) + .with_attributes(0x20) + }) + .collect() +} + +fn write_streaming(entries: &[rar50::ArchiveEntry], solid: bool) -> Vec { + let mut features = FeatureSet::store_only(); + features.solid = solid; + let mut out = Vec::new(); + rar50::write_streaming_archive_to( + entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(3), + rar50::ArchiveExtras::default(), + &rars::WriterResources::default(), + &mut out, + ) + .unwrap(); + out +} + +#[test] +fn streaming_solid_members_share_one_dictionary() { + let entries = solid_test_entries(); + let solid = write_streaming(&entries, true); + let independent = write_streaming(&entries, false); + + // Members two and three repeat member one, so a dictionary that carries + // across members should collapse them almost entirely. + assert!( + solid.len() * 2 < independent.len(), + "solid archive ({}) should be far smaller than independent members ({})", + solid.len(), + independent.len() + ); + + let archive = Archive::parse(&solid).unwrap(); + assert!(archive.main.is_solid(), "archive must be flagged solid"); + let solid_flags: Vec = archive + .files() + .map(|file| file.decoded_compression_info().unwrap().solid) + .collect(); + assert_eq!( + solid_flags, + vec![false, true, true], + "the first member starts the chain and the rest continue it" + ); +} + +#[test] +fn streaming_solid_archives_round_trip() { + let entries = solid_test_entries(); + let archive_bytes = write_streaming(&entries, true); + let archive = Archive::parse(&archive_bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), entries.len()); + let expected = { + let mut buffer = Vec::new(); + entries[0] + .source + .open() + .unwrap() + .read_to_end(&mut buffer) + .unwrap(); + buffer + }; + for (index, entry) in extracted.iter().enumerate() { + assert_eq!(entry.name, format!("member-{index}.bin").into_bytes()); + assert_eq!(entry.data, expected, "member {index} did not survive"); + } +} + +#[test] +fn streaming_solid_chains_history_across_block_and_member_boundaries() { + // Members larger than the 1 MiB block size, so the dictionary has to carry + // both from block to block and from member to member. + let entries: Vec<_> = (0..3u8) + .map(|index| { + let mut data: Vec = (0..1_600_000u32) + .map(|offset| (offset.wrapping_mul(2_654_435_761) >> 24) as u8) + .collect(); + data[0] = index; + rar50::ArchiveEntry::new( + format!("big-{index}.bin").into_bytes(), + rars::EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data)), + ) + .with_attributes(0x20) + }) + .collect(); + + let archive_bytes = write_streaming(&entries, true); + let archive = Archive::parse(&archive_bytes).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 3); + for (index, entry) in extracted.iter().enumerate() { + let mut expected = Vec::new(); + entries[index] + .source + .open() + .unwrap() + .read_to_end(&mut expected) + .unwrap(); + assert_eq!(entry.data, expected, "member {index} did not survive"); + } + + if let Some(output) = reference_test_archive("solid-multiblock", &archive_bytes) { + assert!( + output.status.success(), + "the reference tool rejected a multi-block solid archive\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[test] +fn streaming_solid_output_does_not_depend_on_the_memory_budget() { + let entries = solid_test_entries(); + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = rar50::WriterOptions::new(ArchiveVersion::Rar50, features); + + // The default budget compresses a couple of blocks at a time; the larger + // one lets every core work at once. + let mut tight = Vec::new(); + rar50::write_streaming_archive_to( + &entries, + options, + rar50::ArchiveExtras::default(), + &rars::WriterResources::default(), + &mut tight, + ) + .unwrap(); + + let mut roomy = Vec::new(); + rar50::write_streaming_archive_to( + &entries, + options, + rar50::ArchiveExtras::default(), + &rars::WriterResources::new(4 * 1024 * 1024 * 1024), + &mut roomy, + ) + .unwrap(); + + assert_eq!( + tight, roomy, + "the number of blocks compressed at once must not change the output" + ); +} + +/// Tests an archive with whichever reference tool is installed, returning +/// `None` when neither is. +fn reference_test_archive(label: &str, archive: &[u8]) -> Option { + let dir = scratch::case(&format!("rars-{label}")); + let path = dir.join("archive.rar"); + fs::write(&path, archive).unwrap(); + + let mut result = None; + for tool in ["unrar", "rar"] { + match Command::new(tool).arg("t").arg(&path).output() { + Ok(output) => { + result = Some(output); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => panic!("failed to run {tool}: {error}"), + } + } + result +} + +#[test] +#[ignore = "requires a local unrar or rar command"] +fn reference_rar_accepts_streaming_solid_archive() { + let archive = write_streaming(&solid_test_entries(), true); + let Some(output) = reference_test_archive("streaming-solid", &archive) else { + eprintln!("skipping reference test: no unrar or rar command is installed"); + return; + }; + + assert!( + output.status.success(), + "the reference tool rejected the streaming solid archive\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +/// The exact flag combination from issue #29, which used to abort trying to +/// allocate more than ten gigabytes. +#[test] +fn streaming_writer_handles_solid_encrypted_headers_and_recovery_together() { + let entries: Vec<_> = (0..3u8) + .map(|index| { + let data: Vec = (0..900_000u32) + .map(|offset| (offset.wrapping_mul(2_654_435_761) >> 25) as u8) + .chain(std::iter::once(index)) + .collect(); + rar50::ArchiveEntry::new( + format!("member-{index}.bin"), + rars::EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data)), + ) + .with_attributes(0x20) + .with_password(b"issue-29".to_vec()) + }) + .collect(); + + let mut features = FeatureSet::store_only(); + features.solid = true; + features.header_encryption = true; + let options = + rar50::WriterOptions::new(ArchiveVersion::Rar70, features).with_compression_level(5); + + // A budget far below the archive size: if anything buffered a member or + // the archive, this could not succeed. + let resources = rars::WriterResources::new(192 * 1024 * 1024); + let mut archive_bytes = Vec::new(); + rar50::write_streaming_archive_to( + &entries, + options, + rar50::ArchiveExtras::default().with_recovery_percent(Some(10)), + &resources, + &mut archive_bytes, + ) + .unwrap(); + + let archive = Archive::parse_with_options( + &archive_bytes, + ArchiveReadOptions::with_password(b"issue-29"), + ) + .unwrap(); + assert!(archive.main.is_solid()); + assert!(archive.main.has_recovery_record()); + + let extracted = collect_extract_with_password(&archive, Some(b"issue-29")).unwrap(); + assert_eq!(extracted.len(), 3); + for (index, entry) in extracted.iter().enumerate() { + let mut expected = Vec::new(); + entries[index] + .source + .open() + .unwrap() + .read_to_end(&mut expected) + .unwrap(); + assert_eq!(entry.data, expected, "member {index} did not survive"); + } + + if let Some(output) = + reference_test_archive_with_password("issue-29", &archive_bytes, "issue-29") + { + assert!( + output.status.success(), + "the reference tool rejected the archive\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + // The recovery record has to be usable, not merely present. + let data_range = archive.files().next().unwrap().block.data_range.clone(); + let mut damaged = archive_bytes.clone(); + damaged[data_range.start + 32..data_range.start + 512].fill(0xa5); + + let damaged_archive = + Archive::parse_with_options(&damaged, ArchiveReadOptions::with_password(b"issue-29")) + .unwrap(); + let mut repaired = Vec::new(); + damaged_archive.repair_recovery_to(&mut repaired).unwrap(); + assert_eq!( + repaired, archive_bytes, + "the recovery record should restore the archive exactly" + ); +} + +fn reference_test_archive_with_password( + label: &str, + archive: &[u8], + password: &str, +) -> Option { + let dir = scratch::case(&format!("rars-{label}")); + let path = dir.join("archive.rar"); + fs::write(&path, archive).unwrap(); + + let mut result = None; + for tool in ["unrar", "rar"] { + match Command::new(tool) + .arg("t") + .arg(format!("-p{password}")) + .arg(&path) + .output() + { + Ok(output) => { + result = Some(output); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => panic!("failed to run {tool}: {error}"), + } + } + result +} + +/// Generates deterministic incompressible bytes without holding them, so the +/// source itself does not dominate the memory being measured. +struct GeneratedSource { + remaining: u64, + state: u64, +} + +impl std::io::Read for GeneratedSource { + fn read(&mut self, buffer: &mut [u8]) -> IoResult { + let want = buffer.len().min(self.remaining as usize); + for slot in buffer.iter_mut().take(want) { + self.state = self + .state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + *slot = (self.state >> 33) as u8; + } + self.remaining -= want as u64; + Ok(want) + } +} + +impl std::io::Seek for GeneratedSource { + fn seek(&mut self, _: std::io::SeekFrom) -> IoResult { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "generated sources are read once per pass", + )) + } +} + +/// Writes an archive far larger than the memory budget it is given, with a +/// recovery record whose parity also exceeds that budget. Run with +/// `/usr/bin/time -v` to see the peak resident size. +#[test] +#[ignore = "writes a 300 MiB archive; run manually to measure peak memory"] +fn streaming_writer_stays_within_its_memory_budget_on_a_large_archive() { + const MEMBER_BYTES: u64 = 300 * 1024 * 1024; + const BUDGET: u64 = 128 * 1024 * 1024; + + let entry = rar50::ArchiveEntry::new( + "large.bin", + rars::EntrySource::from_opener(MEMBER_BYTES, || { + Ok(Box::new(GeneratedSource { + remaining: MEMBER_BYTES, + state: 0x2545_f491_4f6c_dd1d, + })) + }), + ) + .with_password(b"issue-29".to_vec()); + + let mut features = FeatureSet::store_only(); + features.solid = true; + features.header_encryption = true; + let options = + rar50::WriterOptions::new(ArchiveVersion::Rar70, features).with_compression_level(1); + + let dir = scratch::case("rars-large"); + let temp = dir.join("archive.rar"); + let mut output = fs::File::create(&temp).unwrap(); + // 50% recovery over a ~300 MiB archive needs more parity than the budget + // allows to hold at once, which forces the striped recovery pass. + rar50::write_streaming_archive_to( + std::slice::from_ref(&entry), + options, + rar50::ArchiveExtras::default().with_recovery_percent(Some(50)), + &rars::WriterResources::new(BUDGET).with_temp_dir(&*dir), + &mut output, + ) + .unwrap(); + drop(output); + + let written = fs::metadata(&temp).unwrap().len(); + assert!( + written > MEMBER_BYTES, + "a 50% recovery record should make the archive larger than its input" + ); +} + +fn streaming_entry(name: &str, data: &[u8]) -> rar50::ArchiveEntry { + rar50::ArchiveEntry::new( + name, + rars::EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data.to_vec())), + ) + .with_attributes(0x20) +} + +fn write_with_extras( + entries: &[rar50::ArchiveEntry], + features: FeatureSet, + extras: rar50::ArchiveExtras<'_>, +) -> Vec { + let mut out = Vec::new(); + rar50::write_streaming_archive_to( + entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(1), + extras, + &rars::WriterResources::default(), + &mut out, + ) + .unwrap(); + out +} + +#[test] +fn streaming_writer_stores_an_archive_comment() { + let entries = [streaming_entry("member.txt", b"comment carrier payload\n")]; + let features = FeatureSet::store_only(); + + let bytes = write_with_extras( + &entries, + features, + rar50::ArchiveExtras::default().with_comment(b"streamed archive comment"), + ); + + let archive = Archive::parse(&bytes).unwrap(); + let comment = archive + .blocks + .iter() + .find_map(|block| match block { + Block::Service(header) if header.name == b"CMT" => Some(header), + _ => None, + }) + .expect("archive has a comment service record"); + assert_eq!(comment.unpacked_size, 24); + assert_eq!(collect_extract(&archive).unwrap().len(), 1); +} + +#[test] +fn streaming_writer_records_archive_metadata() { + let entries = [streaming_entry("member.txt", b"metadata carrier\n")]; + let bytes = write_with_extras( + &entries, + FeatureSet::store_only(), + rar50::ArchiveExtras::default().with_metadata(rar50::ArchiveMetadataEntry { + name: Some(b"original.rar"), + creation_time: Some(0x01D9_0000_0000_0000), + }), + ); + + let archive = Archive::parse(&bytes).unwrap(); + let metadata = archive + .main + .extras + .iter() + .find_map(|extra| match extra { + rar50::MainExtraRecord::ArchiveMetadata(record) => Some(record), + _ => None, + }) + .expect("main header carries archive metadata"); + assert_eq!(metadata.name.as_deref(), Some(b"original.rar".as_slice())); +} + +#[test] +fn streaming_writer_attaches_file_services() { + let entries = [streaming_entry("member.txt", b"service carrier payload\n") + .with_service(rar50::ServiceEntry::new("CMT", "a file comment"))]; + let features = FeatureSet::store_only(); + + let bytes = write_with_extras(&entries, features, rar50::ArchiveExtras::default()); + let archive = Archive::parse(&bytes).unwrap(); + + let services: Vec<_> = archive + .blocks + .iter() + .filter_map(|block| match block { + Block::Service(header) => Some(header.name.clone()), + _ => None, + }) + .collect(); + assert_eq!(services, vec![b"CMT".to_vec()]); + assert_eq!(collect_extract(&archive).unwrap().len(), 1); +} + +#[test] +fn streaming_writer_writes_a_usable_quick_open_index() { + let entries: Vec<_> = (0..4u8) + .map(|index| { + streaming_entry( + &format!("member-{index}.txt"), + format!("quick open payload {index}\n").repeat(8).as_bytes(), + ) + }) + .collect(); + let mut features = FeatureSet::store_only(); + features.quick_open = true; + + // Asked for through the feature set alone, which is the only way to ask. + // This used to say it twice, through an `ArchiveExtras` field as well, and + // that is why it passed while the feature set on its own was being dropped. + let bytes = write_with_extras(&entries, features, rar50::ArchiveExtras::default()); + + let archive = Archive::parse(&bytes).unwrap(); + assert!( + archive + .blocks + .iter() + .any(|block| matches!(block, Block::Service(header) if header.name == b"QO")), + "archive has a quick-open index" + ); + assert_eq!(collect_extract(&archive).unwrap().len(), 4); + + if let Some(output) = reference_test_archive("streaming-quick-open", &bytes) { + assert!( + output.status.success(), + "the reference tool rejected the quick-open archive\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[test] +fn streaming_writer_applies_filters_when_asked() { + // x86-shaped data, which the E8 filter is meant to help with. + let mut data = Vec::new(); + for index in 0..20_000u32 { + data.push(0xe8); + data.extend_from_slice(&index.to_le_bytes()); + data.extend_from_slice(b"\x55\x89\xe5"); + } + let entries = [streaming_entry("program.bin", &data)]; + + let plain = write_with_extras( + &entries, + FeatureSet::store_only(), + rar50::ArchiveExtras::default(), + ); + let filtered = write_with_extras( + &entries, + FeatureSet::store_only(), + rar50::ArchiveExtras::default().with_filter_policy(rar50::FilterPolicy::Auto), + ); + + assert!( + filtered.len() < plain.len(), + "the filter should pay for itself: {} filtered vs {} plain", + filtered.len(), + plain.len() + ); + + let archive = Archive::parse(&filtered).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); +} + +#[test] +fn streaming_writer_drops_an_automatic_filter_it_cannot_afford() { + // Too large to hold whole under this budget, so the filter is skipped + // rather than the write failing. + let data = vec![0xe8u8; 8 * 1024 * 1024]; + let entries = [streaming_entry("large.bin", &data)]; + + let mut out = Vec::new(); + rar50::write_streaming_archive_to( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + rar50::ArchiveExtras::default().with_filter_policy(rar50::FilterPolicy::Auto), + &rars::WriterResources::new(160 * 1024 * 1024), + &mut out, + ) + .unwrap(); + + let archive = Archive::parse(&out).unwrap(); + assert_eq!(collect_extract(&archive).unwrap()[0].data, data); +} + +/// Barely compressible bytes, so the packed payload stays big enough to span +/// several volumes. +fn volume_payload(len: u32) -> Vec { + let mut state = 0x2545_f491_4f6c_dd1du64; + (0..len) + .map(|_| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + (state >> 33) as u8 + }) + .collect() +} + +/// What the capability table says about a RAR 5 volume set has to be what the +/// volume writer does. +/// +/// The table used to promise comments, metadata and quick-open for split RAR 5 +/// sets while the writer refused all three, so refusals for other formats sent +/// people to rar50 and rar50 failed too. The matrix test in write_plan.rs +/// compares `supports` with `validate_plan`, which are two halves of the same +/// table; nothing compared either with the writer. +#[test] +fn the_volume_writer_refuses_exactly_what_the_table_refuses() { + use rars::WriterOption; + + let payload = volume_payload(40_000); + let entries = [streaming_entry("member.bin", &payload)]; + let shape = rars::PlanShape::new().compressed(true).volumes(true); + + for option in [ + WriterOption::ArchiveComment, + WriterOption::ArchiveMetadata, + WriterOption::FileComment, + WriterOption::Feature(rars::Feature::QuickOpen), + WriterOption::RecoveryRecord, + ] { + let mut features = FeatureSet::store_only(); + let mut extras = rar50::ArchiveExtras::default(); + let mut entries = entries.clone(); + match option { + WriterOption::ArchiveComment => extras = extras.with_comment(b"comment"), + WriterOption::ArchiveMetadata => { + extras = extras.with_metadata(rar50::ArchiveMetadataEntry { + name: Some(b"set.rar"), + creation_time: Some(0x01dc_d60e_662d_7a32), + }) + } + WriterOption::FileComment => { + entries[0] = entries[0] + .clone() + .with_service(rar50::ServiceEntry::new(b"CMT".to_vec(), b"note".to_vec())) + } + WriterOption::Feature(rars::Feature::QuickOpen) => features.quick_open = true, + WriterOption::RecoveryRecord => extras = extras.with_recovery_percent(Some(5)), + _ => unreachable!("every option in the list above is handled"), + } + + let mut sink = rar50::CollectedVolumes::new(); + let result = rar50::write_streaming_volumes_to( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(1), + extras, + 20_000, + &mut sink, + &rars::WriterResources::default(), + ); + + assert_eq!( + result.is_ok(), + rars::supports(ArchiveVersion::Rar50, option, shape), + "{option:?}: the table and the volume writer disagree, writer said {result:?}" + ); + } +} + +/// A file comment on a volume set was validated with the member and then +/// dropped: `prepare_volume_member` never reads an entry's services, so the +/// comment appeared in no volume and the write reported success. +/// +/// Asserting the refusal alone would not have caught it, because the +/// capability table agreed with the writer that it was supported. What the +/// writer must never do is accept the request and lose it. +#[test] +fn a_file_comment_on_a_volume_set_is_refused_or_written() { + let payload = volume_payload(40_000); + let entries = [ + streaming_entry("member.bin", &payload).with_service(rar50::ServiceEntry::new( + b"CMT".to_vec(), + b"volume note".to_vec(), + )), + ]; + + let mut sink = rar50::CollectedVolumes::new(); + let result = rar50::write_streaming_volumes_to( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + rar50::ArchiveExtras::default(), + 20_000, + &mut sink, + &rars::WriterResources::default(), + ); + + if result.is_err() { + return; + } + let volumes = sink.take(); + let found = volumes.iter().any(|volume| { + Archive::parse(volume) + .is_ok_and(|archive| archive.services().any(|service| service.name == b"CMT")) + }); + assert!( + found, + "the write was accepted, so the comment has to be in one of the {} volumes", + volumes.len() + ); +} + +/// A set with no members used to report success while writing no volumes at +/// all: the member loop never ran, and the writer finished without asking the +/// sink to start one. The volume writer that was deleted refused this. +#[test] +fn a_volume_set_with_no_members_is_refused() { + let mut sink = rar50::CollectedVolumes::new(); + let result = rar50::write_streaming_volumes_to( + &[], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()), + rar50::ArchiveExtras::default(), + 20_000, + &mut sink, + &rars::WriterResources::default(), + ); + + assert!(result.is_err(), "an empty set reported success"); + assert!(sink.take().is_empty()); +} + +/// Solid members are coded as one chain, so the filter search never runs for +/// them. Both entry points have to say so; the archive path used to and the +/// volume path silently dropped the request. +#[test] +fn both_rar50_entry_points_refuse_a_filter_with_solid() { + let payload = volume_payload(40_000); + let entries = [streaming_entry("member.bin", &payload)]; + let mut features = FeatureSet::store_only(); + features.solid = true; + let options = + rar50::WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(1); + let extras = || { + rar50::ArchiveExtras::default().with_filter_policy(rar50::FilterPolicy::Explicit( + rars::FilterSpec::whole(rars::FilterKind::E8), + )) + }; + + let mut sink = rar50::CollectedVolumes::new(); + let volumes = rar50::write_streaming_volumes_to( + &entries, + options, + extras(), + 20_000, + &mut sink, + &rars::WriterResources::default(), + ); + let mut out = Vec::new(); + let archive = rar50::write_streaming_archive_to( + &entries, + options, + extras(), + &rars::WriterResources::default(), + &mut out, + ); + + assert!( + volumes.is_err(), + "the volume path took the filter and dropped it" + ); + assert!(archive.is_err()); +} + +fn write_volume_set( + entries: &[rar50::ArchiveEntry], + features: FeatureSet, + extras: rar50::ArchiveExtras<'_>, + max_payload: u64, +) -> Vec> { + let mut sink = rar50::CollectedVolumes::new(); + rar50::write_streaming_volumes_to( + entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(1), + extras, + max_payload, + &mut sink, + &rars::WriterResources::default(), + ) + .unwrap(); + sink.take() +} + +/// A volume set used to report nothing at all: the writer built its plan with +/// `progress: None` and its compression stage was called without the callback +/// the single-archive path uses. +#[test] +fn a_volume_set_reports_compression_progress() { + use rars::{WriteOperation, WriteProgressEvent}; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + + // Blocks are a megabyte, so three of them is what proves the reporting is + // incremental rather than one jump at the end. + let payload = volume_payload(3 * 1024 * 1024); + let entries = [streaming_entry("progress.bin", &payload)]; + + let started = AtomicBool::new(false); + let finished = AtomicBool::new(false); + let last = AtomicU64::new(0); + let advances = AtomicU64::new(0); + let reporter = |event: WriteProgressEvent<'_>| match event { + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Compression, + total_bytes, + .. + } => { + assert_eq!(total_bytes, Some(payload.len() as u64)); + started.store(true, Ordering::Relaxed); + } + WriteProgressEvent::Advanced { + operation: WriteOperation::Compression, + completed_bytes, + total_bytes, + .. + } => { + assert!(completed_bytes >= last.swap(completed_bytes, Ordering::Relaxed)); + assert!(completed_bytes <= total_bytes); + advances.fetch_add(1, Ordering::Relaxed); + } + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Compression, + .. + } => finished.store(true, Ordering::Relaxed), + _ => {} + }; + + let mut sink = rar50::CollectedVolumes::new(); + rar50::write_streaming_volumes_with_progress( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + rar50::ArchiveExtras::default(), + 900_000, + &mut sink, + &rars::WriterResources::default(), + Some(&reporter), + ) + .unwrap(); + + assert!(started.load(Ordering::Relaxed), "no operation start"); + assert!(finished.load(Ordering::Relaxed), "no operation finish"); + assert!( + advances.load(Ordering::Relaxed) >= 3, + "expected a report per block, got {}", + advances.load(Ordering::Relaxed) + ); + assert_eq!(last.load(Ordering::Relaxed), payload.len() as u64); + assert!(sink.take().len() > 1); +} + +#[test] +fn streaming_volume_set_round_trips() { + let payload = volume_payload(120_000); + let entries = [streaming_entry("split.bin", &payload)]; + + let volumes = write_volume_set( + &entries, + FeatureSet::store_only(), + rar50::ArchiveExtras::default(), + 20_000, + ); + assert!(volumes.len() > 2, "expected several volumes"); + + let archives: Vec<_> = volumes + .iter() + .map(|bytes| Archive::parse(bytes).unwrap()) + .collect(); + let extracted = collect_extract_volumes(&archives).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn streaming_volume_set_carries_encryption_across_splits() { + let payload = volume_payload(90_000); + let entries = [streaming_entry("secret.bin", &payload).with_password(b"volume-pass".to_vec())]; + let features = FeatureSet::store_only(); + + // A payload size that is not a multiple of the cipher block, so splits + // land mid-block. + let volumes = write_volume_set(&entries, features, rar50::ArchiveExtras::default(), 12_345); + assert!(volumes.len() > 2); + + let archives: Vec<_> = volumes + .iter() + .map(|bytes| { + Archive::parse_with_options(bytes, ArchiveReadOptions::with_password(b"volume-pass")) + .unwrap() + }) + .collect(); + let extracted = collect_extract_volumes_with_password(&archives, Some(b"volume-pass")).unwrap(); + assert_eq!(extracted[0].data, payload); +} + +#[test] +fn streaming_volume_set_recovers_each_volume() { + let payload = volume_payload(80_000); + let entries = [streaming_entry("protected.bin", &payload)]; + let features = FeatureSet::store_only(); + + let volumes = write_volume_set( + &entries, + features, + rar50::ArchiveExtras::default().with_recovery_percent(Some(20)), + 30_000, + ); + assert!(volumes.len() > 1); + + // Damage the first volume's payload and repair it from its own record. + let clean = Archive::parse(&volumes[0]).unwrap(); + let data_range = clean.files().next().unwrap().block.data_range.clone(); + let mut damaged = volumes[0].clone(); + damaged[data_range.start + 16..data_range.start + 200].fill(0xa5); + + let damaged_archive = Archive::parse(&damaged).unwrap(); + let mut repaired = Vec::new(); + damaged_archive.repair_recovery_to(&mut repaired).unwrap(); + assert_eq!(repaired, volumes[0]); +} + +#[test] +#[ignore = "requires a local unrar or rar command"] +fn reference_rar_accepts_a_streaming_volume_set() { + let payload = volume_payload(150_000); + let entries = [streaming_entry("split.bin", &payload)]; + let volumes = write_volume_set( + &entries, + FeatureSet::store_only(), + rar50::ArchiveExtras::default(), + 25_000, + ); + assert!(volumes.len() > 2); + + let dir = scratch::case("rars-volset"); + for (index, volume) in volumes.iter().enumerate() { + fs::write(dir.join(format!("set.part{:02}.rar", index + 1)), volume).unwrap(); + } + + let first = dir.join("set.part01.rar"); + let mut result = None; + for tool in ["unrar", "rar"] { + match Command::new(tool).arg("t").arg(&first).output() { + Ok(output) => { + result = Some(output); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => panic!("failed to run {tool}: {error}"), + } + } + let Some(output) = result else { + eprintln!("skipping reference test: no unrar or rar command is installed"); + return; + }; + assert!( + output.status.success(), + "the reference tool rejected the volume set\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn zero_fill_payload() -> Vec { + let mut expected = vec![0u8; 8]; + for _ in 0..8 { + expected.extend_from_slice(b"zero-fill regression payload "); + } + expected +} + +#[test] +fn extracts_rar50_match_that_reaches_past_the_start_of_the_window() { + let bytes = std::fs::read(fixture("zero_fill_out_of_window.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let file = archive.files().next().unwrap(); + + let entry = collect_file(&archive, file).unwrap(); + + assert_eq!(entry.name, b"zerofill.bin"); + assert_eq!(entry.data, zero_fill_payload()); +} + +#[test] +fn streams_rar50_match_that_reaches_past_the_start_of_the_window() { + let bytes = std::fs::read(fixture("zero_fill_out_of_window.rar")).unwrap(); + let archive = Archive::parse(&bytes).unwrap(); + let entries = RefCell::new(Vec::new()); + + archive + .extract_to( + ArchiveReadOptions::default().with_rar50_buffered_decode_limit(0), + |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries + .borrow_mut() + .push((meta.name.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + }, + ) + .unwrap(); + assert_eq!(entries.borrow().len(), 1); + assert_eq!(entries.borrow()[0].0, b"zerofill.bin"); + assert_eq!(*entries.borrow()[0].1.borrow(), zero_fill_payload()); + entries.borrow_mut().clear(); + + extract_volumes_to( + std::slice::from_ref(&archive), + ArchiveReadOptions::default().with_rar50_buffered_decode_limit(0), + |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries + .borrow_mut() + .push((meta.name.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + }, + ) + .unwrap(); + + let entries = entries.into_inner(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, b"zerofill.bin"); + assert_eq!(*entries[0].1.borrow(), zero_fill_payload()); +} + +/// The KDF count byte from the archive encryption header, which sits in the +/// clear right after the marker: CRC32, then vints for header size, type 4, +/// header flags, encryption version and encryption flags, then the byte. +fn head_crypt_kdf_count(bytes: &[u8]) -> u8 { + let mut pos = 8 + 4; + let read_vint = |pos: &mut usize| { + let mut value = 0u64; + let mut shift = 0; + loop { + let byte = bytes[*pos]; + *pos += 1; + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + if byte & 0x80 == 0 { + return value; + } + } + }; + read_vint(&mut pos); // header size + assert_eq!(read_vint(&mut pos), 4, "first block is not HEAD_CRYPT"); + read_vint(&mut pos); // header flags + assert_eq!(read_vint(&mut pos), 0, "unexpected encryption version"); + read_vint(&mut pos); // encryption flags + bytes[pos] +} diff --git a/arca-rar/vendor/rars/tests/rar50_golden.rs b/arca-rar/vendor/rars/tests/rar50_golden.rs new file mode 100644 index 00000000..8cda1b7b --- /dev/null +++ b/arca-rar/vendor/rars/tests/rar50_golden.rs @@ -0,0 +1,243 @@ +#![cfg(feature = "write")] + +//! Byte-for-byte golden fixtures for structural RAR 5/7 writer output. +//! +//! These cover *stored* archives only. Their bytes exercise header layout, +//! locator records, quick-open, recovery records, comments, metadata and +//! volume splitting without involving the LZ encoder, so they must stay +//! identical across writer refactors. Compressed output is deliberately +//! excluded: block granularity is an implementation detail that the streaming +//! writer is expected to change, and it is covered by round-trip and external +//! `unrar` tests instead. +//! +//! Regenerate after an intentional format change with: +//! `RARS_BLESS_GOLDEN=1 cargo test -p rars --test rar50_golden` + +use rars::rar50::{ArchiveEntry, ArchiveMetadataEntry, Rar50Writer, WriterOptions}; +use rars::{ArchiveVersion, EntrySource, Error, FeatureSet, WriterResources}; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; + +fn golden_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/golden") +} + +fn blessing() -> bool { + std::env::var_os("RARS_BLESS_GOLDEN").is_some() +} + +/// Deterministic, mildly compressible bytes so recovery parity has real work +/// to do while fixtures stay reviewable. +fn deterministic_bytes(len: usize, seed: u32) -> Vec { + let mut state = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + (0..len) + .map(|index| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + if index % 7 == 0 { + b'A' + (index % 23) as u8 + } else { + (state >> 24) as u8 + } + }) + .collect() +} + +fn entry(name: &[u8], data: &[u8]) -> ArchiveEntry { + ArchiveEntry::new( + name.to_vec(), + EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(data.to_vec())), + ) + .with_mtime(Some(0x5000_0000)) + .with_attributes(0x20) +} + +fn assert_golden(name: &str, produced: &[u8]) { + let path = golden_dir().join(name); + if blessing() { + std::fs::create_dir_all(golden_dir()).expect("create golden fixture directory"); + std::fs::write(&path, produced).expect("write golden fixture"); + return; + } + + let expected = std::fs::read(&path).unwrap_or_else(|error| { + panic!( + "missing golden fixture {}: {error}\n\ + regenerate with RARS_BLESS_GOLDEN=1 cargo test -p rars --test rar50_golden", + path.display() + ) + }); + + if expected == produced { + return; + } + + let first_difference = expected + .iter() + .zip(produced) + .position(|(left, right)| left != right); + panic!( + "{} changed: golden is {} bytes, produced {} bytes, first difference at {:?}\n\ + if this change is intentional, regenerate with \ + RARS_BLESS_GOLDEN=1 cargo test -p rars --test rar50_golden", + name, + expected.len(), + produced.len(), + first_difference + ); +} + +/// Large fixtures are pinned by digest rather than stored whole, so the repo +/// does not carry hundreds of kilobytes of binary per case. +fn assert_golden_digest(name: &str, produced: &[u8]) { + let hash = Sha256::digest(produced) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let digest = format!("{hash} {} bytes\n", produced.len()); + let path = golden_dir().join(name); + if blessing() { + std::fs::create_dir_all(golden_dir()).expect("create golden fixture directory"); + std::fs::write(&path, digest).expect("write golden digest"); + return; + } + + let expected = std::fs::read_to_string(&path).unwrap_or_else(|error| { + panic!( + "missing golden digest {}: {error}\n\ + regenerate with RARS_BLESS_GOLDEN=1 cargo test -p rars --test rar50_golden", + path.display() + ) + }); + assert_eq!( + expected, digest, + "{name} digest changed; if intentional, regenerate with \ + RARS_BLESS_GOLDEN=1 cargo test -p rars --test rar50_golden" + ); +} + +fn stored_options(target: ArchiveVersion) -> WriterOptions { + WriterOptions::new(target, FeatureSet::store_only()).with_compression_level(0) +} + +fn write_stored(entries: &[ArchiveEntry], options: WriterOptions) -> Result, Error> { + Rar50Writer::new(options).entries(entries.to_vec()).finish() +} + +#[test] +fn golden_stored_archive_layout_is_stable() { + let first = deterministic_bytes(4096, 1); + let second = deterministic_bytes(37, 2); + let entries = [entry(b"first.bin", &first), entry(b"second.bin", &second)]; + + assert_golden( + "stored_rar50.rar", + &write_stored(&entries, stored_options(ArchiveVersion::Rar50)).unwrap(), + ); + assert_golden( + "stored_rar70.rar", + &write_stored(&entries, stored_options(ArchiveVersion::Rar70)).unwrap(), + ); +} + +#[test] +fn golden_comment_and_metadata_layout_is_stable() { + let data = deterministic_bytes(1024, 3); + let entries = [entry(b"payload.bin", &data)]; + + let features = FeatureSet::store_only(); + let options = WriterOptions::new(ArchiveVersion::Rar50, features); + + let commented = Rar50Writer::new(options) + .entries(entries.to_vec()) + .archive_comment(Some(b"golden archive comment")) + .finish() + .unwrap(); + assert_golden("stored_comment.rar", &commented); + + let with_metadata = Rar50Writer::new(options) + .entries(entries.to_vec()) + .archive_comment(Some(b"golden archive comment")) + .archive_metadata(Some(ArchiveMetadataEntry { + name: Some(b"golden.rar"), + creation_time: Some(0x01D9_0000_0000_0000), + })) + .finish() + .unwrap(); + assert_golden("stored_comment_metadata.rar", &with_metadata); +} + +#[test] +fn golden_quick_open_layout_is_stable() { + let data = deterministic_bytes(2048, 4); + let entries = [ + entry(b"alpha.bin", &data), + entry(b"beta.bin", &data[..512]), + entry(b"gamma.bin", &data[..7]), + ]; + + let mut features = FeatureSet::store_only(); + features.quick_open = true; + let options = WriterOptions::new(ArchiveVersion::Rar50, features); + + assert_golden( + "stored_quick_open.rar", + &write_stored(&entries, options).unwrap(), + ); +} + +#[test] +fn golden_recovery_record_layout_is_stable() { + let data = deterministic_bytes(8192, 5); + let entries = [entry(b"protected.bin", &data)]; + + let options = stored_options(ArchiveVersion::Rar50); + + for percent in [1u64, 5, 10, 50] { + let archive = Rar50Writer::new(options) + .entries(entries.to_vec()) + .recovery_percent(Some(percent)) + .finish() + .unwrap(); + assert_golden(&format!("stored_recovery_{percent}.rar"), &archive); + } +} + +#[test] +fn golden_recovery_record_over_200kib_is_stable() { + // Above 200 KiB the recovery planner switches to the fixed 200-data-shard + // layout, which is the case real archives hit. + let data = deterministic_bytes(300 * 1024, 6); + let entries = [entry(b"large.bin", &data)]; + + let options = stored_options(ArchiveVersion::Rar50); + + let archive = Rar50Writer::new(options) + .entries(entries.to_vec()) + .recovery_percent(Some(10)) + .finish() + .unwrap(); + assert_golden_digest("stored_recovery_large.sha256", &archive); +} + +#[test] +fn golden_volume_set_layout_is_stable() { + let data = deterministic_bytes(5000, 7); + let options = stored_options(ArchiveVersion::Rar50); + + let mut sink = rars::rar50::CollectedVolumes::new(); + rars::rar50::write_streaming_volumes_to( + &[entry(b"split.bin", &data)], + options, + rars::rar50::ArchiveExtras::default(), + 1500, + &mut sink, + &WriterResources::default(), + ) + .unwrap(); + let volumes = sink.take(); + + assert!(volumes.len() > 1, "expected a multi-volume set"); + for (index, volume) in volumes.iter().enumerate() { + assert_golden(&format!("stored_volume_{index}.rar"), volume); + } +} diff --git a/arca-rar/vendor/rars/tests/rar50_split_limits.rs b/arca-rar/vendor/rars/tests/rar50_split_limits.rs new file mode 100644 index 00000000..ee732776 --- /dev/null +++ b/arca-rar/vendor/rars/tests/rar50_split_limits.rs @@ -0,0 +1,180 @@ +#![cfg(feature = "write")] + +use rars::rar50::{self, Archive, Block}; +use rars::{ArchiveReadOptions, Error}; +use std::cell::RefCell; +use std::io::{self, Write}; +use std::path::Path; +use std::rc::Rc; + +#[path = "support/scratch.rs"] +mod scratch; + +struct Capture(Rc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn options(limit: u64, password: Option<&[u8]>) -> ArchiveReadOptions<'_> { + password + .map_or_else( + ArchiveReadOptions::default, + ArchiveReadOptions::with_password, + ) + .with_rar50_buffered_decode_limit(limit) +} + +fn fixtures(prefix: &str, count: usize, padded: bool, password: Option<&[u8]>) -> Vec { + (1..=count) + .map(|index| { + let number = if padded { + format!("{index:02}") + } else { + index.to_string() + }; + Archive::parse_path_with_password( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar50") + .join(format!("{prefix}.part{number}.rar")), + password, + ) + .unwrap() + }) + .collect() +} + +fn collect( + volumes: &[Archive], + limit: u64, + password: Option<&[u8]>, +) -> Result>, Error> { + let mut outputs = Vec::new(); + rar50::extract_volumes_to(volumes, options(limit, password), |_| { + let data = Rc::new(RefCell::new(Vec::new())); + outputs.push(data.clone()); + Ok(Box::new(Capture(data))) + })?; + Ok(outputs + .into_iter() + .map(|data| data.borrow().clone()) + .collect()) +} + +fn cause(error: &Error) -> &Error { + match error { + Error::AtEntry { source, .. } + | Error::AtArchiveOffset { source, .. } + | Error::InVolume { source, .. } => cause(source), + error => error, + } +} + +#[test] +fn streamed_split_members_match_buffered_plain_encrypted_and_solid_fixtures() { + for (prefix, count, padded, password) in [ + ("multivol", 3, false, None), + ("encrypted_multivol", 3, false, Some(b"password".as_slice())), + ("solid_multivol", 6, true, None), + ] { + let volumes = fixtures(prefix, count, padded, password); + let expected = collect(&volumes, u64::MAX, password).unwrap(); + let streamed = collect(&volumes, 1, password).unwrap(); + assert_eq!(streamed, expected, "{prefix}"); + assert!(!streamed.is_empty()); + } +} + +#[test] +fn oversized_filtered_split_member_returns_the_same_typed_limit_as_unsplit() { + let payload = b"\xe8\0\0\0\0filtered payload block\n".repeat(256); + let entry = rar50::ArchiveEntry::new( + b"filtered".to_vec(), + rars::EntrySource::from_bytes(payload.clone()), + ); + let mut sink = rar50::CollectedVolumes::new(); + let root = scratch::case("filtered-split-limit"); + rar50::write_streaming_volumes_to( + &[entry], + rar50::WriterOptions::new(rars::ArchiveVersion::Rar50, rars::FeatureSet::store_only()), + rar50::ArchiveExtras::default() + .with_filter_policy(rar50::FilterPolicy::explicit(rar50::FilterKind::E8)), + 32, + &mut sink, + &rars::WriterResources::new(128 * 1024 * 1024).with_temp_dir(root.to_path_buf()), + ) + .unwrap(); + let volumes: Vec<_> = sink + .take() + .into_iter() + .map(|bytes| Archive::parse_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + assert!(volumes[0].files().next().unwrap().is_split_after()); + assert_eq!( + collect(&volumes, payload.len() as u64, None).unwrap(), + vec![payload.clone()] + ); + let error = collect(&volumes, payload.len() as u64 - 1, None).unwrap_err(); + assert!( + matches!(cause(&error), Error::Rar50BufferedDecodeLimitExceeded { limit, required } + if *limit == payload.len() as u64 - 1 && *required == payload.len() as u64), + "{error:?}" + ); +} + +#[test] +fn split_streaming_integrity_failure_can_follow_partial_emission() { + let mut volumes = fixtures("multivol", 3, false, None); + let expected = collect(&volumes, u64::MAX, None).unwrap(); + for block in &mut volumes.last_mut().unwrap().blocks { + if let Block::File(file) = block { + file.hash = None; + file.data_crc32 = Some(0); + } + } + for limit in [1, u64::MAX] { + let emitted = Rc::new(RefCell::new(Vec::new())); + let error = rar50::extract_volumes_to(&volumes, options(limit, None), |_| { + Ok(Box::new(Capture(emitted.clone()))) + }) + .unwrap_err(); + assert!( + matches!(cause(&error), Error::Crc32Mismatch { .. }), + "{error:?}" + ); + if limit == 1 { + assert_eq!(*emitted.borrow(), expected[0]); + } else { + assert!(emitted.borrow().is_empty()); + } + } +} + +#[test] +fn split_streaming_preserves_sink_io_failure() { + struct Refuse; + impl Write for Refuse { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sink refused", + )) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let volumes = fixtures("multivol", 3, false, None); + let error = rar50::extract_volumes_to(&volumes, options(1, None), |_| Ok(Box::new(Refuse))) + .unwrap_err(); + assert!( + matches!(cause(&error), Error::Io(error) if error.kind == io::ErrorKind::PermissionDenied), + "{error:?}" + ); +} diff --git a/arca-rar/vendor/rars/tests/rarvm_regressions.rs b/arca-rar/vendor/rars/tests/rarvm_regressions.rs new file mode 100644 index 00000000..c4517f75 --- /dev/null +++ b/arca-rar/vendor/rars/tests/rarvm_regressions.rs @@ -0,0 +1,152 @@ +use rars::crc32::crc32; +use rars::rar15_40::Archive; +use rars::{ArchiveReadOptions, Result}; +use std::cell::RefCell; +use std::io::{Result as IoResult, Write}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar15_40/rarvm") + .join(name) +} + +struct CollectWriter { + data: Rc>>, +} + +struct CollectedEntry { + name: Vec, + data: Vec, +} + +impl Write for CollectWriter { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.data.borrow_mut().extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +fn collect_extract(archive: &Archive) -> Result> { + let entries = RefCell::new(Vec::new()); + archive.extract_to(ArchiveReadOptions::default(), |meta| { + let data = Rc::new(RefCell::new(Vec::new())); + entries.borrow_mut().push((meta.clone(), Rc::clone(&data))); + Ok(Box::new(CollectWriter { data })) + })?; + Ok(entries + .into_inner() + .into_iter() + .map(|(meta, data)| CollectedEntry { + name: meta.name, + data: data.borrow().clone(), + }) + .collect()) +} + +#[test] +fn solid_e8_filters_use_member_relative_offsets() { + let expected_lead = std::fs::read(fixture("solid_e8_filter_lead.txt")).unwrap(); + let expected_exe = std::fs::read(fixture("solid_e8_filter_payload.exe")).unwrap(); + let archive = Archive::parse_path(fixture("solid_e8_filter_member_offset.rar")).unwrap(); + let files: Vec<_> = archive.files().collect(); + + assert!(archive.main.is_solid()); + assert_eq!(files.len(), 2); + assert_eq!(files[0].name, b"lead.txt"); + assert_eq!(files[0].pack_size, 76); + assert_eq!(files[0].unp_size, 800); + assert_eq!(files[1].name, b"tiny_e8e9.exe"); + assert_eq!(files[1].pack_size, 295); + assert_eq!(files[1].unp_size, 5_884); + assert!(files[1].is_solid()); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 2); + assert_eq!(extracted[0].data, expected_lead); + assert_eq!(extracted[1].data, expected_exe); +} + +#[test] +fn vm_filter_control_stream_accepts_32_bit_encoded_integers() { + let archive = Archive::parse_path(fixture("vm_encoded_u32_filter.rar")).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"bsdcat.exe"); + + let mut extracted = Vec::new(); + archive + .extract_to(ArchiveReadOptions::default(), |meta| { + extracted.push(meta.name.clone()); + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + + assert_eq!(extracted, vec![b"bsdcat.exe".to_vec()]); +} + +#[test] +fn non_standard_vm_filter_uses_generic_executor() { + let archive = Archive::parse_path(fixture("generic_delta_padding_mutation.rar")).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"itanium_synthetic_bundles.bin"); + assert_eq!(files[0].unp_size, 1_048_576); + assert_eq!(files[0].file_crc, 0x3908_6451); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data.len(), 1_048_576); + assert_eq!(crc32(&extracted[0].data), 0x3908_6451); +} + +#[test] +fn ppmd_embedded_vm_filter_is_applied() { + // Generated with RAR 3.00 using `-m5 -mc10:16t+ -mce+`: text/PPMd is + // forced for the member while the x86 executable filter is also forced, + // so the filter record is carried by PPMd escape 3 instead of LZ symbols. + let archive = Archive::parse_path(fixture("ppmd_embedded_vm_filter.rar")).unwrap(); + let files: Vec<_> = archive.files().collect(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].name, b"ppmd_branch_mix.bin"); + assert_eq!(files[0].pack_size, 3_351); + assert_eq!(files[0].unp_size, 710_400); + assert_eq!(files[0].file_crc, 0xa0fa_ad59); + + let extracted = collect_extract(&archive).unwrap(); + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].data.len(), 710_400); + assert_eq!(crc32(&extracted[0].data), 0xa0fa_ad59); +} + +#[test] +fn real_executable_filter_archive_decodes() { + let archive = Archive::parse_path(fixture("filter_bsdcat_exe.rar")).unwrap(); + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"bsdcat.exe"); + assert_eq!(crc32(&extracted[0].data), 0x4db1_0349); +} + +#[test] +fn vm_delta_filter_accepts_more_than_thirty_two_channels() { + let mut expected = Vec::with_capacity(400 * 64); + for row in 0..400u32 { + for channel in 0..64u32 { + expected.push((channel * 7 + row * 3) as u8); + } + } + let archive = Archive::parse_path(fixture("delta_64_channels.rar")).unwrap(); + + let extracted = collect_extract(&archive).unwrap(); + + assert_eq!(extracted.len(), 1); + assert_eq!(extracted[0].name, b"delta64.bin"); + assert_eq!(extracted[0].data, expected); +} diff --git a/arca-rar/vendor/rars/tests/reader_32bit_lengths.rs b/arca-rar/vendor/rars/tests/reader_32bit_lengths.rs new file mode 100644 index 00000000..1c2bf9a9 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_32bit_lengths.rs @@ -0,0 +1,91 @@ +use rars::{ArchiveReadOptions, ArchiveReader, Error}; +use std::io; + +fn vint(mut value: u64, out: &mut Vec) { + while value >= 128 { + out.push((value as u8 & 0x7f) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +fn block(body: &[u8]) -> Vec { + let mut encoded = Vec::new(); + vint(body.len() as u64, &mut encoded); + encoded.extend_from_slice(body); + let mut block = rars::crc32::crc32(&encoded).to_le_bytes().to_vec(); + block.extend_from_slice(&encoded); + block +} + +fn advertised_member(unpacked_size: u64, compression_info: u64, split_flags: u8) -> rars::Archive { + let mut bytes = b"Rar!\x1a\x07\x01\x00".to_vec(); + bytes.extend_from_slice(&block(&[1, 0, u8::from(split_flags != 0)])); + + // FILE block: type, header flags, file flags, unpacked size, attributes, + // compression info, host OS, and the one-byte name. No payload is needed + // because the advertised lengths must be rejected before decoding. + let mut file = vec![2, split_flags, 0]; + vint(unpacked_size, &mut file); + file.push(0); + vint(compression_info, &mut file); + file.extend_from_slice(&[0, 1, b'x']); + bytes.extend_from_slice(&block(&file)); + ArchiveReader::read_owned(bytes).expect("the size declaration must parse") +} + +fn assert_invalid_header(error: Error, expected: &'static str) { + assert!( + matches!(error.root_cause(), Error::InvalidHeader(message) if *message == expected), + "expected {expected:?}, got {error:?}" + ); +} + +#[test] +fn compressed_declarations_exceeding_32_bit_usize_fail_before_decode() { + // v0 method 1 with dictionary power 15 advertises exactly 4 GiB. Neither + // case has a payload large enough to satisfy its declared requirements. + for (unpacked_size, compression_info, expected) in [ + ( + 1u64 << 32, + 0x80, + "RAR 5 unpacked size overflows host address size", + ), + ( + 1, + 0x3c80, + "RAR 5 dictionary size overflows host address size", + ), + ] { + let archive = advertised_member(unpacked_size, compression_info, 0); + let file = archive.members().next().expect("one file"); + assert_eq!(file.meta.unpacked_size, unpacked_size); + if usize::BITS >= 64 { + continue; + } + + let options = ArchiveReadOptions::new().with_rar50_buffered_decode_limit(u64::MAX); + assert_invalid_header( + archive + .extract_to_with_options(options, |_| Ok(Box::new(io::sink()))) + .unwrap_err(), + expected, + ); + assert_invalid_header( + archive + .extract_to_parallel_buffered_with_options(options, |_| Ok(Box::new(io::sink()))) + .unwrap_err(), + expected, + ); + + let volumes = [ + advertised_member(unpacked_size, compression_info, 0x10), + advertised_member(unpacked_size, compression_info, 0x08), + ]; + assert_invalid_header( + rars::extract_volumes_to_with_options(&volumes, options, |_| Ok(Box::new(io::sink()))) + .unwrap_err(), + expected, + ); + } +} diff --git a/arca-rar/vendor/rars/tests/reader_cancellation.rs b/arca-rar/vendor/rars/tests/reader_cancellation.rs new file mode 100644 index 00000000..bbb701df --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_cancellation.rs @@ -0,0 +1,364 @@ +#![cfg(feature = "write")] + +use rars::{ + ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, Error, ErrorKind, ReadCancellation, +}; +use std::{ + cell::RefCell, + io::{self, Write}, + rc::Rc, +}; + +#[path = "support/scratch.rs"] +mod scratch; + +const FORMATS: [ArchiveVersion; 8] = [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, +]; + +#[test] +fn cancellation_from_the_last_directory_callback_is_not_success() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let mut builder = Builder::new(format); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for parallel in [false, true] { + let token = ReadCancellation::new(); + let open = |_: &rars::ExtractedEntryMeta| { + token.cancel(); + Ok(Box::new(io::sink()) as Box) + }; + let options = ArchiveReadOptions::new().with_cancellation(&token); + let result = if parallel { + archive.extract_to_parallel_buffered_with_options(options, open) + } else { + archive.extract_to_with_options(options, open) + }; + assert_eq!(result.unwrap_err().kind(), ErrorKind::Cancelled); + } + } +} + +#[test] +fn compressed_encrypted_splits_keep_cancellation_ahead_of_checksum_diagnostics() { + let mut state = 0x1234_5678u32; + let noise: Vec = (0..4096) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let data = noise.repeat(3); + for format in [ArchiveVersion::Rar29, ArchiveVersion::Rar50] { + let mut builder = Builder::new(format) + .password(Some(b"secret".to_vec())) + .volume_size(Some(512)); + builder + .add_bytes(b"split".to_vec(), data.clone(), None, None) + .unwrap(); + let mut volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for cancel in [false, true] { + for buffered in [u64::MAX, 0] { + // Buffered extraction verifies before publication. Corrupt only + // the streaming case, where the sink cancellation precedes CRC. + if cancel && buffered == 0 { + match volumes.last_mut().unwrap() { + rars::Archive::Rar15To40(a) => { + for block in &mut a.blocks { + if let rars::rar15_40::Block::File(f) = block { + assert!(!f.is_stored()); + f.file_crc ^= 1; + } + } + } + rars::Archive::Rar50Plus(a) => { + for block in &mut a.blocks { + if let rars::rar50::Block::File(f) = block { + assert!(!f.is_stored()); + f.data_crc32 = Some(0); + f.hash = None; + } + } + } + _ => unreachable!(), + } + } + let token = ReadCancellation::new(); + let bytes = Rc::new(RefCell::new(Vec::new())); + let result = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::with_password(b"secret") + .with_cancellation(&token) + .with_rar50_buffered_decode_limit(buffered), + |_| { + Ok(Box::new(Sink { + bytes: bytes.clone(), + cancel: cancel.then(|| token.clone()), + fail: false, + })) + }, + ); + if cancel { + assert_eq!(result.unwrap_err().kind(), ErrorKind::Cancelled); + assert!((1..=17).contains(&bytes.borrow().len())); + } else { + result.unwrap(); + assert_eq!(*bytes.borrow(), data); + } + } + } + } +} + +struct Sink { + bytes: Rc>>, + cancel: Option, + fail: bool, +} +impl Write for Sink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if let Some(token) = &self.cancel { + token.cancel(); + } + if self.fail { + return Err(io::ErrorKind::PermissionDenied.into()); + } + // A short write must return to a cancellation checkpoint, even in + // buffered parallel publication and after legacy codec I/O adapters. + let n = bytes.len().min(17); + self.bytes.borrow_mut().extend_from_slice(&bytes[..n]); + Ok(n) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn precancelled_parsing_precedes_signature_checks_and_path_opens() { + let token = ReadCancellation::new(); + token.clone().cancel(); + let options = ArchiveReadOptions::new().with_cancellation(&token); + let path = scratch::case("cancelled-reader").join("missing.rar"); + let results = [ + ArchiveReader::read_with_options(b"", options).map(|_| ()), + ArchiveReader::read_owned_with_options(vec![], options).map(|_| ()), + ArchiveReader::read_path_with_options(&path, options).map(|_| ()), + rars::rar13::Archive::parse_with_options(b"", options).map(|_| ()), + rars::rar13::Archive::parse_owned_with_options(vec![], options).map(|_| ()), + rars::rar13::Archive::parse_path_with_options(&path, options).map(|_| ()), + rars::rar15_40::Archive::parse_with_options(b"", options).map(|_| ()), + rars::rar15_40::Archive::parse_owned_with_options(vec![], options).map(|_| ()), + rars::rar15_40::Archive::parse_path_with_options(&path, options).map(|_| ()), + rars::rar50::Archive::parse_with_options(b"", options).map(|_| ()), + rars::rar50::Archive::parse_owned_with_options(vec![], options).map(|_| ()), + rars::rar50::Archive::parse_path_with_options(&path, options).map(|_| ()), + ]; + for result in results { + assert_eq!(result.unwrap_err().kind(), ErrorKind::Cancelled); + } +} + +#[test] +fn extraction_observes_callbacks_without_retaining_parse_policy() { + for format in FORMATS { + for stored in [true, false] { + let mut builder = Builder::new(format).store(stored); + for name in [b"first".as_slice(), b"second"] { + builder + .add_bytes(name.to_vec(), vec![42; 8192], None, None) + .unwrap(); + } + let parse_token = ReadCancellation::new(); + let archive = ArchiveReader::read_owned_with_options( + builder.to_bytes().unwrap(), + ArchiveReadOptions::new().with_cancellation(&parse_token), + ) + .unwrap(); + parse_token.cancel(); + for parallel in [false, true] { + for buffered in [0, u64::MAX] { + // None, active token, pre-cancelled, open callback, short + // output callback, real open failure, real sink failure. + for mode in 0..7 { + let token = ReadCancellation::new(); + if mode == 2 { + token.cancel(); + } + let mut options = + ArchiveReadOptions::new().with_rar50_buffered_decode_limit(buffered); + if mode != 0 { + options = options.with_cancellation(&token); + } + let bytes = Rc::new(RefCell::new(Vec::new())); + let mut opened = 0; + let open = |_: &rars::ExtractedEntryMeta| { + opened += 1; + if mode == 3 || mode == 5 { + token.cancel(); + } + if mode == 5 { + return Err(Error::from(io::Error::from( + io::ErrorKind::PermissionDenied, + ))); + } + Ok(Box::new(Sink { + bytes: bytes.clone(), + cancel: (mode == 4 || mode == 6).then(|| token.clone()), + fail: mode == 6, + }) as Box) + }; + let result = if parallel { + archive.extract_to_parallel_buffered_with_options(options, open) + } else { + archive.extract_to_with_options(options, open) + }; + match mode { + 0 | 1 => { + result.unwrap(); + assert_eq!(*bytes.borrow(), vec![42; 16384]); + assert_eq!(opened, 2); + } + 2..=4 => { + assert_eq!( + result.unwrap_err().kind(), + ErrorKind::Cancelled, + "{format:?}, mode {mode}" + ); + assert_eq!(opened, usize::from(mode != 2)); + if mode == 4 { + assert!((1..=17).contains(&bytes.borrow().len())); + } else { + assert!(bytes.borrow().is_empty()); + } + } + _ => { + let err = result.unwrap_err(); + if mode == 5 { + assert!( + matches!(err.root_cause(), Error::Io(e) if e.kind == io::ErrorKind::PermissionDenied) + ); + } else { + // Older codecs already map sink errors to codec errors. + // Cancellation must not change that existing failure. + let open = |_: &rars::ExtractedEntryMeta| { + Ok(Box::new(Sink { + bytes: Rc::default(), + cancel: None, + fail: true, + }) + as Box) + }; + let options = ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(buffered); + let baseline = if parallel { + archive.extract_to_parallel_buffered_with_options( + options, open, + ) + } else { + archive.extract_to_with_options(options, open) + } + .unwrap_err(); + assert_eq!( + format!("{:?}", err.root_cause()), + format!("{:?}", baseline.root_cause()) + ); + assert_ne!(err.kind(), ErrorKind::Cancelled); + } + assert_eq!(opened, 1); + assert!(bytes.borrow().is_empty()); + } + } + } + } + } + } + } +} + +#[test] +fn cancellation_stops_solid_history_and_encrypted_split_extraction() { + for format in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut b = Builder::new(format).solid(true); + for name in [b"discarded".as_slice(), b"wanted"] { + b.add_bytes(name.to_vec(), vec![42; 8192], None, None) + .unwrap(); + } + let a = ArchiveReader::read_owned(b.to_bytes().unwrap()).unwrap(); + let token = ReadCancellation::new(); + let mut opened = 0; + let err = a + .extract_to_with_options(ArchiveReadOptions::new().with_cancellation(&token), |_| { + opened += 1; + Ok(Box::new(Sink { + bytes: Rc::default(), + cancel: Some(token.clone()), + fail: false, + })) + }) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::Cancelled); + assert_eq!(opened, 1); + } + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut b = Builder::new(format) + .store(true) + .volume_size(Some(512)) + .password((format != ArchiveVersion::Rar14).then(|| b"secret".to_vec())); + b.add_bytes(b"split".to_vec(), vec![42; 2048], None, None) + .unwrap(); + let volumes: Vec<_> = b + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for cancel in [false, true] { + let token = ReadCancellation::new(); + let bytes = Rc::new(RefCell::new(Vec::new())); + let result = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::with_password(b"secret").with_cancellation(&token), + |_| { + Ok(Box::new(Sink { + bytes: bytes.clone(), + cancel: cancel.then(|| token.clone()), + fail: false, + })) + }, + ); + if cancel { + assert_eq!(result.unwrap_err().kind(), ErrorKind::Cancelled); + assert!((1..=17).contains(&bytes.borrow().len())); + } else { + result.unwrap(); + assert_eq!(*bytes.borrow(), vec![42; 2048]); + } + } + } +} diff --git a/arca-rar/vendor/rars/tests/reader_dictionary_limit.rs b/arca-rar/vendor/rars/tests/reader_dictionary_limit.rs new file mode 100644 index 00000000..4e48b0f1 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_dictionary_limit.rs @@ -0,0 +1,136 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, ErrorKind}; + +#[test] +fn dictionary_limit_covers_streaming_and_encrypted_split_members() { + let mut state = 0x1234_5678u32; + let noise: Vec = (0..4096) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let mut builder = Builder::new(format).password(Some(b"secret".to_vec())); + builder + .add_bytes(b"file".to_vec(), noise.repeat(3), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let required = match &archive { + Archive::Rar50Plus(a) => { + let f = a.files().next().unwrap(); + assert!(!f.is_stored()); + f.decoded_compression_info().unwrap().dictionary_size + } + _ => unreachable!(), + }; + let volumes: Vec<_> = builder + .volume_size(Some(512)) + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for buffered in [0, u64::MAX] { + for split in [false, true] { + for reject in [false, true] { + // A missing password on refusal proves admission precedes decryption. + let options = ArchiveReadOptions::with_optional_password(if reject { + None + } else { + Some(b"secret".as_slice()) + }) + .with_rar50_buffered_decode_limit(buffered) + .with_rar50_dictionary_size_limit(if reject { 0 } else { required }); + let mut opened = 0; + let open = |_: &rars::ExtractedEntryMeta| { + opened += 1; + Ok(Box::new(std::io::sink()) as Box) + }; + let result = if split { + rars::extract_volumes_to_with_options(&volumes, options, open) + } else { + archive.extract_to_with_options(options, open) + }; + if reject { + let error = result.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!(error.entry_context().unwrap().0, b"file"); + assert_eq!(opened, 0); + } else { + result.unwrap(); + assert_eq!(opened, 1); + } + } + } + } + } +} + +#[test] +fn zero_dictionary_limit_allows_stored_members_and_directories() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"dir/file".to_vec(), vec![42; 32], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let mut opened = 0; + archive + .extract_to_with_options( + ArchiveReadOptions::new().with_rar50_dictionary_size_limit(0), + |_| { + opened += 1; + Ok(Box::new(std::io::sink())) + }, + ) + .unwrap(); + assert_eq!(opened, 2); +} + +#[test] +fn invalid_compression_info_keeps_member_context_during_dictionary_admission() { + let mut builder = Builder::new(ArchiveVersion::Rar50).compression_level(Some(1)); + builder + .add_bytes(b"file".to_vec(), b"data".repeat(32), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for raw in [2 | 128, (1 << 15) | 128, (16 << 10) | 128] { + let mut invalid = archive.clone(); + let Archive::Rar50Plus(native) = &mut invalid else { + unreachable!() + }; + for block in &mut native.blocks { + if let rars::rar50::Block::File(file) = block { + file.compression_info = raw; + } + } + let mut opened = false; + let error = invalid + .extract_to_with_options( + ArchiveReadOptions::new().with_rar50_dictionary_size_limit(u64::MAX), + |_| { + opened = true; + Ok(Box::new(std::io::sink())) + }, + ) + .unwrap_err(); + assert!(!opened); + assert_eq!( + error.entry_context(), + Some((b"file".as_slice(), "checking dictionary limit")) + ); + assert_eq!( + error.kind(), + if raw & 0x3f == 2 { + ErrorKind::UnsupportedFeature + } else { + ErrorKind::InvalidArchive + } + ); + } +} diff --git a/arca-rar/vendor/rars/tests/reader_failure_output.rs b/arca-rar/vendor/rars/tests/reader_failure_output.rs new file mode 100644 index 00000000..5b1c84fc --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_failure_output.rs @@ -0,0 +1,140 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, Error}; +use std::{cell::RefCell, io, io::Write, rc::Rc}; + +const VERSIONS: [ArchiveVersion; 7] = [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, +]; + +struct Capture { + bytes: Rc>>, + remaining: usize, +} + +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.remaining == 0 { + return Err(io::ErrorKind::PermissionDenied.into()); + } + let count = bytes.len().min(self.remaining); + self.bytes.borrow_mut().extend_from_slice(&bytes[..count]); + self.remaining -= count; + Ok(count) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn archive(version: ArchiveVersion, stored: bool) -> Archive { + let mut builder = Builder::new(version).store(stored); + for (name, data) in [ + (b"first".as_slice(), b"earlier output".to_vec()), + (b"failing", vec![42; 4096]), + (b"later", b"must not be opened".to_vec()), + ] { + builder.add_bytes(name.to_vec(), data, None, None).unwrap(); + } + ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() +} + +fn corrupt_second_checksum(archive: &mut Archive) { + match archive { + Archive::Rar13(a) => a.entries[1].header.file_crc ^= 1, + Archive::Rar15To40(a) => { + for block in &mut a.blocks { + if let rars::rar15_40::Block::File(file) = block { + if file.name == b"failing" { + file.file_crc ^= 1; + } + } + } + } + Archive::Rar50Plus(a) => { + for block in &mut a.blocks { + if let rars::rar50::Block::File(file) = block { + if file.name == b"failing" { + file.hash = None; + file.data_crc32 = Some(file.data_crc32.unwrap() ^ 1); + } + } + } + } + _ => unreachable!(), + } +} + +#[test] +fn integrity_failure_can_leave_the_entire_unverified_member_written() { + for version in VERSIONS { + for stored in [true, false] { + let mut archive = archive(version, stored); + corrupt_second_checksum(&mut archive); + let bytes = Rc::new(RefCell::new(Vec::new())); + let mut opened = Vec::new(); + let error = archive + .extract_to_with_options( + ArchiveReadOptions::new().with_rar50_buffered_decode_limit(0), + |meta| { + opened.push(meta.name.clone()); + Ok(Box::new(Capture { + bytes: bytes.clone(), + remaining: usize::MAX, + })) + }, + ) + .unwrap_err(); + assert_eq!(error.entry_context().unwrap().0, b"failing"); + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + assert_eq!(opened, [b"first".to_vec(), b"failing".to_vec()]); + let mut expected = b"earlier output".to_vec(); + expected.extend_from_slice(&[42; 4096]); + assert_eq!(*bytes.borrow(), expected, "{version:?}, stored={stored}"); + } + } +} + +#[test] +fn sink_failure_keeps_the_accepted_prefix_and_stops_later_members() { + for version in VERSIONS { + for stored in [true, false] { + let archive = archive(version, stored); + let bytes = Rc::new(RefCell::new(Vec::new())); + let mut opened = Vec::new(); + let error = archive + .extract_to_with_options( + ArchiveReadOptions::new().with_rar50_buffered_decode_limit(0), + |meta| { + opened.push(meta.name.clone()); + Ok(Box::new(Capture { + bytes: bytes.clone(), + remaining: if meta.name == b"failing" { + 17 + } else { + usize::MAX + }, + })) + }, + ) + .unwrap_err(); + assert!( + matches!(error.root_cause(), Error::Io(e) + if e.kind == io::ErrorKind::PermissionDenied), + "{version:?}, stored={stored}: {error:?}" + ); + assert_eq!(error.entry_context().unwrap().0, b"failing"); + assert_eq!(opened, [b"first".to_vec(), b"failing".to_vec()]); + let mut expected = b"earlier output".to_vec(); + expected.extend_from_slice(&[42; 17]); + assert_eq!(*bytes.borrow(), expected, "{version:?}, stored={stored}"); + } + } +} diff --git a/arca-rar/vendor/rars/tests/reader_failure_paths.rs b/arca-rar/vendor/rars/tests/reader_failure_paths.rs new file mode 100644 index 00000000..9e9069b9 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_failure_paths.rs @@ -0,0 +1,506 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, ErrorKind}; +use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct Fault { + operations: usize, + fail_at: Option, +} +impl Fault { + fn check(&mut self) -> io::Result<()> { + self.operations += 1; + if self.fail_at == Some(self.operations) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "injected source failure", + )); + } + Ok(()) + } + fn arm(&mut self, fail_at: Option) { + self.operations = 0; + self.fail_at = fail_at; + } +} +struct Source { + bytes: Cursor>, + fault: Arc>, +} +impl Read for Source { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + self.fault.lock().unwrap().check()?; + let count = bytes.len().min(17); + self.bytes.read(&mut bytes[..count]) + } +} +impl Seek for Source { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.fault.lock().unwrap().check()?; + self.bytes.seek(from) + } +} +fn images() -> Vec<(ArchiveVersion, bool, Vec)> { + let mut cases = Vec::new(); + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar30, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for stored in [false, true] { + for encrypted in [false, true] { + if encrypted && matches!(version, ArchiveVersion::Rar13 | ArchiveVersion::Rar14) { + continue; + } + let mut builder = Builder::new(version) + .compression_level(Some(if stored { 0 } else { 1 })) + .password(encrypted.then(|| b"secret".to_vec())); + builder + .add_bytes( + b"payload".to_vec(), + b"repeatable member payload\n".repeat(16), + None, + None, + ) + .unwrap(); + cases.push((version, encrypted, builder.to_bytes().unwrap())); + } + } + } + cases +} +fn parse( + bytes: &[u8], + fault: &Arc>, + options: ArchiveReadOptions<'_>, +) -> rars::Result { + ArchiveReader::read_reader_with_options( + Source { + bytes: Cursor::new(bytes.to_vec()), + fault: fault.clone(), + }, + options, + ) +} + +#[test] +fn every_seekable_parser_source_failure_is_reported_without_a_partial_archive() { + for (version, encrypted, bytes) in images() { + let options = + ArchiveReadOptions::with_optional_password(encrypted.then_some(b"secret".as_slice())); + let fault = Arc::new(Mutex::new(Fault::default())); + parse(&bytes, &fault, options).unwrap(); + let operations = fault.lock().unwrap().operations; + assert!(operations > 0); + for fail_at in 1..=operations { + fault.lock().unwrap().arm(Some(fail_at)); + let error = parse(&bytes, &fault, options).unwrap_err(); + assert_eq!( + error.kind(), + ErrorKind::Io, + "{version:?}, encrypted {encrypted}, operation {fail_at}: {error}" + ); + assert!(error.to_string().contains("injected source failure")); + } + } +} + +struct Capture(Arc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn source_failures_abort_member_publication_and_the_archive_can_be_retried() { + let expected = b"repeatable member payload\n".repeat(16); + for (version, encrypted, bytes) in images() { + for streaming in [false, true] { + let mut options = ArchiveReadOptions::with_optional_password( + encrypted.then_some(b"secret".as_slice()), + ); + if streaming { + options = options.with_rar50_buffered_decode_limit(0); + } + let fault = Arc::new(Mutex::new(Fault::default())); + let archive = parse(&bytes, &fault, options).unwrap(); + let output = Arc::new(Mutex::new(Vec::new())); + fault.lock().unwrap().arm(None); + archive + .extract_to_with_options(options, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap(); + assert_eq!(*output.lock().unwrap(), expected); + let operations = fault.lock().unwrap().operations; + for fail_at in 1..=operations { + output.lock().unwrap().clear(); + fault.lock().unwrap().arm(Some(fail_at)); + let error = archive + .extract_to_with_options(options, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap_err(); + assert!( + matches!(error.kind(), ErrorKind::Io), + "{version:?}, streaming {streaming}, operation {fail_at}: {error}" + ); + assert!(error.to_string().contains("injected source failure")); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()), + "{version:?}, encrypted {encrypted}, streaming {streaming}, operation {fail_at}: {error}" + ); + let output = output.lock().unwrap(); + assert!( + expected.starts_with(&output), + "failure must leave only a correct prefix" + ); + } + output.lock().unwrap().clear(); + fault.lock().unwrap().arm(None); + archive + .extract_to_with_options(options, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap(); + assert_eq!(*output.lock().unwrap(), expected); + } + } +} + +struct FailingSink; +impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "injected sink failure", + )) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn sink_failures_preserve_io_identity_and_member_context() { + for (version, encrypted, bytes) in images() { + for streaming in [false, true] { + let mut options = ArchiveReadOptions::with_optional_password( + encrypted.then_some(b"secret".as_slice()), + ); + if streaming { + options = options.with_rar50_buffered_decode_limit(0); + } + let fault = Arc::new(Mutex::new(Fault::default())); + let archive = parse(&bytes, &fault, options).unwrap(); + let error = archive + .extract_to_with_options(options, |_| Ok(Box::new(FailingSink))) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{version:?}: {error}"); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + assert!(error.to_string().contains("injected sink failure")); + let rars::Error::Io(source) = error.root_cause() else { + panic!("{error}") + }; + let original = source.source().downcast_ref::().unwrap(); + assert_eq!(original.kind(), io::ErrorKind::PermissionDenied); + } + } +} + +#[test] +fn split_and_parallel_sink_failures_keep_member_context() { + for version in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar30, + ArchiveVersion::Rar50, + ] { + for stored in [false, true] { + let mut builder = Builder::new(version) + .compression_level(Some(if stored { 0 } else { 1 })) + .volume_size(Some(if stored { 512 } else { 8 })); + // Store mode uses noise; compressed mode uses repeated bytes and + // very small fragments so the writer cannot choose a stored fallback. + let mut state = 0x12345678u32; + let payload: Vec = (0..16384) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let payload = if stored { + payload + } else { + b"compressible member payload".repeat(128) + }; + builder + .add_bytes(b"payload".to_vec(), payload, None, None) + .unwrap(); + let mut volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|b| ArchiveReader::read_owned(b).unwrap()) + .collect(); + assert!(volumes.len() > 1, "{version:?}, stored={stored}"); + match &volumes[0] { + Archive::Rar15To40(a) => assert_eq!(a.files().next().unwrap().is_stored(), stored), + Archive::Rar50Plus(a) => assert_eq!(a.files().next().unwrap().is_stored(), stored), + _ => unreachable!(), + } + let error = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::default(), + |_| Ok(Box::new(FailingSink)), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{version:?}: {error}"); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + assert!(error.to_string().contains("injected sink failure")); + // A separately corrupted final integrity record must still name + // the logical split member after all fragments have been decoded. + if let Archive::Rar15To40(last) = volumes.last_mut().unwrap() { + for block in &mut last.blocks { + if let rars::rar15_40::Block::File(file) = block { + file.file_crc ^= 1; + } + } + let error = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::default(), + |_| Ok(Box::new(io::sink())), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ChecksumMismatch); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + } + } + } + for (_, encrypted, bytes) in images() { + let options = + ArchiveReadOptions::with_optional_password(encrypted.then_some(b"secret".as_slice())); + let archive = ArchiveReader::read_owned_with_options(bytes, options).unwrap(); + let error = archive + .extract_to_parallel_buffered_with_options(options, |_| Ok(Box::new(FailingSink))) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + } +} + +#[test] +fn encrypted_header_source_failures_remain_io_errors() { + for version in [ArchiveVersion::Rar30, ArchiveVersion::Rar50] { + let mut builder = Builder::new(version) + .password(Some(b"secret".to_vec())) + .header_encryption(true); + builder + .add_bytes(b"payload".to_vec(), b"data".to_vec(), None, None) + .unwrap(); + let bytes = builder.to_bytes().unwrap(); + let options = ArchiveReadOptions::with_password(b"secret"); + let fault = Arc::new(Mutex::new(Fault::default())); + parse(&bytes, &fault, options).unwrap(); + let operations = fault.lock().unwrap().operations; + for fail_at in 1..=operations { + fault.lock().unwrap().arm(Some(fail_at)); + let error = parse(&bytes, &fault, options).unwrap_err(); + assert_eq!( + error.kind(), + ErrorKind::Io, + "{version:?}, operation {fail_at}: {error}" + ); + assert!(error.to_string().contains("injected source failure")); + } + } +} + +#[test] +fn parallel_stored_source_failure_retains_member_context() { + for version in [ArchiveVersion::Rar15, ArchiveVersion::Rar50] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"payload".to_vec(), b"data".to_vec(), None, None) + .unwrap(); + let bytes = builder.to_bytes().unwrap(); + let fault = Arc::new(Mutex::new(Fault::default())); + let options = ArchiveReadOptions::default(); + let archive = parse(&bytes, &fault, options).unwrap(); + fault.lock().unwrap().arm(Some(1)); + let error = archive + .extract_to_parallel_buffered_with_options(options, |_| Ok(Box::new(io::sink()))) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + } +} + +#[test] +fn removed_file_sources_report_io_with_member_context() { + let directory = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/reader-failure-tests"); + std::fs::create_dir_all(&directory).unwrap(); + for (version, encrypted, bytes) in images() + .into_iter() + .filter(|(version, _, _)| matches!(version, ArchiveVersion::Rar30 | ArchiveVersion::Rar50)) + { + for streaming in [false, true] { + let path = directory.join(format!( + "removed-{}-{version:?}-{encrypted}-{streaming}.rar", + std::process::id() + )); + std::fs::write(&path, &bytes).unwrap(); + let mut options = ArchiveReadOptions::with_optional_password( + encrypted.then_some(b"secret".as_slice()), + ); + if streaming { + options = options.with_rar50_buffered_decode_limit(0); + } + let archive = ArchiveReader::read_path_with_options(&path, options).unwrap(); + std::fs::remove_file(&path).unwrap(); + let error = archive + .extract_to_with_options(options, |_| Ok(Box::new(io::sink()))) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{version:?}: {error}"); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + let rars::Error::Io(source) = error.root_cause() else { + panic!("{error}") + }; + assert_eq!( + source.source().downcast_ref::().unwrap().kind(), + io::ErrorKind::NotFound + ); + } + } +} + +#[test] +fn compressed_worker_and_regular_volume_source_failures_keep_context() { + for version in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar30, + ] { + let mut builder = Builder::new(version).compression_level(Some(1)); + builder + .add_bytes( + b"payload".to_vec(), + b"compressible member".repeat(256), + None, + None, + ) + .unwrap(); + let bytes = builder.to_bytes().unwrap(); + let options = ArchiveReadOptions::default(); + let fault = Arc::new(Mutex::new(Fault::default())); + let archive = parse(&bytes, &fault, options).unwrap(); + let Archive::Rar15To40(native) = &archive else { + unreachable!() + }; + assert!(!native.files().next().unwrap().is_stored()); + for parallel in [false, true] { + fault.lock().unwrap().arm(Some(1)); + let error = if parallel { + archive.extract_to_parallel_buffered_with_options(options, |_| { + Ok(Box::new(io::sink())) + }) + } else { + rars::extract_volumes_to_with_options( + std::slice::from_ref(&archive), + options, + |_| Ok(Box::new(io::sink())), + ) + } + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(b"payload".as_slice()) + ); + } + } +} +#[test] +fn invalid_stored_hash_records_keep_decode_context() { + let archive = ArchiveReader::read_owned( + include_bytes!("fixtures/rar50/crc32_wrong_beside_blake2sp.rar").to_vec(), + ) + .unwrap(); + let Archive::Rar50Plus(native) = &archive else { + unreachable!() + }; + let name = native.files().next().unwrap().name.clone(); + let mut invalid = archive.clone(); + let Archive::Rar50Plus(native) = &mut invalid else { + unreachable!() + }; + for block in &mut native.blocks { + if let rars::rar50::Block::File(file) = block { + let hash = file.hash.as_mut().unwrap(); + assert_eq!(hash.hash_type, 0); + hash.data.pop(); + } + } + let error = invalid + .extract_to_with_options(ArchiveReadOptions::default(), |_| Ok(Box::new(io::sink()))) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::InvalidArchive); + assert_eq!(error.entry_context(), Some((name.as_slice(), "decoding"))); +} + +#[test] +fn historical_aes_compressed_member_keeps_source_open_errors() { + let directory = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/reader-failure-tests"); + std::fs::create_dir_all(&directory).unwrap(); + let path = directory.join(format!("winrar-aes-{}.rar", std::process::id())); + std::fs::write( + &path, + include_bytes!("fixtures/rar15_40/encrypted/per_file_rar300_password.rar"), + ) + .unwrap(); + let archive = ArchiveReader::read_path(&path).unwrap(); + let Archive::Rar15To40(native) = &archive else { + unreachable!() + }; + let file = native.files().next().unwrap(); + assert_eq!(file.unp_ver, 29); + assert!(file.is_encrypted() && !file.is_stored()); + let name = file.name.clone(); + std::fs::remove_file(&path).unwrap(); + let error = archive + .extract_to_with_options(ArchiveReadOptions::with_password(b"password"), |_| { + Ok(Box::new(io::sink())) + }) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io); + assert_eq!( + error.entry_context().map(|(name, _)| name), + Some(name.as_slice()) + ); +} diff --git a/arca-rar/vendor/rars/tests/reader_header_limits.rs b/arca-rar/vendor/rars/tests/reader_header_limits.rs new file mode 100644 index 00000000..be28e9dd --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_header_limits.rs @@ -0,0 +1,366 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, Error, ErrorKind}; +#[path = "support/scratch.rs"] +mod scratch; + +fn headers(a: &Archive, raw: &[u8]) -> Vec { + match a { + Archive::Rar13(a) => std::iter::once(a.main.head_size as u64) + .chain(a.entries.iter().map(|e| e.header.head_size as u64)) + .collect(), + Archive::Rar15To40(a) => { + let mut sizes = vec![a.main.head_size as u64]; + for b in &a.blocks { + let h = match b { + rars::rar15_40::Block::File(f) => &f.block, + rars::rar15_40::Block::NewSub(s) => &s.file.block, + rars::rar15_40::Block::Comment(c) => &c.block, + rars::rar15_40::Block::Protect(p) => &p.block, + rars::rar15_40::Block::End(h) | rars::rar15_40::Block::Unknown(h) => h, + _ => unreachable!(), + }; + // A nested main comment is represented separately in blocks. + if h.offset >= 7 + a.main.head_size as usize { + sizes.push(h.head_size as u64); + } + } + sizes + } + Archive::Rar50Plus(a) => { + let size = |h: &rars::rar50::BlockHeader| { + 4 + u64::from((64 - h.header_size.leading_zeros()).div_ceil(7).max(1)) + + h.header_size + }; + let mut sizes = Vec::new(); + if a.main.encrypted_headers { + // Generated archive-encryption headers have a one-byte size. + let declared = raw[a.sfx_offset + 12]; + assert!(declared < 128); + sizes.push(5 + u64::from(declared)); + } + sizes.push(size(&a.main.block)); + sizes.extend(a.blocks.iter().map(|b| { + size(match b { + rars::rar50::Block::File(f) | rars::rar50::Block::Service(f) => &f.block, + rars::rar50::Block::End(e) => &e.block, + rars::rar50::Block::Unknown(h) => h, + _ => unreachable!(), + }) + })); + sizes + } + _ => unreachable!(), + } +} + +fn typed_read(raw: &[u8], options: ArchiveReadOptions<'_>, owned: bool) -> rars::Result { + match rars::find_archive_start(raw, rars::SFX_SCAN_LIMIT) + .unwrap() + .family + { + rars::ArchiveFamily::Rar13 => { + if owned { + rars::rar13::Archive::parse_owned_with_options(raw.to_vec(), options) + .map(Archive::Rar13) + } else { + rars::rar13::Archive::parse_with_options(raw, options).map(Archive::Rar13) + } + } + rars::ArchiveFamily::Rar15To40 => { + if owned { + rars::rar15_40::Archive::parse_owned_with_options(raw.to_vec(), options) + .map(Archive::Rar15To40) + } else { + rars::rar15_40::Archive::parse_with_options(raw, options).map(Archive::Rar15To40) + } + } + rars::ArchiveFamily::Rar50Plus => { + if owned { + rars::rar50::Archive::parse_owned_with_options(raw.to_vec(), options) + .map(Archive::Rar50Plus) + } else { + rars::rar50::Archive::parse_with_options(raw, options).map(Archive::Rar50Plus) + } + } + _ => unreachable!(), + } +} + +#[test] +fn header_limits_cover_families_entry_points_sfx_and_fresh_calls() { + let dir = scratch::case("header-limits"); + let path = dir.join("archive.rar"); + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let mut b = Builder::new(format).store(true); + for name in [b"first".as_slice(), b"second".as_slice()] { + b.add_bytes(name.to_vec(), vec![42; 64], None, None) + .unwrap(); + } + let original = b.to_bytes().unwrap(); + let expected = ArchiveReader::read(&original).unwrap(); + let sizes = headers(&expected, &original); + let count = sizes.len() as u64; + let bytes = sizes.iter().sum::(); + for sfx in [false, true] { + let mut raw = if sfx { + b"MZ small test stub".to_vec() + } else { + vec![] + }; + raw.extend_from_slice(&original); + std::fs::write(&path, &raw).unwrap(); + for options in [ + ArchiveReadOptions::new(), + ArchiveReadOptions::new() + .with_max_header_count(count) + .with_max_header_bytes(bytes), + ArchiveReadOptions::new() + .with_max_header_count(u64::MAX) + .with_max_header_bytes(u64::MAX), + ArchiveReadOptions::new().with_max_header_count(count - 1), + ArchiveReadOptions::new().with_max_header_bytes(bytes - 1), + ArchiveReadOptions::new() + .with_max_header_count(0) + .with_max_header_bytes(0), + ArchiveReadOptions::new().with_max_header_bytes(0), + ] { + let reject_count = options.max_header_count.is_some_and(|n| n < count); + let reject_bytes = options.max_header_bytes.is_some_and(|n| n < bytes); + let typed_path = match &expected { + Archive::Rar13(_) => { + rars::rar13::Archive::parse_path_with_options(&path, options) + .map(Archive::Rar13) + } + Archive::Rar15To40(_) => { + rars::rar15_40::Archive::parse_path_with_options(&path, options) + .map(Archive::Rar15To40) + } + Archive::Rar50Plus(_) => { + rars::rar50::Archive::parse_path_with_options(&path, options) + .map(Archive::Rar50Plus) + } + _ => unreachable!(), + }; + let results = [ + ArchiveReader::read_with_options(&raw, options), + ArchiveReader::read_owned_with_options(raw.clone(), options), + ArchiveReader::read_path_with_options(&path, options), + typed_read(&raw, options, false), + typed_read(&raw, options, true), + typed_path, + // Options are configuration, not mutable shared accounting. + ArchiveReader::read_with_options(&raw, options), + ]; + for result in results { + if reject_count || reject_bytes { + let e = result.unwrap_err(); + assert_eq!(e.kind(), ErrorKind::ResourceLimit, "{format:?}: {e}"); + assert!(matches!(e, Error::AtArchiveOffset { .. })); + if reject_count { + assert!( + matches!(e.root_cause(), Error::HeaderCountLimitExceeded {limit,required} + if *required == limit+1) + ); + } else { + let required = if options.max_header_bytes == Some(0) { + sizes[0] + } else { + bytes + }; + assert!( + matches!(e.root_cause(), Error::HeaderBytesLimitExceeded {required:r,..} if *r==required) + ); + } + } else { + let a = result.unwrap(); + assert_eq!(a.members().count(), 2); + // Parser policy must not become an extraction-output quota. + a.test(None).unwrap(); + } + } + } + } + } +} + +#[test] +fn comments_and_encrypted_headers_are_charged_once_by_plaintext_size() { + let dir = scratch::case("encrypted-header-limits"); + let path = dir.join("archive.rar"); + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for encrypted in [false, true] { + if encrypted && matches!(format, ArchiveVersion::Rar14 | ArchiveVersion::Rar29) { + continue; + } + let comment = + (!(encrypted && format == ArchiveVersion::Rar30)).then(|| b"comment".repeat(30)); + let mut b = Builder::new(format) + .store(true) + .comment(comment.clone()) + .password(encrypted.then(|| b"secret".to_vec())) + .header_encryption(encrypted); + b.add_bytes(b"file".to_vec(), vec![42; 16], None, None) + .unwrap(); + let raw = b.to_bytes().unwrap(); + std::fs::write(&path, &raw).unwrap(); + let options = ArchiveReadOptions::with_password(b"secret"); + let a = ArchiveReader::read_with_options(&raw, options).unwrap(); + let sizes = headers(&a, &raw); + let bytes = sizes.iter().sum::(); + let count = sizes.len() as u64; + for options in [ + options + .with_max_header_count(count) + .with_max_header_bytes(bytes), + options.with_max_header_count(count - 1), + options.with_max_header_bytes(bytes - 1), + ] { + for result in [ + ArchiveReader::read_with_options(&raw, options), + ArchiveReader::read_path_with_options(&path, options), + ] { + if options.max_header_count == Some(count - 1) + || options.max_header_bytes == Some(bytes - 1) + { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ResourceLimit); + } else { + let a = result.unwrap(); + assert_eq!(a.comment(Some(b"secret")).unwrap(), comment); + a.test(Some(b"secret")).unwrap(); + } + } + } + } + } +} + +#[test] +fn unknown_headers_count_and_existing_tail_handling_stays_tolerant() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut b = Builder::new(format).store(true); + b.add_bytes(b"file".to_vec(), vec![42; 32], None, None) + .unwrap(); + let mut raw = b.to_bytes().unwrap(); + let a = ArchiveReader::read(&raw).unwrap(); + match &a { + Archive::Rar15To40(a) => { + let h = &a.files().next().unwrap().block; + raw[h.offset + 2] = 0x7f; + let crc = + rars::crc32::crc32(&raw[h.offset + 2..h.offset + h.head_size as usize]) as u16; + raw[h.offset..h.offset + 2].copy_from_slice(&crc.to_le_bytes()); + } + Archive::Rar50Plus(a) => { + let h = &a.files().next().unwrap().block; + assert!(h.header_size < 128); + raw[h.offset + 5] = 0x7f; + let crc = + rars::crc32::crc32(&raw[h.offset + 4..h.offset + 5 + h.header_size as usize]); + raw[h.offset..h.offset + 4].copy_from_slice(&crc.to_le_bytes()); + } + Archive::Rar13(_) => {} + _ => unreachable!(), + } + let a = ArchiveReader::read(&raw).unwrap(); + if format != ArchiveVersion::Rar14 { + assert_eq!(a.members().count(), 0); + } + let sizes = headers(&a, &raw); + let count = sizes.len() as u64; + let bytes = sizes.iter().sum(); + // Short tails are ignored by legacy iteration; RAR5 stops at its end + // record. Neither should consume an extra header allowance. + raw.extend_from_slice(&[0x12, 0x34]); + ArchiveReader::read_with_options( + &raw, + ArchiveReadOptions::new() + .with_max_header_count(count) + .with_max_header_bytes(bytes), + ) + .unwrap(); + let e = ArchiveReader::read_with_options( + &raw, + ArchiveReadOptions::new().with_max_header_count(count - 1), + ) + .unwrap_err(); + assert!( + matches!(e.root_cause(),Error::HeaderCountLimitExceeded {required,..} if *required==count) + ); + } +} + +#[test] +fn physical_volume_parses_have_independent_allowances() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut b = Builder::new(format).store(true).volume_size(Some(512)); + b.add_bytes(b"split".to_vec(), vec![42; 2048], None, None) + .unwrap(); + let raw_volumes = b.build_volumes(None).unwrap(); + assert!(raw_volumes.len() > 1); + let mut volumes = Vec::new(); + for raw in raw_volumes { + let a = ArchiveReader::read(&raw).unwrap(); + let sizes = headers(&a, &raw); + volumes.push( + ArchiveReader::read_owned_with_options( + raw, + ArchiveReadOptions::new() + .with_max_header_count(sizes.len() as u64) + .with_max_header_bytes(sizes.iter().sum()), + ) + .unwrap(), + ); + } + rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::new() + .with_max_header_count(0) + .with_max_header_bytes(0), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap(); + } +} + +#[test] +fn directories_and_empty_members_still_consume_header_allowance() { + let mut b = Builder::new(ArchiveVersion::Rar50).store(true); + b.add_directory(b"dir".to_vec(), None, None).unwrap(); + b.add_bytes(b"empty".to_vec(), vec![], None, None).unwrap(); + let raw = b.to_bytes().unwrap(); + assert_eq!(ArchiveReader::read(&raw).unwrap().members().count(), 2); + let e = + ArchiveReader::read_with_options(&raw, ArchiveReadOptions::new().with_max_header_count(2)) + .unwrap_err(); + assert!(matches!( + e.root_cause(), + Error::HeaderCountLimitExceeded { + limit: 2, + required: 3 + } + )); +} diff --git a/arca-rar/vendor/rars/tests/reader_output_limit.rs b/arca-rar/vendor/rars/tests/reader_output_limit.rs new file mode 100644 index 00000000..bea05365 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_output_limit.rs @@ -0,0 +1,861 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReader, ArchiveVersion, Builder}; + +fn archive(size: usize) -> rars::rar50::Archive { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), vec![42; size], None, None) + .unwrap(); + match ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() { + Archive::Rar50Plus(a) => a, + _ => unreachable!(), + } +} + +#[test] +fn unknown_size_flag_does_not_mean_zero_or_the_placeholder_value() { + for size in [0, 32] { + let a = archive(size); + let mut file = a.files().next().unwrap().clone(); + assert_eq!(file.known_unpacked_size(), Some(size as u64)); + file.file_flags |= 0x0008; + for placeholder in [0, 1, u64::MAX] { + file.unpacked_size = placeholder; + assert_eq!(file.known_unpacked_size(), None); + } + file.file_flags &= !0x0008; + assert_eq!(file.known_unpacked_size(), Some(u64::MAX)); + } +} + +use rars::{ArchiveReadOptions, Error, ErrorKind}; +use std::{cell::RefCell, io::Write, rc::Rc}; + +struct Capture(Rc>>); +impl Write for Capture { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn rar5_output_admission_covers_modes_and_preserves_content() { + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for (stored, solid) in [(true, false), (false, false), (false, true)] { + let mut builder = Builder::new(format).store(stored).solid(solid); + builder + .add_bytes(b"file".to_vec(), vec![42; 256], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for buffered in [0, 1024] { + for parallel in [false, true] { + for limit in [None, Some(256), Some(255), Some(0)] { + let mut options = + ArchiveReadOptions::new().with_rar50_buffered_decode_limit(buffered); + options.max_member_output_bytes = limit; + let data = Rc::new(RefCell::new(Vec::new())); + let mut opened = 0; + let open = |_: &rars::ExtractedEntryMeta| { + opened += 1; + Ok(Box::new(Capture(data.clone())) as Box) + }; + let result = if parallel { + archive.extract_to_parallel_buffered_with_options(options, open) + } else { + archive.extract_to_with_options(options, open) + }; + if limit.is_some_and(|limit| limit < 256) { + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ResourceLimit); + assert_eq!(err.entry_context().unwrap().0, b"file"); + assert!(matches!( + err.root_cause(), + Error::MemberOutputLimitExceeded { required: 256, .. } + )); + assert_eq!(opened, 0); + } else { + result.unwrap(); + assert_eq!(opened, 1); + assert_eq!(*data.borrow(), vec![42; 256]); + } + } + } + } + } + } +} + +#[test] +fn unknown_logical_size_is_refused_only_under_an_output_policy() { + for total in [false, true] { + for size in [0, 32] { + let mut a = archive(size); + for b in &mut a.blocks { + if let rars::rar50::Block::File(f) = b { + f.file_flags |= 0x8; + } + } + // Characterize existing behavior; the limit does not silently change + // default extraction to a new decode-to-end implementation. + a.extract_to(ArchiveReadOptions::new(), |_| Ok(Box::new(std::io::sink()))) + .unwrap(); + for limit in [0, 32, u64::MAX] { + let err = a + .extract_to( + output_options(ArchiveReadOptions::new(), limit, total), + |_| panic!("unsupported unknown size opened output"), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::UnsupportedFeature); + } + } + } +} + +#[test] +fn split_limit_uses_final_size_and_preserves_early_unknown_placeholders() { + for total in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .password(Some(b"secret".to_vec())) + .volume_size(Some(512)); + builder + .add_bytes(b"file".to_vec(), vec![42; 2048], None, None) + .unwrap(); + let mut volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|b| ArchiveReader::read_owned(b).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for a in &mut volumes { + if let Archive::Rar50Plus(a) = a { + for b in &mut a.blocks { + if let rars::rar50::Block::File(f) = b { + if f.is_split_after() { + f.file_flags |= 0x8; + f.unpacked_size = u64::MAX; + } + } + } + } + } + for limit in [2047, 2048] { + let options = + output_options(ArchiveReadOptions::with_password(b"secret"), limit, total); + let data = Rc::new(RefCell::new(Vec::new())); + let mut opened = 0; + let result = rars::extract_volumes_to_with_options(&volumes, options, |_| { + opened += 1; + Ok(Box::new(Capture(data.clone()))) + }); + if limit < 2048 { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ResourceLimit); + assert_eq!(opened, 0); + } else { + result.unwrap(); + assert_eq!(*data.borrow(), vec![42; 2048]); + } + } + if let Archive::Rar50Plus(a) = volumes.last_mut().unwrap() { + for b in &mut a.blocks { + if let rars::rar50::Block::File(f) = b { + f.file_flags |= 0x8; + } + } + } + let err = rars::extract_volumes_to_with_options( + &volumes, + output_options( + ArchiveReadOptions::with_password(b"secret"), + u64::MAX, + total, + ), + |_| panic!("unknown final size opened output"), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::UnsupportedFeature); + } +} + +#[test] +fn legacy_output_limits_cover_sequential_parallel_and_discarded_solid_members() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for stored in [true, false] { + let mut builder = Builder::new(format).store(stored); + builder + .add_bytes(b"file".to_vec(), vec![42; 256], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for parallel in [false, true] { + for limit in [255, 256] { + let mut opened = 0; + let data = Rc::new(RefCell::new(Vec::new())); + let open = |_: &rars::ExtractedEntryMeta| { + opened += 1; + Ok(Box::new(Capture(data.clone())) as Box) + }; + let options = ArchiveReadOptions::new().with_max_member_output_bytes(limit); + let result = if parallel { + archive.extract_to_parallel_buffered_with_options(options, open) + } else { + archive.extract_to_with_options(options, open) + }; + if limit < 256 { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ResourceLimit); + assert_eq!(opened, 0); + } else { + result.unwrap(); + assert_eq!(*data.borrow(), vec![42; 256]); + } + } + } + } + } + for format in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(format).solid(true); + builder + .add_bytes(b"discarded".to_vec(), vec![42; 256], None, None) + .unwrap(); + builder + .add_bytes(b"wanted".to_vec(), vec![42; 16], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + // Discarding output does not exempt a preceding solid member from policy. + let err = archive + .extract_to_with_options( + ArchiveReadOptions::new().with_max_member_output_bytes(16), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ResourceLimit); + assert_eq!(err.entry_context().unwrap().0, b"discarded"); + } +} + +#[test] +fn legacy_split_ceiling_applies_to_the_whole_member() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ] { + let mut builder = Builder::new(format) + .store(true) + .password((format != ArchiveVersion::Rar14).then(|| b"secret".to_vec())) + .volume_size(Some(512)); + builder + .add_bytes(b"file".to_vec(), vec![42; 2048], None, None) + .unwrap(); + let volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|b| ArchiveReader::read_owned(b).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for limit in [512, 2048] { + let mut opened = 0; + let data = Rc::new(RefCell::new(Vec::new())); + let result = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::with_password(b"secret").with_max_member_output_bytes(limit), + |_| { + opened += 1; + Ok(Box::new(Capture(data.clone()))) + }, + ); + if limit < 2048 { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ResourceLimit); + assert_eq!(opened, 0); + } else { + result.unwrap(); + assert_eq!(*data.borrow(), vec![42; 2048]); + } + } + } +} + +#[test] +fn actual_stored_output_is_guarded_when_legacy_header_understates_size() { + for total in [false, true] { + for encrypted in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar14) + .store(true) + .password(encrypted.then(|| b"secret".to_vec())); + builder + .add_bytes(b"prefix".to_vec(), vec![42; 8], None, None) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), vec![42; 64], None, None) + .unwrap(); + let mut archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + if let Archive::Rar13(a) = &mut archive { + a.entries[1].header.unp_size = 16; + } + let data = Rc::new(RefCell::new(Vec::new())); + let mut opened = 0; + let err = archive + .extract_to_with_options( + output_options(ArchiveReadOptions::with_password(b"secret"), 32, total), + |_| { + opened += 1; + Ok(Box::new(Capture(data.clone()))) + }, + ) + .unwrap_err(); + assert_eq!(opened, 2); // Header admission passed, runtime enforcement refused. + assert!(data.borrow().len() <= if total { 32 } else { 40 }); + assert_eq!(err.kind(), ErrorKind::ResourceLimit); + assert_eq!(err.entry_context().unwrap().0, b"file"); + if total { + assert!(matches!( + err.root_cause(), + Error::TotalOutputLimitExceeded { + used: 8, + required: 72, + .. + } + )); + } + } + } +} + +#[test] +fn empty_members_and_directories_do_not_consume_output_allowance() { + for total in [false, true] { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"empty".to_vec(), vec![], None, None) + .unwrap(); + if format == ArchiveVersion::Rar50 { + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + } + let a = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + a.extract_to_with_options(output_options(ArchiveReadOptions::new(), 0, total), |_| { + Ok(Box::new(std::io::sink())) + }) + .unwrap(); + } + } +} + +#[test] +fn split_runtime_guard_and_sink_errors_keep_their_identity() { + for total in [false, true] { + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::PermissionDenied.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(format).store(true).volume_size(Some(512)); + builder + .add_bytes(b"file".to_vec(), vec![42; 2048], None, None) + .unwrap(); + let mut volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|b| ArchiveReader::read_owned(b).unwrap()) + .collect(); + // Deliberately corrupt the final integrity record: a sink refusal must + // not be replaced with that unrelated checksum diagnostic. + match volumes.last_mut().unwrap() { + Archive::Rar13(a) => a.entries.last_mut().unwrap().header.file_crc ^= 1, + Archive::Rar15To40(a) => { + for b in &mut a.blocks { + if let rars::rar15_40::Block::File(f) = b { + f.file_crc ^= 1; + } + } + } + Archive::Rar50Plus(a) => { + for b in &mut a.blocks { + if let rars::rar50::Block::File(f) = b { + f.data_crc32 = Some(0); + f.hash = None; + } + } + } + _ => unreachable!(), + } + let err = rars::extract_volumes_to_with_options( + &volumes, + output_options(ArchiveReadOptions::new(), 2048, total), + |_| Ok(Box::new(Broken)), + ) + .unwrap_err(); + assert!( + matches!(err.root_cause(), Error::Io(e) if e.kind == std::io::ErrorKind::PermissionDenied) + ); + // These stored families copy chained data even if the final header lies. + if format != ArchiveVersion::Rar29 { + match volumes.last_mut().unwrap() { + Archive::Rar13(a) => a.entries.last_mut().unwrap().header.unp_size = 16, + Archive::Rar50Plus(a) => { + for b in &mut a.blocks { + if let rars::rar50::Block::File(f) = b { + f.unpacked_size = 16; + } + } + } + _ => unreachable!(), + } + let data = Rc::new(RefCell::new(Vec::new())); + let err = rars::extract_volumes_to_with_options( + &volumes, + output_options(ArchiveReadOptions::new(), 512, total), + |_| Ok(Box::new(Capture(data.clone()))), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ResourceLimit); + assert!(data.borrow().len() <= 512); + } + } + } +} + +#[test] +fn filtered_output_is_counted_once_and_limits_remain_distinct() { + for total in [false, true] { + let a = ArchiveReader::read(include_bytes!("fixtures/rar50/filter_delta.rar")).unwrap(); + let expected = Rc::new(RefCell::new(Vec::new())); + a.extract_to(None, |_| Ok(Box::new(Capture(expected.clone())))) + .unwrap(); + let size = expected.borrow().len() as u64; + assert!(size > 0); + let actual = Rc::new(RefCell::new(Vec::new())); + a.extract_to_with_options( + output_options(ArchiveReadOptions::new(), size, total), + |_| Ok(Box::new(Capture(actual.clone()))), + ) + .unwrap(); + assert_eq!(*actual.borrow(), *expected.borrow()); + let err = a + .extract_to_with_options( + output_options(ArchiveReadOptions::new(), size - 1, total), + |_| panic!("oversized filtered member opened output"), + ) + .unwrap_err(); + assert!(matches!( + err.root_cause(), + Error::MemberOutputLimitExceeded { .. } | Error::TotalOutputLimitExceeded { .. } + )); + let err = a + .extract_to_with_options( + output_options(ArchiveReadOptions::new(), size, total) + .with_rar50_buffered_decode_limit(0), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap_err(); + assert!(matches!( + err.root_cause(), + Error::Rar50BufferedDecodeLimitExceeded { .. } + )); + } +} + +#[test] +fn allowance_resets_between_members_including_solid_and_parallel_paths() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + for solid in [false, true] { + let mut builder = Builder::new(format).solid(solid); + for name in [b"first".as_slice(), b"second".as_slice()] { + builder + .add_bytes(name.to_vec(), vec![42; 32], None, None) + .unwrap(); + } + let a = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for parallel in [false, true] { + let data = Rc::new(RefCell::new(Vec::new())); + let open = |_: &rars::ExtractedEntryMeta| { + Ok(Box::new(Capture(data.clone())) as Box) + }; + let options = ArchiveReadOptions::new().with_max_member_output_bytes(32); + if parallel { + a.extract_to_parallel_buffered_with_options(options, open) + .unwrap(); + } else { + a.extract_to_with_options(options, open).unwrap(); + } + assert_eq!(*data.borrow(), vec![42; 64]); + } + } + } +} + +#[test] +fn total_quota_counts_members_in_order_across_all_families_and_modes() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for (stored, solid) in [(true, false), (false, false), (false, true)] { + let mut builder = Builder::new(format).store(stored).solid(solid); + for name in [b"first".as_slice(), b"second".as_slice()] { + builder + .add_bytes(name.to_vec(), vec![42; 32], None, None) + .unwrap(); + } + let a = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for buffered in [0, 1024] { + for parallel in [false, true] { + for total in [None, Some(0), Some(63), Some(64), Some(u64::MAX)] { + let mut options = ArchiveReadOptions::new() + .with_max_member_output_bytes(32) + .with_rar50_buffered_decode_limit(buffered); + options.max_total_output_bytes = total; + // Reusing options must never retain a previous call's consumption. + for _ in 0..2 { + let mut opened = Vec::new(); + let data = Rc::new(RefCell::new(Vec::new())); + let open = |meta: &rars::ExtractedEntryMeta| { + opened.push(meta.name.clone()); + if solid && meta.name == b"first" { + Ok(Box::new(std::io::sink()) as Box) + } else { + Ok(Box::new(Capture(data.clone())) as Box) + } + }; + let result = if parallel { + a.extract_to_parallel_buffered_with_options(options, open) + } else { + a.extract_to_with_options(options, open) + }; + if let Some(limit) = total.filter(|n| *n < 64) { + let used = if limit == 0 { 0 } else { 32 }; + let err = result.unwrap_err(); + assert!( + matches!(err.root_cause(), Error::TotalOutputLimitExceeded { + limit:l, used:u, required:r + } if *l == limit && *u == used && *r == used + 32) + ); + assert_eq!(opened.len(), (used / 32) as usize); + assert_eq!( + err.entry_context().unwrap().0, + if used == 0 { + b"first".as_slice() + } else { + b"second" + } + ); + } else { + result.unwrap(); + assert_eq!(opened, [b"first", b"second".as_slice()]); + assert_eq!(*data.borrow(), vec![42; if solid { 32 } else { 64 }]); + } + } + } + } + } + } + } +} + +#[test] +fn typed_parallel_calls_admit_total_before_later_payload_errors() { + for format in [ArchiveVersion::Rar29, ArchiveVersion::Rar50] { + let mut builder = Builder::new(format).store(true); + for name in [b"first".as_slice(), b"second".as_slice()] { + builder + .add_bytes(name.to_vec(), vec![42; 32], None, None) + .unwrap(); + } + let mut a = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + // A later corrupt member must not hide the quota error or prevent the + // admitted prefix from being published by either typed parallel API. + let mut opened = 0; + let options = ArchiveReadOptions::new().with_max_total_output_bytes(63); + let err = match &mut a { + Archive::Rar15To40(a) => { + for b in &mut a.blocks { + if let rars::rar15_40::Block::File(f) = b { + if f.name == b"second" { + f.file_crc ^= 1; + } + } + } + a.extract_to_parallel_buffered(options, |_| { + opened += 1; + Ok(Box::new(std::io::sink())) + }) + .unwrap_err() + } + Archive::Rar50Plus(a) => { + for b in &mut a.blocks { + if let rars::rar50::Block::File(f) = b { + if f.name == b"second" { + f.data_crc32 = Some(0); + f.hash = None; + } + } + } + a.extract_to_parallel_buffered(options, |_| { + opened += 1; + Ok(Box::new(std::io::sink())) + }) + .unwrap_err() + } + _ => unreachable!(), + }; + assert_eq!(opened, 1); + assert!(matches!( + err.root_cause(), + Error::TotalOutputLimitExceeded { + used: 32, + required: 64, + .. + } + )); + } +} + +#[test] +fn total_quota_persists_across_regular_split_and_following_volume_members() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let legacy = !matches!(format, ArchiveVersion::Rar50 | ArchiveVersion::Rar70); + let mut builder = Builder::new(format) + .store(true) + .volume_size(Some(512)) + .password((format != ArchiveVersion::Rar14).then(|| b"secret".to_vec())); + for (name, size) in [ + (b"first".as_slice(), 32), + (b"split".as_slice(), 2048), + (b"last".as_slice(), 32), + ] { + if legacy && name != b"split" { + continue; + } + builder + .add_bytes(name.to_vec(), vec![42; size], None, None) + .unwrap(); + } + let mut volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|b| ArchiveReader::read_owned(b).unwrap()) + .collect(); + assert!(volumes.len() > 1); + if legacy { + // The legacy writer supports only one input in a volume set. + // Assemble parsed ordinary sources around its split sources to test + // extraction accounting across all three routes in one operation. + let ordinary = |name: &[u8]| { + let mut b = Builder::new(format).store(true); + b.add_bytes(name.to_vec(), vec![42; 32], None, None) + .unwrap(); + ArchiveReader::read_owned(b.to_bytes().unwrap()).unwrap() + }; + volumes.insert(0, ordinary(b"first")); + volumes.push(ordinary(b"last")); + } + let selected = rars::read_volume_member_at_with_options( + &volumes, + 0, + ArchiveReadOptions::with_password(b"secret").with_max_total_output_bytes(2112), + ) + .unwrap(); + assert_eq!(selected, Some(vec![42; 32])); + // Selecting the first member still charges later discarded members. + let err = rars::read_volume_member_at_with_options( + &volumes, + 0, + ArchiveReadOptions::with_password(b"secret").with_max_total_output_bytes(2111), + ) + .unwrap_err(); + assert!(matches!( + err.root_cause(), + Error::TotalOutputLimitExceeded { .. } + )); + for (limit, used, required, count) in [ + (2079, 32, 2080, 1), + (2111, 2080, 2112, 2), + (2112, 2112, 2112, 3), + ] { + let data = Rc::new(RefCell::new(Vec::new())); + let mut opened = 0; + let result = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::with_password(b"secret").with_max_total_output_bytes(limit), + |_| { + opened += 1; + Ok(Box::new(Capture(data.clone()))) + }, + ); + assert_eq!(opened, count, "{format:?}, limit={limit}"); + assert_eq!(data.borrow().len() as u64, used); + if limit < 2112 { + let err = result.unwrap_err(); + assert!( + matches!(err.root_cause(), Error::TotalOutputLimitExceeded {used:u,required:r,..} if *u==used && *r==required) + ); + } else { + result.unwrap(); + } + } + } +} + +fn output_options( + options: ArchiveReadOptions<'_>, + limit: u64, + total: bool, +) -> ArchiveReadOptions<'_> { + if total { + options.with_max_total_output_bytes(limit) + } else { + options.with_max_member_output_bytes(limit) + } +} + +#[test] +fn total_admission_precedes_compressed_payload_decryption_including_split_streams() { + let mut state = 0x1234_5678u32; + let noise: Vec = (0..4096) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect(); + let payload = noise.repeat(3); + for format in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let mut builder = Builder::new(format).password(Some(b"secret".to_vec())); + builder + .add_bytes(b"file".to_vec(), payload.clone(), None, None) + .unwrap(); + let a = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + if let Archive::Rar50Plus(a) = &a { + assert!(!a.files().next().unwrap().is_stored()); + } + let volumes: Vec<_> = builder + .volume_size(Some(512)) + .build_volumes(None) + .unwrap() + .into_iter() + .map(|b| ArchiveReader::read_owned(b).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for buffered in [0, u64::MAX] { + for split in [false, true] { + for reject in [false, true] { + // Preserve early password-presence validation on fragments; + // a wrong password proves refusal precedes payload decryption. + let options = ArchiveReadOptions::with_password(if reject { + b"wrong" + } else { + b"secret" + }) + .with_rar50_buffered_decode_limit(buffered) + .with_max_total_output_bytes(payload.len() as u64 - u64::from(reject)); + let data = Rc::new(RefCell::new(Vec::new())); + let mut opened = 0; + let open = |_: &rars::ExtractedEntryMeta| { + opened += 1; + Ok(Box::new(Capture(data.clone())) as Box) + }; + let result = if split { + rars::extract_volumes_to_with_options(&volumes, options, open) + } else { + a.extract_to_with_options(options, open) + }; + if reject { + assert_eq!(opened, 0); + assert!(matches!( + result.unwrap_err().root_cause(), + Error::TotalOutputLimitExceeded { used: 0, .. } + )); + } else { + result.unwrap(); + assert_eq!(*data.borrow(), payload); + } + } + } + } + } +} + +#[test] +fn zero_total_allows_empty_archives() { + for format in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + // The builder requires an entry. Remove it from the parsed member list + // to exercise empty reader iteration without changing writer policy. + let mut builder = Builder::new(format).store(true); + builder + .add_bytes(b"placeholder".to_vec(), vec![], None, None) + .unwrap(); + let mut a = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + match &mut a { + Archive::Rar13(a) => a.entries.clear(), + Archive::Rar15To40(a) => a + .blocks + .retain(|b| !matches!(b, rars::rar15_40::Block::File(_))), + Archive::Rar50Plus(a) => a + .blocks + .retain(|b| !matches!(b, rars::rar50::Block::File(_))), + _ => unreachable!(), + } + assert_eq!(a.members().count(), 0); + a.extract_to_parallel_buffered_with_options( + ArchiveReadOptions::new().with_max_total_output_bytes(0), + |_| panic!("empty archive opened a member"), + ) + .unwrap(); + } +} diff --git a/arca-rar/vendor/rars/tests/reader_scratch.rs b/arca-rar/vendor/rars/tests/reader_scratch.rs new file mode 100644 index 00000000..b059d6fe --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_scratch.rs @@ -0,0 +1,344 @@ +#![cfg(feature = "write")] +#![cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +#[path = "support/scratch.rs"] +mod scratch; +use rars::rar50::{ArchiveEntry, Rar50Writer, WriterOptions}; +use rars::{ + Archive, ArchiveReadOptions, ArchiveReader, ArchiveVersion, EntrySource, Error, FeatureSet, + FilterKind, FilterPolicy, Rar50Scratch, +}; +use std::{cell::RefCell, io, io::Write, rc::Rc, sync::Arc}; + +struct Capture(Rc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn payload() -> Vec { + (0..196_608) + .map(|i| [0xe8, 1, 0, 0, 0, 0xe9, 3, 0, 0, 0, 0xeb, 0x14][i % 12]) + .collect() +} +fn entry(data: &[u8], encrypted: bool) -> ArchiveEntry { + let entry = ArchiveEntry::new( + b"file".to_vec(), + EntrySource::from_bytes(Arc::<[u8]>::from(data.to_vec())), + ); + if encrypted { + entry.with_password(b"secret".to_vec()) + } else { + entry + } +} +fn options(version: ArchiveVersion) -> WriterOptions { + WriterOptions::new(version, FeatureSet::store_only()) +} +fn archive(kind: FilterKind, encrypted: bool, version: ArchiveVersion) -> Archive { + ArchiveReader::read_owned( + Rar50Writer::new(options(version)) + .entries([entry(&payload(), encrypted)]) + .filter_policy(FilterPolicy::explicit(kind)) + .finish() + .unwrap(), + ) + .unwrap() +} + +#[test] +fn disk_scratch_handles_all_filters_encryption_and_parallel_entry_points() { + let dir = scratch::case("reader-scratch-filters"); + let policy = Rar50Scratch::new(&*dir, 1_000_000).with_filter_memory_limit(128 * 1024); + for version in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for kind in [ + FilterKind::E8, + FilterKind::E8E9, + FilterKind::Arm, + FilterKind::Delta { channels: 3 }, + ] { + for encrypted in [false, true] { + let archive = archive(kind, encrypted, version); + let mut options = ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(1024) + .with_rar50_scratch(&policy); + options.password = encrypted.then_some(b"secret"); + let output = Rc::new(RefCell::new(Vec::new())); + archive + .extract_to_parallel_buffered_with_options(options, |_| { + Ok(Box::new(Capture(output.clone()))) + }) + .unwrap(); + assert_eq!(*output.borrow(), payload()); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } + } + } +} + +#[test] +fn scratch_and_filter_limits_refuse_before_publication_and_clean_up() { + let dir = scratch::case("reader-scratch-limits"); + let archive = archive(FilterKind::E8, false, ArchiveVersion::Rar50); + for (disk, memory, filter_error) in [ + (1000, 131072, false), + (1_000_000, 1024, true), + (393216, 131072, false), + ] { + let policy = Rar50Scratch::new(&*dir, disk).with_filter_memory_limit(memory); + let output = Rc::new(RefCell::new(Vec::new())); + let error = archive + .extract_to_with_options( + ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(1024) + .with_rar50_scratch(&policy), + |_| Ok(Box::new(Capture(output.clone()))), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ResourceLimit); + assert!(if filter_error { + matches!( + error.root_cause(), + Error::Rar50FilterMemoryLimitExceeded { .. } + ) + } else { + matches!(error.root_cause(), Error::Rar50ScratchLimitExceeded { .. }) + }); + assert!(output.borrow().is_empty()); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } +} + +#[test] +fn integrity_and_sink_failures_release_scratch() { + let dir = scratch::case("reader-scratch-failures"); + let policy = Rar50Scratch::new(&*dir, 1_000_000); + let mut archive = archive( + FilterKind::Delta { channels: 3 }, + false, + ArchiveVersion::Rar50, + ); + let options = ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(1024) + .with_rar50_scratch(&policy); + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::PermissionDenied.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let error = archive + .extract_to_with_options(options, |_| Ok(Box::new(Broken))) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + if let Archive::Rar50Plus(a) = &mut archive { + for block in &mut a.blocks { + if let rars::rar50::Block::File(file) = block { + file.hash = None; + file.data_crc32 = Some(0); + } + } + } + let output = Rc::new(RefCell::new(Vec::new())); + let error = archive + .extract_to_with_options(options, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ChecksumMismatch); + assert!(output.borrow().is_empty()); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); +} + +#[test] +fn split_filtered_members_use_scratch_and_keep_limit_errors() { + let dir = scratch::case("reader-scratch-split"); + for encrypted in [false, true] { + let mut sink = rars::rar50::CollectedVolumes::new(); + rars::rar50::write_streaming_volumes_to( + &[entry(&payload(), encrypted)], + options(ArchiveVersion::Rar50), + rars::rar50::ArchiveExtras::default() + .with_filter_policy(FilterPolicy::explicit(FilterKind::E8)), + 64, + &mut sink, + &rars::WriterResources::default().with_temp_dir(&*dir), + ) + .unwrap(); + let volumes: Vec<_> = sink + .take() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for disk in [1_000_000, 100] { + let policy = Rar50Scratch::new(&*dir, disk); + let mut options = ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(1024) + .with_rar50_scratch(&policy); + options.password = encrypted.then_some(b"secret"); + let output = Rc::new(RefCell::new(Vec::new())); + let result = rars::extract_volumes_to_with_options(&volumes, options, |_| { + Ok(Box::new(Capture(output.clone()))) + }); + if disk == 100 { + assert!(matches!( + result.unwrap_err().root_cause(), + Error::Rar50ScratchLimitExceeded { .. } + )); + assert!(output.borrow().is_empty()); + } else { + result.unwrap(); + assert_eq!(*output.borrow(), payload()); + } + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } + } +} + +#[test] +fn publication_cancellation_cleans_up_scratch() { + let dir = scratch::case("reader-scratch-cancel"); + let policy = Rar50Scratch::new(&*dir, 1_000_000); + let token = rars::ReadCancellation::new(); + struct Cancel(rars::ReadCancellation); + impl Write for Cancel { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.cancel(); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let archive = archive(FilterKind::E8, false, ArchiveVersion::Rar50); + let error = archive + .extract_to_with_options( + ArchiveReadOptions::new() + .with_cancellation(&token) + .with_rar50_buffered_decode_limit(1024) + .with_rar50_scratch(&policy), + |_| Ok(Box::new(Cancel(token.clone()))), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Cancelled); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); +} + +#[test] +fn filtered_checksums_cannot_validate_raw_output_with_or_without_scratch() { + let dir = scratch::case("reader-scratch-fallback"); + let policy = Rar50Scratch::new(&*dir, 1_000_000); + let mut archive = archive(FilterKind::E8, false, ArchiveVersion::Rar50); + let Archive::Rar50Plus(a) = &mut archive else { + unreachable!() + }; + let file = a.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + let raw = rars::codec::rar50::Unpack50Decoder::new() + .decode_member_with_dictionary( + &file.packed_data(a).unwrap(), + info.algorithm_version, + payload().len(), + info.dictionary_size as usize, + false, + rars::codec::rar50::DecodeMode::LzNoFilters, + ) + .unwrap(); + assert_ne!(raw, payload()); + for block in &mut a.blocks { + if let rars::rar50::Block::File(file) = block { + file.hash = None; + file.data_crc32 = Some(rars::crc32::crc32(&raw)); + } + } + for scratch in [false, true] { + let mut options = ArchiveReadOptions::new(); + if scratch { + options = options + .with_rar50_buffered_decode_limit(1024) + .with_rar50_scratch(&policy); + } + let output = Rc::new(RefCell::new(Vec::new())); + let error = archive + .extract_to_with_options(options, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + rars::Error::Crc32Mismatch { .. } + )); + assert!(output.borrow().is_empty()); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } +} + +#[test] +fn unfiltered_zero_matches_stream_with_and_without_scratch() { + let dir = scratch::case("reader-scratch-zero-matches"); + let policy = Rar50Scratch::new(&*dir, 400_000); + let plain = vec![0; 131_089]; + let archive = ArchiveReader::read_owned( + Rar50Writer::new(options(ArchiveVersion::Rar50)) + .entries([entry(&plain, false)]) + .filter_policy(FilterPolicy::None) + .finish() + .unwrap(), + ) + .unwrap(); + for scratch in [false, true] { + let mut options = ArchiveReadOptions::new().with_rar50_buffered_decode_limit(0); + if scratch { + options = options.with_rar50_scratch(&policy); + } + let output = Rc::new(RefCell::new(Vec::new())); + archive + .extract_to_with_options(options, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap(); + assert_eq!(*output.borrow(), plain); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } +} + +#[test] +fn scratch_decoder_failure_discards_output_and_removes_temporary_files() { + let dir = scratch::case("reader-scratch-truncated-source"); + let path = dir.join("input.rar"); + let bytes = Rar50Writer::new(options(ArchiveVersion::Rar50)) + .entries([entry(&[0; 2000], false)]) + .filter_policy(FilterPolicy::None) + .finish() + .unwrap(); + std::fs::write(&path, bytes).unwrap(); + let archive = ArchiveReader::read_path(&path).unwrap(); + let Archive::Rar50Plus(raw) = &archive else { + unreachable!() + }; + let data_start = raw.files().next().unwrap().block.data_range.start; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(data_start as u64) + .unwrap(); + let policy = Rar50Scratch::new(&*dir, 10_000); + let output = Rc::new(RefCell::new(Vec::new())); + let error = archive + .extract_to_with_options( + ArchiveReadOptions::new() + .with_rar50_buffered_decode_limit(0) + .with_rar50_scratch(&policy), + |_| Ok(Box::new(Capture(output.clone()))), + ) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::Codec(rars::codec::Error::NeedMoreInput) + )); + assert!(output.borrow().is_empty()); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 1); +} diff --git a/arca-rar/vendor/rars/tests/reader_signatures.rs b/arca-rar/vendor/rars/tests/reader_signatures.rs new file mode 100644 index 00000000..e031591d --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_signatures.rs @@ -0,0 +1,261 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::{detect_archive_family, find_archive_start, rar15_40, rar50, ArchiveReadOptions, Error}; + +#[test] +fn rar5_path_parser_reports_missing_files_and_missing_signatures() { + let root = scratch::case("rar5-path-input-errors"); + let path = root.join("missing.rar"); + let error = rar50::Archive::parse_path(&path).unwrap_err(); + assert!(matches!(error.root_cause(), Error::Io(io) if io.kind == std::io::ErrorKind::NotFound)); + + std::fs::write(&path, b"ordinary bytes, with no RAR marker").unwrap(); + assert!(matches!( + rar50::Archive::parse_path(&path), + Err(Error::UnsupportedSignature) + )); + + let cancelled = rars::ReadCancellation::new(); + cancelled.cancel(); + assert!(matches!( + rar50::Archive::parse_path_with_options( + &path, + ArchiveReadOptions::new().with_cancellation(&cancelled) + ), + Err(Error::Cancelled) + )); + let signature = detect_archive_family(b"Rar!\x1a\x07\x01\0").unwrap(); + assert!(matches!( + rar50::Archive::parse_path_with_signature( + &path, + signature, + ArchiveReadOptions::new().with_cancellation(&cancelled) + ), + Err(Error::Cancelled) + )); +} + +#[cfg(target_pointer_width = "32")] +#[test] +fn rar5_path_parser_rejects_sparse_archives_above_native_size() { + let root = scratch::case("rar5-native32-sparse-size"); + let path = root.join("large.rar"); + let mut file = std::fs::File::create(&path).unwrap(); + std::io::Write::write_all(&mut file, b"Rar!\x1a\x07\x01\0").unwrap(); + file.set_len(u64::from(u32::MAX) + 1).unwrap(); + drop(file); + + let signature = detect_archive_family(b"Rar!\x1a\x07\x01\0").unwrap(); + for result in [ + rar50::Archive::parse_path(&path), + rar50::Archive::parse_path_with_signature(&path, signature, ArchiveReadOptions::new()), + ] { + assert!(matches!( + result, + Err(Error::InvalidHeader("RAR 5 archive size overflows usize")) + )); + } +} + +#[test] +fn typed_path_parsers_validate_explicit_signatures_and_sfx_offsets() { + let legacy = include_bytes!("fixtures/rar15_40/rar300/stored_multivol_rar300.rar"); + let modern = include_bytes!("fixtures/rar50/stored.rar"); + let root = scratch::case("reader-signature-boundaries"); + let prefix = b"an SFX stub without a RAR signature"; + for (is_modern, bytes) in [(false, legacy.as_slice()), (true, modern.as_slice())] { + let mut image = prefix.to_vec(); + image.extend_from_slice(bytes); + let path = root.join(if is_modern { + "modern.rar" + } else { + "legacy.rar" + }); + std::fs::write(&path, &image).unwrap(); + let signature = find_archive_start(&image, image.len()).unwrap(); + assert_eq!(signature.offset, prefix.len()); + if is_modern { + for archive in [ + rar50::Archive::parse_path(&path).unwrap(), + rar50::Archive::parse_path_with_signature_and_password(&path, signature, None) + .unwrap(), + ] { + assert_eq!(archive.sfx_offset, prefix.len()); + let file = archive.files().next().unwrap(); + assert_eq!( + file.packed_data(&archive).unwrap(), + b"Hello, RAR 5.0 fixture world.\n" + ); + } + assert!(matches!( + rar50::Archive::parse(legacy), + Err(Error::UnsupportedSignature) + )); + let wrong = detect_archive_family(legacy).unwrap(); + assert!(matches!( + rar50::Archive::parse_path_with_signature(&path, wrong, ArchiveReadOptions::new()), + Err(Error::UnsupportedSignature) + )); + } else { + for archive in [ + rar15_40::Archive::parse_path(&path).unwrap(), + rar15_40::Archive::parse_path_with_signature_and_password(&path, signature, None) + .unwrap(), + ] { + assert_eq!(archive.sfx_offset, prefix.len()); + assert!(archive.files().next().is_some()); + } + assert!(matches!( + rar15_40::Archive::parse(modern), + Err(Error::UnsupportedSignature) + )); + let wrong = detect_archive_family(modern).unwrap(); + assert!(matches!( + rar15_40::Archive::parse_path_with_signature( + &path, + wrong, + ArchiveReadOptions::new() + ), + Err(Error::UnsupportedSignature) + )); + } + for offset in [prefix.len() + 1, image.len(), image.len() + 1, usize::MAX] { + let mut invalid = signature; + invalid.offset = offset; + let result = if is_modern { + rar50::Archive::parse_path_with_signature(&path, invalid, ArchiveReadOptions::new()) + .map(|_| ()) + } else { + rar15_40::Archive::parse_path_with_signature( + &path, + invalid, + ArchiveReadOptions::new(), + ) + .map(|_| ()) + }; + assert!(result.is_err(), "offset {offset}"); + } + std::fs::write( + &path, + if is_modern { + legacy.as_slice() + } else { + modern.as_slice() + }, + ) + .unwrap(); + let result = if is_modern { + rar50::Archive::parse_path(&path).map(|_| ()) + } else { + rar15_40::Archive::parse_path(&path).map(|_| ()) + }; + assert!(matches!(result, Err(Error::UnsupportedSignature))); + } +} + +#[test] +fn legacy_stored_member_reports_source_truncation_after_header_read() { + let root = scratch::case("legacy-source-truncation"); + let path = root.join("truncated.rar"); + let mut builder = rars::Builder::new(rars::ArchiveVersion::Rar30).compression_level(Some(0)); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + std::fs::write(&path, builder.to_bytes().unwrap()).unwrap(); + let archive = rar15_40::Archive::parse_path(&path).unwrap(); + let file = archive.files().next().unwrap(); + assert!(file.is_stored()); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len((file.packed_range.start + 2) as u64) + .unwrap(); + let mut output = Vec::new(); + let error = file.write_to(&archive, None, &mut output).unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::InvalidHeader("RAR 1.5 stored file ended before unpacked size") + )); + assert_eq!(output, b"pa"); +} + +#[test] +fn explicit_legacy_signatures_still_require_the_exact_marker_block() { + let root = scratch::case("legacy-marker-admission"); + let path = root.join("marker.rar"); + let legacy = include_bytes!("fixtures/rar15_40/rar300/stored_multivol_rar300.rar"); + let signature = detect_archive_family(legacy).unwrap(); + let mut wrong_size = legacy[..7].to_vec(); + wrong_size[5] = 8; + wrong_size.push(0); + for bytes in [legacy[7..20].to_vec(), wrong_size] { + std::fs::write(&path, bytes).unwrap(); + let error = rar15_40::Archive::parse_path_with_signature( + &path, + signature, + ArchiveReadOptions::new(), + ) + .unwrap_err(); + assert!(matches!( + error, + Error::InvalidHeader("RAR 1.5 marker block is invalid") + )); + } +} + +#[test] +fn seekable_legacy_parser_retains_historical_protection_headers() { + let root = scratch::case("legacy-protect-header"); + let path = root.join("protected.rar"); + let bytes = include_bytes!("fixtures/rar15_40/rar250_protect_head_rr5.rar"); + std::fs::write(&path, bytes).unwrap(); + let memory = rar15_40::Archive::parse(bytes).unwrap(); + let seekable = rar15_40::Archive::parse_path(&path).unwrap(); + assert!(memory + .blocks + .iter() + .any(|b| matches!(b, rar15_40::Block::Protect(_)))); + assert_eq!(memory.blocks, seekable.blocks); +} + +#[test] +fn source_failure_while_rereading_legacy_main_header_stays_an_io_error() { + struct Reader { + bytes: std::io::Cursor>, + main_reads: usize, + } + impl std::io::Read for Reader { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + if self.bytes.position() == 7 { + self.main_reads += 1; + if self.main_reads == 3 { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "main header read failed", + )); + } + } + std::io::Read::read(&mut self.bytes, output) + } + } + impl std::io::Seek for Reader { + fn seek(&mut self, from: std::io::SeekFrom) -> std::io::Result { + std::io::Seek::seek(&mut self.bytes, from) + } + } + let mut builder = rars::Builder::new(rars::ArchiveVersion::Rar30); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let error = rars::ArchiveReader::read_reader(Reader { + bytes: std::io::Cursor::new(builder.to_bytes().unwrap()), + main_reads: 0, + }) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert!(error.to_string().contains("main header read failed")); +} diff --git a/arca-rar/vendor/rars/tests/reader_source.rs b/arca-rar/vendor/rars/tests/reader_source.rs new file mode 100644 index 00000000..60eb7554 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_source.rs @@ -0,0 +1,222 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder}; +#[path = "support/scratch.rs"] +mod scratch; +use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, Mutex, +}; + +struct Tracked { + data: Cursor>, + reads: Arc, + dropped: Arc, + fail: Arc, +} +impl Read for Tracked { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.fail.load(Ordering::Relaxed) { + return Err(io::ErrorKind::PermissionDenied.into()); + } + // Exercise repeated short reads through signature/header/payload adapters. + let len = bytes.len().min(137); + let count = self.data.read(&mut bytes[..len])?; + self.reads.fetch_add(count, Ordering::Relaxed); + Ok(count) + } +} +impl Seek for Tracked { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.data.seek(from) + } +} +impl Drop for Tracked { + fn drop(&mut self) { + self.dropped.store(true, Ordering::Relaxed); + } +} +fn tracked(bytes: Vec) -> Tracked { + let mut data = Cursor::new(bytes); + data.set_position(19); + Tracked { + data, + reads: Arc::default(), + dropped: Arc::default(), + fail: Arc::default(), + } +} +struct Capture(Arc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn seekable_sources_cover_families_encryption_sfx_and_parallel_extraction() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for stored in [true, false] { + let encrypted = version != ArchiveVersion::Rar14; + let password = encrypted.then_some(b"secret".as_slice()); + let mut builder = Builder::new(version) + .store(stored) + .password(password.map(Vec::from)) + .header_encryption(matches!( + version, + ArchiveVersion::Rar30 + | ArchiveVersion::Rar40 + | ArchiveVersion::Rar50 + | ArchiveVersion::Rar70 + )); + let mut expected = Vec::new(); + for index in 0..4 { + let data = vec![index as u8; 2048 + index]; + expected.extend_from_slice(&data); + builder + .add_bytes(index.to_string().into_bytes(), data, None, None) + .unwrap(); + } + let mut bytes = vec![0; 31]; + bytes.extend(builder.to_bytes().unwrap()); + let source = tracked(bytes); + let dropped = source.dropped.clone(); + let mut options = ArchiveReadOptions::new(); + options.password = password; + let archive = ArchiveReader::read_reader_with_options(source, options).unwrap(); + let clone = archive.clone(); + drop(archive); + assert!(!dropped.load(Ordering::Relaxed)); + for parallel in [false, true] { + let output = Arc::new(Mutex::new(Vec::new())); + let open = |_: &rars::ExtractedEntryMeta| { + Ok(Box::new(Capture(output.clone())) as Box) + }; + if parallel { + clone.extract_to_parallel_buffered(password, open).unwrap(); + } else { + clone.extract_to(password, open).unwrap(); + } + assert_eq!(*output.lock().unwrap(), expected); + } + drop(clone); + assert!(dropped.load(Ordering::Relaxed)); + } + } +} + +#[test] +fn parsing_skips_large_payloads_and_keeps_the_source_for_later_reads() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let data = vec![42; rars::SFX_SCAN_LIMIT * 2]; + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), data.clone(), None, None) + .unwrap(); + let source = tracked(builder.to_bytes().unwrap()); + let reads = source.reads.clone(); + let fail = source.fail.clone(); + let archive = ArchiveReader::read_reader(source).unwrap(); + assert!(reads.load(Ordering::Relaxed) < data.len()); + if version != ArchiveVersion::Rar14 { + assert!( + reads.load(Ordering::Relaxed) < 1024, + "standard strong-signature parsing reads headers, not the payload scan" + ); + } + assert_eq!(archive.read_member(b"file", None).unwrap().unwrap(), data); + fail.store(true, Ordering::Relaxed); + let error = archive + .extract_to(None, |_| Ok(Box::new(io::sink()))) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + } +} + +#[test] +fn caller_sources_observe_parse_limits_and_precancellation() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), vec![42; 128], None, None) + .unwrap(); + let bytes = builder.to_bytes().unwrap(); + let error = ArchiveReader::read_reader_with_options( + Cursor::new(bytes.clone()), + ArchiveReadOptions::new().with_max_header_count(0), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::ResourceLimit); + let source = tracked(bytes); + let reads = source.reads.clone(); + let token = rars::ReadCancellation::new(); + token.cancel(); + let error = ArchiveReader::read_reader_with_options( + source, + ArchiveReadOptions::new().with_cancellation(&token), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Cancelled); + assert_eq!(reads.load(Ordering::Relaxed), 0); +} + +#[test] +fn split_members_can_read_from_multiple_caller_owned_sources() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true).volume_size(Some(512)); + let data = vec![42; 2048]; + builder + .add_bytes(b"file".to_vec(), data.clone(), None, None) + .unwrap(); + let volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_reader(tracked(bytes)).unwrap()) + .collect(); + assert!(volumes.len() > 1); + let output = Arc::new(Mutex::new(Vec::new())); + rars::extract_volumes_to(&volumes, None, |_| Ok(Box::new(Capture(output.clone())))) + .unwrap(); + assert_eq!(*output.lock().unwrap(), data); + } +} + +#[cfg(unix)] +#[test] +fn owned_file_handle_remains_usable_after_its_path_is_removed() { + let dir = scratch::case("reader-owned-file"); + let path = dir.join("archive.rar"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + std::fs::write(&path, builder.to_bytes().unwrap()).unwrap(); + let archive = ArchiveReader::read_reader(std::fs::File::open(&path).unwrap()).unwrap(); + std::fs::remove_file(path).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); +} diff --git a/arca-rar/vendor/rars/tests/reader_stored_lengths.rs b/arca-rar/vendor/rars/tests/reader_stored_lengths.rs new file mode 100644 index 00000000..ca2598a1 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_stored_lengths.rs @@ -0,0 +1,151 @@ +#![cfg(feature = "write")] + +use rars::{Archive, ArchiveReader, ArchiveVersion, Builder, Error}; +#[path = "support/scratch.rs"] +mod scratch; + +fn stored(version: ArchiveVersion, encrypted: bool, volumes: bool) -> Vec> { + let mut builder = Builder::new(version) + .store(true) + .password((encrypted && !volumes).then(|| b"secret".to_vec())); + if volumes { + builder = builder.volume_size(Some(128)); + } + builder + .add_bytes(b"zero.bin".to_vec(), vec![0; 256], None, None) + .unwrap(); + if volumes { + let mut volumes = builder.build_volumes(None).unwrap(); + if encrypted { + // The writer does not emit old-format encrypted volume sets. Form + // one by splitting the legitimate single-member ciphertext while + // retaining continuous cipher state across the volume boundaries. + let bytes = stored(version, true, false).pop().unwrap(); + let archive = rars::rar13::Archive::parse(&bytes).unwrap(); + let encrypted_data = &bytes[archive.entries[0].packed_range.clone()]; + let mut position = 0; + for volume in &mut volumes { + let archive = rars::rar13::Archive::parse(volume).unwrap(); + let range = archive.entries[0].packed_range.clone(); + let len = range.len(); + volume[range].copy_from_slice(&encrypted_data[position..position + len]); + volume[24] |= 0x04; + position += len; + } + assert_eq!(position, encrypted_data.len()); + } + volumes + } else { + vec![builder.to_bytes().unwrap()] + } +} + +#[test] +fn rar13_stored_members_reject_incorrect_declared_lengths_even_when_checksum_matches() { + for version in [ArchiveVersion::Rar13, ArchiveVersion::Rar14] { + for encrypted in [false, true] { + if encrypted && !cfg!(feature = "encryption") { + continue; + } + let original = stored(version, encrypted, false).pop().unwrap(); + let password = encrypted.then_some(b"secret".as_slice()); + for declared in [0u32, 1, 255, 257] { + let mut malformed = original.clone(); + malformed[11..15].copy_from_slice(&declared.to_le_bytes()); + let archive = ArchiveReader::read_owned(malformed).unwrap(); + assert!( + archive.read_member_at(0, password).is_err(), + "{version:?}, encrypted={encrypted}, declared={declared}" + ); + if let Archive::Rar13(archive) = archive { + let error = archive.entries[0] + .write_to(&archive, password, &mut std::io::sink()) + .unwrap_err(); + assert!(matches!(error.root_cause(), Error::InvalidHeader(_))); + } + } + } + } +} + +#[test] +fn rar13_stored_path_sources_reject_short_successful_copies() { + let scratch = scratch::case("rar13-truncated-stored-source"); + for version in [ArchiveVersion::Rar13, ArchiveVersion::Rar14] { + let original = stored(version, false, false).pop().unwrap(); + let path = scratch.join("archive.rar"); + std::fs::write(&path, &original).unwrap(); + let archive = ArchiveReader::read_path(&path).unwrap(); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len((original.len() - 255) as u64) + .unwrap(); + let error = archive.read_member_at(0, None).unwrap_err(); + assert!(matches!(error.root_cause(), Error::InvalidHeader(_))); + } +} + +#[test] +fn rar13_split_stored_members_verify_reassembled_length() { + for version in [ArchiveVersion::Rar13, ArchiveVersion::Rar14] { + for encrypted in [false, true] { + if encrypted && !cfg!(feature = "encryption") { + continue; + } + let mut bytes = stored(version, encrypted, true); + assert!(bytes.len() > 1); + let valid: Vec<_> = bytes + .iter() + .map(|bytes| ArchiveReader::read(bytes).unwrap()) + .collect(); + let password = encrypted.then_some(b"secret".as_slice()); + assert_eq!( + rars::read_volume_member_at(&valid, 0, password) + .unwrap() + .unwrap(), + vec![0; 256] + ); + let last = bytes.last_mut().unwrap(); + last[11..15].copy_from_slice(&1u32.to_le_bytes()); + let archives: Vec<_> = bytes + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(rars::read_volume_member_at(&archives, 0, password).is_err()); + } + } +} + +#[test] +fn rar13_split_stored_path_sources_reject_truncation() { + let scratch = scratch::case("rar13-truncated-split-stored-source"); + for version in [ArchiveVersion::Rar13, ArchiveVersion::Rar14] { + for encrypted in [false, true] { + if encrypted && !cfg!(feature = "encryption") { + continue; + } + let bytes = stored(version, encrypted, true); + let archives: Vec<_> = bytes + .iter() + .enumerate() + .map(|(index, bytes)| { + let path = scratch.join(format!("part-{index}.rar")); + std::fs::write(&path, bytes).unwrap(); + ArchiveReader::read_path(&path).unwrap() + }) + .collect(); + let index = bytes.len() - 1; + let path = scratch.join(format!("part-{index}.rar")); + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_len((bytes[index].len() - 1) as u64) + .unwrap(); + let password = encrypted.then_some(b"secret".as_slice()); + assert!(rars::read_volume_member_at(&archives, 0, password).is_err()); + } + } +} diff --git a/arca-rar/vendor/rars/tests/reader_verification.rs b/arca-rar/vendor/rars/tests/reader_verification.rs new file mode 100644 index 00000000..c6426870 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_verification.rs @@ -0,0 +1,644 @@ +#![cfg(feature = "write")] + +use rars::codec::rar50::{DecodedChunk, StreamDecodeError, Unpack50Decoder}; +use rars::rar50::{ArchiveEntry, Block, Rar50Writer, WriterOptions}; +use rars::{ + test_volumes_with_options, test_volumes_with_options_and_callback, Archive, ArchiveReadOptions, + ArchiveReader, ArchiveVersion, Builder, EntrySource, Error, ErrorKind, FeatureSet, FilterKind, + FilterPolicy, FilterSpec, ReadCancellation, +}; +use std::io::{self, Cursor, Read, Seek, SeekFrom}; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; + +#[path = "support/scratch.rs"] +mod scratch; + +const DICTIONARY: u64 = 128 * 1024; + +fn entry(name: &[u8], data: Vec) -> ArchiveEntry { + ArchiveEntry::new(name.to_vec(), EntrySource::from_bytes(data)) +} + +fn options() -> WriterOptions { + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(3) + .with_dictionary_size(DICTIONARY) +} + +fn encoded(data: Vec, policy: FilterPolicy) -> Vec { + Rar50Writer::new(options()) + .entry(entry(b"payload", data)) + .filter_policy(policy) + .finish() + .unwrap() +} + +fn late_filtered() -> (Vec, Vec) { + let mut data = b"abc" + .iter() + .copied() + .cycle() + .take(300 * 1024) + .collect::>(); + data.extend_from_slice(&b"\xe8\0\0\0\0filtered bytes\n".repeat(2048)); + let policy = FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8, 300 * 1024..data.len())); + (encoded(data.clone(), policy), data) +} + +fn cause(error: &Error) -> &Error { + match error { + Error::AtEntry { source, .. } + | Error::AtArchiveOffset { source, .. } + | Error::InVolume { source, .. } => cause(source), + error => error, + } +} + +#[test] +fn verification_avoids_a_second_output_buffer_and_preserves_decode_routing() { + let data = b"abc" + .iter() + .copied() + .cycle() + .take(4 * 1024 * 1024) + .collect::>(); + let archive = ArchiveReader::read_owned(encoded(data, FilterPolicy::None)).unwrap(); + let budget = ArchiveReadOptions::new().with_max_reader_workspace_bytes(1024 * 1024); + assert_eq!( + archive.test_with_options(budget).unwrap_err().kind(), + ErrorKind::ResourceLimit + ); + // Leave room for geometric output growth, but not a second 4 MiB + // publication buffer retained beside the decoded bytes. + let buffered_budget = + ArchiveReadOptions::new().with_max_reader_workspace_bytes(7 * 1024 * 1024); + archive.test_with_options(buffered_budget).unwrap(); + assert_eq!( + archive + .extract_to_parallel_buffered_with_options(buffered_budget, |_| Ok( + Box::new(io::sink()) + )) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit, + "ordinary publication retains a second output buffer" + ); + archive + .test_with_options(budget.with_rar50_buffered_decode_limit(0)) + .unwrap(); + assert_eq!( + archive + .test_with_options(budget.with_rar50_buffered_decode_limit(u64::MAX)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!( + archive + .test_with_options(budget.with_rar50_dictionary_size_limit(DICTIONARY - 1)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); + assert_eq!( + archive + .test_with_options(ArchiveReadOptions::new().with_max_reader_workspace_bytes(64 * 1024)) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); +} + +#[test] +fn filtered_verification_charges_total_output_once_and_preserves_explicit_routing() { + let (bytes, data) = late_filtered(); + let archive = ArchiveReader::read_owned(bytes.clone()).unwrap(); + let Archive::Rar50Plus(typed) = &archive else { + unreachable!() + }; + let file = typed.files().next().unwrap(); + let info = file.decoded_compression_info().unwrap(); + let mut emitted = 0; + let result = Unpack50Decoder::new().decode_member_from_reader_with_dictionary_to_sink( + &mut &bytes[file.block.data_range.clone()], + info.algorithm_version, + data.len(), + info.dictionary_size as usize, + false, + |chunk| { + emitted += match chunk { + DecodedChunk::Bytes(bytes) => bytes.len(), + DecodedChunk::Repeated { len, .. } => len, + }; + Ok::<(), io::Error>(()) + }, + ); + assert!(matches!(result, Err(StreamDecodeError::FilteredMember))); + assert!( + emitted >= 64 * 1024, + "filter must follow a real streaming flush" + ); + let exact = ArchiveReadOptions::new() + .with_max_member_output_bytes(data.len() as u64) + .with_max_total_output_bytes(data.len() as u64); + let mut seen = Vec::new(); + archive + .test_with_options_and_callback(exact, |meta| { + seen.push(meta.name.clone()); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, [b"payload".to_vec()]); + assert_eq!(archive.read_member_at(0, None).unwrap().unwrap(), data); + assert!(matches!( + cause( + &archive + .test_with_options(exact.with_max_total_output_bytes(data.len() as u64 - 1)) + .unwrap_err() + ), + Error::TotalOutputLimitExceeded { .. } + )); + assert!(matches!( + cause( + &archive + .test_with_options(exact.with_rar50_buffered_decode_limit(0)) + .unwrap_err() + ), + Error::Rar50BufferedDecodeLimitExceeded { limit: 0, .. } + )); + archive + .test_with_options(exact.with_rar50_buffered_decode_limit(data.len() as u64)) + .unwrap(); +} + +struct ProbeReader { + data: Cursor>, + start: u64, + armed: Arc, + attempts: Arc, + fail: bool, + cancel_on_read: Option, +} +impl Read for ProbeReader { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.armed.load(Ordering::Relaxed) + && self.data.position() == self.start + && !bytes.is_empty() + { + let attempt = self.attempts.fetch_add(1, Ordering::Relaxed); + if self.fail { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "source refused", + )); + } + if attempt == 0 { + if let Some(token) = &self.cancel_on_read { + token.cancel(); + } + } + } + self.data.read(bytes) + } +} +impl Seek for ProbeReader { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.data.seek(from) + } +} +fn reader( + bytes: Vec, + fail: bool, + cancel_on_read: Option, +) -> (Archive, Arc) { + let archive = ArchiveReader::read(&bytes).unwrap(); + let Archive::Rar50Plus(typed) = archive else { + unreachable!() + }; + let start = typed.files().next().unwrap().block.data_range.start as u64; + let armed = Arc::new(AtomicBool::new(false)); + let attempts = Arc::new(AtomicUsize::new(0)); + let source = ProbeReader { + data: Cursor::new(bytes), + start, + armed: armed.clone(), + attempts: attempts.clone(), + fail, + cancel_on_read, + }; + let archive = ArchiveReader::read_reader(source).unwrap(); + armed.store(true, Ordering::Relaxed); + (archive, attempts) +} + +#[test] +fn admission_precedes_probe_and_source_errors_are_never_retried() { + let bytes = encoded(b"abc".repeat(128 * 1024), FilterPolicy::None); + let (archive, attempts) = reader(bytes.clone(), true, None); + let error = archive + .test_with_options(ArchiveReadOptions::new().with_max_member_output_bytes(1)) + .unwrap_err(); + assert!(matches!( + cause(&error), + Error::MemberOutputLimitExceeded { .. } + )); + assert_eq!(attempts.load(Ordering::Relaxed), 0); + assert_eq!(archive.test(None).unwrap_err().kind(), ErrorKind::Io); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + let mut archive = ArchiveReader::read_owned(bytes).unwrap(); + let Archive::Rar50Plus(typed) = &mut archive else { + unreachable!() + }; + for block in &mut typed.blocks { + if let Block::File(file) = block { + file.hash = None; + file.data_crc32 = Some(0); + } + } + let mut seen = 0; + let error = archive + .test_with_options_and_callback(ArchiveReadOptions::new(), |_| { + seen += 1; + Ok(()) + }) + .unwrap_err(); + assert!(matches!(cause(&error), Error::Crc32Mismatch { .. })); + assert_eq!( + seen, 0, + "failed parallel verification publishes no metadata" + ); +} + +#[test] +fn cancellation_during_payload_read_is_not_hidden_or_reported_as_success() { + let (bytes, _) = late_filtered(); + let token = ReadCancellation::new(); + let (archive, attempts) = reader(bytes, false, Some(token.clone())); + let mut seen = 0; + let error = archive + .test_with_options_and_callback(ArchiveReadOptions::new().with_cancellation(&token), |_| { + seen += 1; + Ok(()) + }) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Cancelled); + assert_eq!( + attempts.load(Ordering::Relaxed), + 1, + "default buffered verification reads the packed stream once" + ); + assert_eq!(seen, 0); +} + +#[test] +fn filtered_member_above_the_default_buffer_limit_is_not_replayed() { + let (bytes, _) = late_filtered(); + let (mut archive, attempts) = reader(bytes, false, None); + let required = 512 * 1024 * 1024 + 1; + let Archive::Rar50Plus(typed) = &mut archive else { + unreachable!() + }; + for block in &mut typed.blocks { + if let Block::File(file) = block { + file.unpacked_size = required; + } + } + let error = archive.test(None).unwrap_err(); + assert!( + matches!(cause(&error), Error::Rar50BufferedDecodeLimitExceeded { limit: 536_870_912, required: value } if *value == required) + ); + assert_eq!(attempts.load(Ordering::Relaxed), 1); +} + +#[cfg(feature = "encryption")] +#[test] +fn filtered_encrypted_verification_detects_corruption() { + let (_, data) = late_filtered(); + let policy = FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8, 300 * 1024..data.len())); + let bytes = Rar50Writer::new(options()) + .entry(entry(b"payload", data.clone()).with_password(b"secret")) + .filter_policy(policy) + .finish() + .unwrap(); + let mut archive = ArchiveReader::read_owned(bytes).unwrap(); + let exact = + ArchiveReadOptions::with_password(b"secret").with_max_total_output_bytes(data.len() as u64); + archive.test_with_options(exact).unwrap(); + assert_eq!( + archive.read_member_at(0, Some(b"secret")).unwrap().unwrap(), + data + ); + let Archive::Rar50Plus(typed) = &mut archive else { + unreachable!() + }; + for block in &mut typed.blocks { + if let Block::File(file) = block { + file.hash = None; + file.data_crc32 = Some(0); + } + } + assert!(matches!( + cause(&archive.test_with_options(exact).unwrap_err()), + Error::Crc32Mismatch { .. } + )); +} + +#[test] +fn callbacks_are_ordered_and_callback_cancellation_and_errors_abort() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(false); + builder + .add_bytes(b"first".to_vec(), b"first".repeat(1000), None, None) + .unwrap(); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"last".to_vec(), vec![], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let mut seen = Vec::new(); + archive + .test_with_options_and_callback(ArchiveReadOptions::new(), |meta| { + seen.push((meta.name.clone(), meta.is_directory)); + Ok(()) + }) + .unwrap(); + assert_eq!( + seen, + [ + (b"first".to_vec(), false), + (b"dir".to_vec(), true), + (b"last".to_vec(), false) + ] + ); + let token = ReadCancellation::new(); + let mut seen = 0; + assert_eq!( + archive + .test_with_options_and_callback( + ArchiveReadOptions::new().with_cancellation(&token), + |_| { + seen += 1; + token.cancel(); + Ok(()) + } + ) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + assert_eq!(seen, 1); + assert_eq!( + archive + .test_with_options_and_callback(ArchiveReadOptions::new(), |_| Err( + Error::InvalidArgument("callback refused") + )) + .unwrap_err(), + Error::InvalidArgument("callback refused") + ); +} + +#[test] +fn legacy_callbacks_and_volume_api_preserve_order_and_output_limits() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"first".to_vec(), b"first".to_vec(), None, None) + .unwrap(); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"last".to_vec(), b"last".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let exact = ArchiveReadOptions::new().with_max_total_output_bytes(9); + let mut seen = Vec::new(); + archive + .test_with_options_and_callback(exact, |meta| { + seen.push(meta.name.clone()); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, [b"first".to_vec(), b"dir".to_vec(), b"last".to_vec()]); + let mut volume_seen = Vec::new(); + test_volumes_with_options_and_callback(std::slice::from_ref(&archive), exact, |meta| { + volume_seen.push(meta.name.clone()); + Ok(()) + }) + .unwrap(); + assert_eq!(volume_seen, seen); + let mut seen = Vec::new(); + let error = archive + .test_with_options_and_callback(exact.with_max_total_output_bytes(8), |meta| { + seen.push(meta.name.clone()); + Ok(()) + }) + .unwrap_err(); + assert!(matches!( + cause(&error), + Error::TotalOutputLimitExceeded { .. } + )); + assert_eq!(seen, [b"first".to_vec(), b"dir".to_vec()]); + } +} + +#[test] +fn solid_and_encrypted_verification_preserves_history_and_exact_output_budgets() { + for encrypted in [false, true] { + if encrypted && !cfg!(feature = "encryption") { + continue; + } + let mut builder = Builder::new(ArchiveVersion::Rar50).store(false).solid(true); + if encrypted { + builder = builder.password(Some(b"secret".to_vec())); + } + let data = b"shared solid dictionary bytes".repeat(8192); + for name in [b"first".to_vec(), b"second".to_vec()] { + builder.add_bytes(name, data.clone(), None, None).unwrap(); + } + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let mut options = + ArchiveReadOptions::new().with_max_total_output_bytes((data.len() * 2) as u64); + options.password = encrypted.then_some(b"secret".as_slice()); + let mut seen = Vec::new(); + archive + .test_with_options_and_callback(options, |meta| { + seen.push(meta.name.clone()); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, [b"first".to_vec(), b"second".to_vec()]); + archive + .test_with_options(options.with_rar50_buffered_decode_limit(0)) + .unwrap(); + if encrypted { + assert!(archive + .test_with_options(ArchiveReadOptions::with_password(b"wrong")) + .is_err()); + } + } +} + +#[test] +fn split_filtered_encrypted_and_solid_members_use_verification_without_duplicate_callbacks() { + for (prefix, count, padded, password) in [ + ("multivol", 3, false, None), + ("encrypted_multivol", 3, false, Some(b"password".as_slice())), + ("solid_multivol", 6, true, None), + ] { + if password.is_some() && !cfg!(feature = "encryption") { + continue; + } + let volumes: Vec<_> = (1..=count) + .map(|index| { + let number = if padded { + format!("{index:02}") + } else { + index.to_string() + }; + ArchiveReader::read_path_with_options( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/rar50") + .join(format!("{prefix}.part{number}.rar")), + ArchiveReadOptions::with_optional_password(password), + ) + .unwrap() + }) + .collect(); + let mut expected = Vec::new(); + rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::with_optional_password(password) + .with_rar50_buffered_decode_limit(u64::MAX), + |meta| { + expected.push(meta.name.clone()); + Ok(Box::new(io::sink())) + }, + ) + .unwrap(); + let mut seen = Vec::new(); + test_volumes_with_options_and_callback( + &volumes, + ArchiveReadOptions::with_optional_password(password), + |meta| { + seen.push(meta.name.clone()); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(seen, expected, "{prefix}"); + } + let (bytes, data) = late_filtered(); + let parsed = ArchiveReader::read_owned(bytes).unwrap(); + // A real filtered archive also exercises the volume API with one physical + // volume; fragmented filters are covered below. + test_volumes_with_options( + &[parsed], + ArchiveReadOptions::new().with_max_total_output_bytes(data.len() as u64), + ) + .unwrap(); +} + +#[test] +fn fragmented_filtered_verification_charges_once_and_preserves_scratch_limits() { + let (_, data) = late_filtered(); + let policy = FilterPolicy::Explicit(FilterSpec::range(FilterKind::E8, 300 * 1024..data.len())); + let mut sink = rars::rar50::CollectedVolumes::new(); + let root = scratch::case("verification-filtered-splits"); + rars::rar50::write_streaming_volumes_to( + &[entry(b"payload", data.clone())], + options(), + rars::rar50::ArchiveExtras::default().with_filter_policy(policy), + 256, + &mut sink, + &rars::WriterResources::new(64 * 1024 * 1024).with_temp_dir(root.to_path_buf()), + ) + .unwrap(); + let volumes = sink + .take() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect::>(); + assert!(volumes.len() > 1); + let exact = ArchiveReadOptions::new().with_max_total_output_bytes(data.len() as u64); + let mut seen = Vec::new(); + test_volumes_with_options_and_callback(&volumes, exact, |meta| { + seen.push(meta.name.clone()); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, [b"payload".to_vec()]); + assert!(matches!( + cause( + &test_volumes_with_options(&volumes, exact.with_rar50_buffered_decode_limit(0)) + .unwrap_err() + ), + Error::Rar50BufferedDecodeLimitExceeded { .. } + )); + let scratch = + rars::Rar50Scratch::new(&*root, 2 * 1024 * 1024).with_filter_memory_limit(128 * 1024); + test_volumes_with_options( + &volumes, + exact + .with_rar50_buffered_decode_limit(0) + .with_rar50_scratch(&scratch), + ) + .unwrap(); + let small = rars::Rar50Scratch::new(&*root, 1).with_filter_memory_limit(128 * 1024); + assert_eq!( + test_volumes_with_options( + &volumes, + exact + .with_rar50_buffered_decode_limit(0) + .with_rar50_scratch(&small) + ) + .unwrap_err() + .kind(), + ErrorKind::ResourceLimit + ); +} + +#[test] +fn split_callbacks_preserve_cancellation_errors_and_incomplete_volume_detection() { + let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/rar50"); + let volumes = (1..=3) + .map(|index| { + ArchiveReader::read_path(fixture.join(format!("multivol.part{index}.rar"))).unwrap() + }) + .collect::>(); + let token = ReadCancellation::new(); + let mut seen = 0; + let error = test_volumes_with_options_and_callback( + &volumes, + ArchiveReadOptions::new().with_cancellation(&token), + |_| { + seen += 1; + token.cancel(); + Ok(()) + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Cancelled); + assert_eq!(seen, 1); + assert_eq!( + test_volumes_with_options_and_callback(&volumes, ArchiveReadOptions::new(), |_| Err( + Error::InvalidArgument("callback refused") + )) + .unwrap_err(), + Error::InvalidArgument("callback refused") + ); + let error = test_volumes_with_options(&volumes[..2], ArchiveReadOptions::new()).unwrap_err(); + assert!(matches!( + cause(&error), + Error::InvalidHeader("RAR 5 split entry is incomplete") + )); + let error = test_volumes_with_options(&volumes[1..], ArchiveReadOptions::new()).unwrap_err(); + assert!(matches!( + cause(&error), + Error::InvalidHeader("RAR 5 split entry is missing its first part") + )); +} diff --git a/arca-rar/vendor/rars/tests/reader_workspace.rs b/arca-rar/vendor/rars/tests/reader_workspace.rs new file mode 100644 index 00000000..ff4a7236 --- /dev/null +++ b/arca-rar/vendor/rars/tests/reader_workspace.rs @@ -0,0 +1,309 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, ErrorKind}; +use std::{cell::RefCell, io::Write, rc::Rc}; + +struct Capture(Rc>>); +impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn workspace_policy_covers_every_codec_and_parallel_publication() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let data = b"reader workspace accounting".repeat(64); + let mut builder = Builder::new(version).store(false); + for name in [b"first".to_vec(), b"second".to_vec()] { + builder.add_bytes(name, data.clone(), None, None).unwrap(); + } + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for parallel in [false, true] { + for limit in [1, 64 << 20] { + let output = Rc::new(RefCell::new(Vec::new())); + let options = ArchiveReadOptions::new().with_max_reader_workspace_bytes(limit); + let open = |_: &rars::ExtractedEntryMeta| { + Ok(Box::new(Capture(output.clone())) as Box) + }; + let result = if parallel { + archive.extract_to_parallel_buffered_with_options(options, open) + } else { + archive.extract_to_with_options(options, open) + }; + if limit == 1 { + assert_eq!( + result.unwrap_err().kind(), + ErrorKind::ResourceLimit, + "{version:?}" + ); + assert!(output.borrow().is_empty()); + } else { + result.unwrap(); + assert_eq!(*output.borrow(), data.repeat(2), "{version:?}"); + } + } + } + } +} + +#[test] +fn stored_sequential_payloads_do_not_require_a_dictionary() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar20, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"file".to_vec(), vec![42; 1024], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + archive + .extract_to_with_options( + ArchiveReadOptions::new().with_max_reader_workspace_bytes(0), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap(); + } +} + +#[test] +fn limited_parallel_legacy_extraction_handles_directories_and_stored_files() { + for version in [ArchiveVersion::Rar15, ArchiveVersion::Rar29] { + let mut builder = Builder::new(version).store(true); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let output = Rc::new(RefCell::new(Vec::new())); + let mut seen = Vec::new(); + archive + .extract_to_parallel_buffered_with_options( + ArchiveReadOptions::new().with_max_reader_workspace_bytes(1 << 20), + |meta| { + seen.push((meta.name.clone(), meta.is_directory)); + Ok(Box::new(Capture(output.clone())) as Box) + }, + ) + .unwrap(); + assert_eq!(seen, [(b"dir".to_vec(), true), (b"file".to_vec(), false)]); + assert_eq!(*output.borrow(), b"payload"); + } +} + +#[test] +fn workspace_refusal_is_not_remapped_to_a_password_or_integrity_failure() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version) + .store(false) + .password(Some(b"pw".to_vec())); + builder + .add_bytes( + b"file".to_vec(), + b"encrypted payload".repeat(32), + None, + None, + ) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for limit in [1, 64 << 20] { + let options = + ArchiveReadOptions::with_password(b"pw").with_max_reader_workspace_bytes(limit); + let result = + archive.extract_to_with_options(options, |_| Ok(Box::new(std::io::sink()))); + if limit == 1 { + assert_eq!( + result.unwrap_err().kind(), + ErrorKind::ResourceLimit, + "{version:?}" + ); + } else { + result.unwrap(); + } + } + } +} + +#[test] +fn workspace_policy_applies_to_compressed_comments_without_changing_their_checksums() { + let comment = b"compressed archive comment".repeat(32); + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version) + .store(false) + .comment(Some(comment.clone())); + builder + .add_bytes(b"file".to_vec(), vec![42; 256], None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let actual = archive + .comment_with_options( + ArchiveReadOptions::new().with_max_reader_workspace_bytes(64 << 20), + ) + .unwrap(); + assert_eq!(actual.as_deref(), Some(&comment[..]), "{version:?}"); + let error = archive + .comment_with_options(ArchiveReadOptions::new().with_max_reader_workspace_bytes(1)) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ResourceLimit, "{version:?}"); + } +} + +#[test] +fn split_descriptor_and_cursor_accounting_is_shared_across_volumes() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar20, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true).volume_size(Some(512)); + let data: Vec<_> = (0..2048).map(|i| i as u8).collect(); + builder + .add_bytes(b"file".to_vec(), data.clone(), None, None) + .unwrap(); + let volumes: Vec<_> = builder + .build_volumes(None) + .unwrap() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(volumes.len() > 1); + for limit in [1, 64 << 20] { + let output = Rc::new(RefCell::new(Vec::new())); + let result = rars::extract_volumes_to_with_options( + &volumes, + ArchiveReadOptions::new().with_max_reader_workspace_bytes(limit), + |_| Ok(Box::new(Capture(output.clone()))), + ); + if limit == 1 { + assert_eq!( + result.unwrap_err().kind(), + ErrorKind::ResourceLimit, + "{version:?}" + ); + assert!(output.borrow().is_empty()); + } else { + result.unwrap(); + assert_eq!(*output.borrow(), data); + } + } + } +} + +#[test] +fn historical_audio_vm_ppmd_and_solid_archives_decode_under_the_policy() { + for bytes in [ + &include_bytes!("fixtures/rar13/BIG80K.RAR")[..], + &include_bytes!("fixtures/rar15_40/rar250/AUDIO.RAR")[..], + &include_bytes!("fixtures/rar15_40/rar250/SOLID.RAR")[..], + &include_bytes!("fixtures/rar15_40/rar300/rarvm_audio_stereo_rar300.rar")[..], + &include_bytes!("fixtures/rar15_40/ppmd/ppmd_solid_rar300.rar")[..], + ] { + let archive = ArchiveReader::read(bytes).unwrap(); + let expected = Rc::new(RefCell::new(Vec::new())); + archive + .extract_to_with_options(ArchiveReadOptions::new(), |_| { + Ok(Box::new(Capture(expected.clone()))) + }) + .unwrap(); + let actual = Rc::new(RefCell::new(Vec::new())); + archive + .extract_to_with_options( + ArchiveReadOptions::new().with_max_reader_workspace_bytes(64 << 20), + |_| Ok(Box::new(Capture(actual.clone()))), + ) + .unwrap(); + assert_eq!(*actual.borrow(), *expected.borrow()); + } +} + +#[test] +fn solid_rar5_streaming_and_selected_reads_share_the_operation_quota() { + let data = b"solid workspace payload".repeat(128); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(false).solid(true); + for name in [b"first".to_vec(), b"second".to_vec()] { + builder.add_bytes(name, data.clone(), None, None).unwrap(); + } + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + for limit in [1, 64 << 20] { + let options = ArchiveReadOptions::new() + .with_max_reader_workspace_bytes(limit) + .with_rar50_buffered_decode_limit(0); + let result = archive.read_member_with_options(b"second", options); + if limit == 1 { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ResourceLimit); + } else { + assert_eq!(result.unwrap(), Some(data.clone())); + } + } +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +#[path = "support/scratch.rs"] +mod scratch; + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +#[test] +fn scratch_filter_allocations_obey_the_shared_quota_and_clean_up() { + use rars::rar50::{ArchiveEntry, Rar50Writer, WriterOptions}; + use rars::{EntrySource, FeatureSet, FilterKind, FilterPolicy, FilterSpec, Rar50Scratch}; + let data = b"Delta scratch workspace".repeat(64); + let bytes = Rar50Writer::new( + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(1), + ) + .entry(ArchiveEntry::new( + b"file".to_vec(), + EntrySource::from_bytes(data.clone()), + )) + .filter_policy(FilterPolicy::Explicit(FilterSpec::whole( + FilterKind::Delta { channels: 2 }, + ))) + .finish() + .unwrap(); + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let dir = scratch::case("reader-workspace-public-scratch"); + let policy = Rar50Scratch::new(&*dir, 65536); + for limit in [1, 64 << 20] { + let output = Rc::new(RefCell::new(Vec::new())); + let options = ArchiveReadOptions::new() + .with_max_reader_workspace_bytes(limit) + .with_rar50_buffered_decode_limit(1) + .with_rar50_scratch(&policy); + let result = archive.extract_to_parallel_buffered_with_options(options, |_| { + Ok(Box::new(Capture(output.clone()))) + }); + if limit == 1 { + assert_eq!(result.unwrap_err().kind(), ErrorKind::ResourceLimit); + assert!(output.borrow().is_empty()); + } else { + result.unwrap(); + assert_eq!(*output.borrow(), data); + } + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 0); + } +} diff --git a/arca-rar/vendor/rars/tests/repair_cancellation.rs b/arca-rar/vendor/rars/tests/repair_cancellation.rs new file mode 100644 index 00000000..437e8146 --- /dev/null +++ b/arca-rar/vendor/rars/tests/repair_cancellation.rs @@ -0,0 +1,107 @@ +#![cfg(all(feature = "write", feature = "recovery"))] + +use rars::{ + ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, ErrorKind, ReadCancellation, +}; +use std::{ + io::{self, Cursor, Read, Seek, SeekFrom}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +#[path = "support/scratch.rs"] +mod scratch; + +struct CancelOnRead { + input: Cursor>, + armed: Arc, + token: ReadCancellation, +} +impl Read for CancelOnRead { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.armed.load(Ordering::Relaxed) { + self.token.cancel(); + } + self.input.read(buffer) + } +} +impl Seek for CancelOnRead { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.input.seek(from) + } +} + +#[test] +fn cancellation_during_recovery_source_reads_survives_each_format_adapter() { + for version in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar40, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true).recovery_percent(Some(10)); + builder + .add_bytes(b"file".to_vec(), vec![42; 20_000], None, None) + .unwrap(); + let bytes = match version { + ArchiveVersion::Rar20 => { + include_bytes!("fixtures/rar15_40/rar250_protect_head_rr5.rar").to_vec() + } + ArchiveVersion::Rar40 => { + include_bytes!("fixtures/rar15_40/rar300/with_compressed_recovery_rar300.rar") + .to_vec() + } + _ => builder.to_bytes().unwrap(), + }; + let token = ReadCancellation::new(); + let armed = Arc::new(AtomicBool::new(false)); + let archive = ArchiveReader::read_reader(CancelOnRead { + input: Cursor::new(bytes), + armed: armed.clone(), + token: token.clone(), + }) + .unwrap(); + armed.store(true, Ordering::Relaxed); + let options = ArchiveReadOptions::new().with_cancellation(&token); + assert_eq!( + archive + .repair_recovery_with_options(options) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + armed.store(false, Ordering::Relaxed); + let repaired = archive.repair_recovery_with_report(None).unwrap(); + assert!(!repaired.report.changed); + } +} + +#[test] +fn failed_or_cancelled_publication_keeps_the_destination_and_removes_staging() { + let dir = scratch::case("repair-publication"); + let result = rars::RecoveryRepairResult { + data: b"repaired".to_vec(), + report: Default::default(), + }; + let path = dir.join("existing"); + std::fs::write(&path, b"keep").unwrap(); + let token = ReadCancellation::new(); + token.cancel(); + assert_eq!( + result + .write_to_path(&path, Some(&token)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + assert_eq!(std::fs::read(&path).unwrap(), b"keep"); + let directory = dir.join("directory"); + std::fs::create_dir(&directory).unwrap(); + assert!(result.write_to_path(&directory, None).is_err()); + assert!(directory.is_dir()); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 2); + result.write_to_path(&path, None).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"repaired"); + assert_eq!(std::fs::read_dir(&*dir).unwrap().count(), 2); +} diff --git a/arca-rar/vendor/rars/tests/rewrite_settings.rs b/arca-rar/vendor/rars/tests/rewrite_settings.rs new file mode 100644 index 00000000..0e141ec7 --- /dev/null +++ b/arca-rar/vendor/rars/tests/rewrite_settings.rs @@ -0,0 +1,933 @@ +#![cfg(feature = "write")] + +use rars::{ArchiveReader, ArchiveVersion, Builder}; + +#[test] +fn preserving_builder_accepts_native_unix_directories() { + for version in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_directory(b"directory".to_vec(), None, Some(0o750)) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!( + source.rewrite_preservation_issues().is_empty(), + "{version:?}: {:?}", + source.rewrite_preservation_issues() + ); + + let mut preserving = source.preserving_builder(None).unwrap(); + preserving + .add_directory(b"renamed".to_vec(), None, Some(0o750)) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + assert!(output.rewrite_preservation_issues().is_empty()); + let directory = output.members().next().unwrap(); + assert!(directory.meta.is_directory); + assert_eq!(directory.meta.attr_source(), rars::AttrSource::Unix); + assert_eq!(directory.meta.file_attr & 0o170777, 0o040750); + } +} + +#[test] +fn preserving_builder_refuses_unix_device_entries() { + for version in [ + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_bytes(b"device".to_vec(), Vec::new(), None, Some(0o020600)) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let member = source.members().next().unwrap(); + assert_eq!(member.meta.attr_source(), rars::AttrSource::Unix); + assert!(!member.meta.is_directory); + assert!( + source + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("special entry type or directory contents")), + "{version:?}: attr={:o}, issues={:?}", + member.meta.file_attr, + source.rewrite_preservation_issues() + ); + assert!(source.preserving_builder(None).is_err()); + } +} + +#[test] +fn preserving_builder_refuses_directory_flag_with_unix_device_mode() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_directory(b"directory".to_vec(), None, Some(0o750)) + .unwrap(); + let mut source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let rars::Archive::Rar50Plus(archive) = &mut source else { + unreachable!() + }; + let file = archive + .blocks + .iter_mut() + .find_map(|block| match block { + rars::rar50::Block::File(file) => Some(file), + _ => None, + }) + .unwrap(); + file.attributes = 0o020600; + + let meta = source.members().next().unwrap().meta; + assert!(meta.is_directory); + assert_eq!(meta.attr_source(), rars::AttrSource::Unix); + assert!(source + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("special entry type or directory contents"))); + assert!(source.preserving_builder(None).is_err()); +} + +#[test] +fn preserving_builder_accepts_rar13_family_and_rar7_sources() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar70, + ] { + let mut builder = Builder::new(version).store(false); + builder + .add_bytes( + b"source".to_vec(), + b"payload payload payload".repeat(8), + None, + None, + ) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!( + source.rewrite_preservation_issues().is_empty(), + "{version:?}: {:?}", + source.rewrite_preservation_issues() + ); + let mut preserving = source.preserving_builder(None).unwrap(); + preserving + .add_bytes( + b"source".to_vec(), + b"payload payload payload".repeat(8), + None, + None, + ) + .unwrap(); + preserving + .add_bytes(b"added".to_vec(), b"new payload".to_vec(), None, None) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + assert!(output.rewrite_preservation_issues().is_empty()); + assert_eq!( + output.read_member_at(0, None).unwrap().unwrap(), + b"payload payload payload".repeat(8) + ); + assert_eq!( + output.read_member_at(1, None).unwrap().unwrap(), + b"new payload" + ); + } +} + +#[test] +fn preserving_builder_keeps_rar20_unpacker_26() { + let payload = b"older RAR 2.x payload ".repeat(64); + let mut builder = Builder::new(ArchiveVersion::Rar20).store(true); + builder + .add_bytes(b"source".to_vec(), payload.clone(), None, None) + .unwrap(); + let mut bytes = builder.to_bytes().unwrap(); + let original = ArchiveReader::read_owned(bytes.clone()).unwrap(); + let rars::Archive::Rar15To40(original) = original else { + unreachable!() + }; + let file = original.files().next().unwrap(); + let start = file.block.offset; + let end = start + usize::from(file.block.head_size); + bytes[start + 24] = 26; + let crc = (rars::crc32::crc32(&bytes[start + 2..end]) as u16).to_le_bytes(); + bytes[start..start + 2].copy_from_slice(&crc); + + let source = ArchiveReader::read_owned(bytes).unwrap(); + assert!(source.rewrite_preservation_issues().is_empty()); + assert_eq!(source.read_member_at(0, None).unwrap().unwrap(), payload); + let mut preserving = source.preserving_builder(None).unwrap(); + preserving + .add_bytes(b"source".to_vec(), payload.clone(), None, None) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + let rars::Archive::Rar15To40(ref archive) = output else { + unreachable!() + }; + assert_eq!(archive.files().next().unwrap().unp_ver, 26); + assert_eq!(output.read_member_at(0, None).unwrap().unwrap(), payload); +} + +#[test] +fn retained_legacy_comment_metadata_requires_the_comment_to_be_copied() { + let mut builder = Builder::new(ArchiveVersion::Rar30) + .store(true) + .comment(Some(b"archive note".to_vec())); + builder + .add_bytes(b"source".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(source.rewrite_preservation_issues().is_empty()); + + let mut preserving = source.preserving_builder(None).unwrap(); + preserving + .add_bytes(b"source".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + assert!(matches!( + preserving.to_bytes(), + Err(rars::Error::InvalidArgument( + "retained archive comment metadata requires a RAR3/4 archive comment" + )) + )); + let output = ArchiveReader::read_owned( + preserving + .comment(Some(b"archive note".to_vec())) + .to_bytes() + .unwrap(), + ) + .unwrap(); + assert_eq!( + output.comment(None).unwrap(), + Some(b"archive note".to_vec()) + ); +} + +#[test] +fn preserving_builder_requires_password_for_encrypted_legacy_archive_comment() { + let mut builder = Builder::new(ArchiveVersion::Rar30) + .store(true) + .comment(Some(b"private note".to_vec())) + .archive_comment_password(Some(b"secret".to_vec())); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(source.rewrite_preservation_issues().is_empty()); + assert!(matches!( + source.preserving_builder(None), + Err(rars::Error::NeedPassword) + )); + + let mut preserving = source + .preserving_builder(Some(b"secret")) + .unwrap() + .store(true) + .comment(Some(b"private note".to_vec())); + preserving + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + preserving + .set_entry_encryption(b"file", None, None) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + assert!(matches!( + output.comment(None), + Err(rars::Error::NeedPassword) + )); + assert_eq!( + output.comment(Some(b"secret")).unwrap(), + Some(b"private note".to_vec()) + ); + assert_eq!( + output.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); +} + +#[test] +fn preserving_builder_keeps_rar7_compression_version() { + use rars::{rar50, EntrySource, FeatureSet}; + + let payload = b"RAR7 compression payload ".repeat(256); + let entry = rar50::ArchiveEntry::new( + b"source".to_vec(), + EntrySource::from_bytes(std::sync::Arc::<[u8]>::from(payload.clone())), + ); + let bytes = rar50::Rar50Writer::new( + rar50::WriterOptions::new(ArchiveVersion::Rar70, FeatureSet::default()) + .with_dictionary_size(192 * 1024), + ) + .entry(entry) + .finish() + .unwrap(); + let source = ArchiveReader::read_owned(bytes).unwrap(); + let rars::Archive::Rar50Plus(ref archive) = source else { + unreachable!() + }; + assert_eq!(archive.files().next().unwrap().compression_info & 0x3f, 1); + let mut preserving = source.preserving_builder(None).unwrap(); + assert_eq!(preserving.format(), ArchiveVersion::Rar70); + preserving + .add_bytes(b"source".to_vec(), payload.clone(), None, None) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + let rars::Archive::Rar50Plus(ref archive) = output else { + unreachable!() + }; + assert_eq!(archive.files().next().unwrap().compression_info & 0x3f, 1); + assert_eq!(output.read_member_at(0, None).unwrap().unwrap(), payload); +} + +#[test] +fn preserving_builder_requires_password_for_encrypted_legacy_members() { + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar40, + ] { + let mut builder = Builder::new(version) + .store(true) + .password(Some(b"secret".to_vec())); + builder + .add_bytes(b"encrypted".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!( + source.rewrite_preservation_issues().is_empty(), + "{version:?}" + ); + assert!(matches!( + source.preserving_builder(None), + Err(rars::Error::NeedPassword) + )); + assert!(matches!( + source.preserving_builder(Some(b"")), + Err(rars::Error::NeedPassword) + )); + let mut preserving = source.preserving_builder(Some(b"secret")).unwrap(); + preserving + .add_bytes(b"encrypted".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + assert_eq!( + output.read_member_at(0, Some(b"secret")).unwrap().unwrap(), + b"payload", + "{version:?}" + ); + } +} + +#[test] +fn preserving_builder_refuses_an_sfx_prefix_it_cannot_preserve() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let mut prefixed = b"MZ-stub".to_vec(); + prefixed.extend(builder.to_bytes().unwrap()); + let source = ArchiveReader::read_owned(prefixed).unwrap(); + assert!(source + .rewrite_preservation_issues() + .contains(&"SFX executable prefix".to_string())); + assert!(matches!( + source.preserving_builder(None), + Err(rars::Error::InvalidArgument( + "archive has unsupported preservation settings" + )) + )); +} + +#[test] +fn preservation_preflight_refuses_metadata_it_cannot_rewrite() { + use rars::{rar50, Archive}; + + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let Archive::Rar50Plus(seed) = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + else { + unreachable!() + }; + assert!(Archive::Rar50Plus(seed.clone()) + .rewrite_preservation_issues() + .is_empty()); + let check = |mutate: fn(&mut rar50::Archive), expected: &str| { + let mut candidate = seed.clone(); + mutate(&mut candidate); + let archive = Archive::Rar50Plus(candidate); + let issues = archive.rewrite_preservation_issues(); + assert!( + issues.iter().any(|issue| issue.contains(expected)), + "missing {expected:?}: {issues:?}" + ); + assert!(archive.preserving_builder(None).is_err()); + }; + check( + |archive| archive.main.archive_flags |= 1 << 20, + "main header metadata", + ); + check(|archive| archive.main.archive_flags |= 1, "volume layout"); + check( + |archive| archive.main.volume_number = Some(1), + "volume layout", + ); + check( + |archive| archive.main.block.flags |= 2, + "main header metadata", + ); + check( + |archive| archive.main.block.data_size = Some(1), + "main header metadata", + ); + check( + |archive| archive.main.archive_flags |= 8, + "recovery flag and service disagree", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.name = b"../file".to_vec(); + } + }, + "unsupported output name", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.host_os = 99; + } + }, + "unknown host attributes", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.block.flags |= 8; + } + }, + "split-volume layout", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.host_os = 1; + file.attributes = 1 << 20; + } + }, + "unsupported or inconsistent attributes", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.host_os = 1; + file.file_flags |= 1; + file.attributes = 0o100644; + } + }, + "unsupported or inconsistent attributes", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.host_os = 0; + file.attributes = 0x10; + } + }, + "unsupported or inconsistent attributes", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.block.flags |= 0x100; + } + }, + "unsupported, duplicate or incomplete metadata", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.file_flags |= 8; + } + }, + "unsupported, duplicate or incomplete metadata", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.compression_info |= 1 << 21; + } + }, + "unsupported, duplicate or incomplete metadata", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.compression_info = (file.compression_info & !0x3f) | 2; + } + }, + "unsupported compression algorithm", + ); + check( + |archive| { + if let rar50::Block::File(file) = &mut archive.blocks[0] { + file.compression_info |= 0x40; + } + }, + "solid dependency without a solid archive", + ); + check( + |archive| { + if let rar50::Block::End(end) = archive.blocks.last_mut().unwrap() { + end.flags = 1; + } + }, + "end header flags", + ); + check( + |archive| { + if let rar50::Block::End(end) = archive.blocks.last_mut().unwrap() { + end.block.flags = 1; + } + }, + "end header flags", + ); + check( + |archive| { + if let rar50::Block::End(end) = archive.blocks.last_mut().unwrap() { + end.block.header_size = 4; + } + }, + "end header flags", + ); + check( + |archive| { + let mut unknown = archive.main.block.clone(); + unknown.header_type = 99; + archive.blocks.push(rar50::Block::Unknown(unknown)); + }, + "unknown archive block", + ); +} + +#[test] +fn preservation_preflight_checks_derived_service_and_locator_consistency() { + use rars::{rar50, Archive}; + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .recovery_percent(Some(5)) + .archive_metadata(None, false, true) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let Archive::Rar50Plus(seed) = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + else { + unreachable!() + }; + assert!(Archive::Rar50Plus(seed.clone()) + .rewrite_preservation_issues() + .is_empty()); + + let mut invalid_service = seed.clone(); + let service = invalid_service + .blocks + .iter_mut() + .find_map(|block| match block { + rar50::Block::Service(service) if service.name == b"QO" => Some(service), + _ => None, + }) + .unwrap(); + service.attributes = 1; + assert!(Archive::Rar50Plus(invalid_service) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("unsupported derived service"))); + + let mut missing_service = seed.clone(); + missing_service + .blocks + .retain(|block| !matches!(block, rar50::Block::Service(service) if service.name == b"QO")); + assert!(Archive::Rar50Plus(missing_service) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("locator refers to a missing service"))); + + let mut missing_recovery = seed.clone(); + missing_recovery + .blocks + .retain(|block| !matches!(block, rar50::Block::Service(service) if service.name == b"RR")); + let issues = Archive::Rar50Plus(missing_recovery).rewrite_preservation_issues(); + assert!(issues + .iter() + .any(|issue| issue.contains("recovery flag and service disagree"))); + assert!(issues + .iter() + .any(|issue| issue.contains("locator refers to a missing service"))); + + let mut conflicting_encryption = seed; + conflicting_encryption.main.encrypted_headers = true; + assert!(Archive::Rar50Plus(conflicting_encryption) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("quick-open index with encrypted headers"))); +} + +#[test] +fn preservation_preflight_checks_every_derived_service_field() { + use rars::{rar50, Archive, FileTimes, FileTimestamp}; + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .recovery_percent(Some(5)) + .archive_metadata(None, false, true) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let Archive::Rar50Plus(seed) = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + else { + unreachable!() + }; + let check = |name: &[u8], mutate: fn(&mut rar50::FileHeader)| { + let mut candidate = seed.clone(); + let service = candidate + .blocks + .iter_mut() + .find_map(|block| match block { + rar50::Block::Service(service) if service.name == name => Some(service), + _ => None, + }) + .unwrap(); + mutate(service); + let issues = Archive::Rar50Plus(candidate).rewrite_preservation_issues(); + assert!( + issues + .iter() + .any(|issue| issue.contains("unsupported derived service")), + "{name:?}: {issues:?}" + ); + }; + let q = b"QO"; + check(q, |service| service.encrypted = true); + check(q, |service| service.mtime = Some(1)); + check(q, |service| { + service.file_times = Some(FileTimes { + modified: Some(FileTimestamp::Unix { + seconds: 1, + nanoseconds: 0, + }), + ..FileTimes::default() + }); + }); + check(q, |service| service.file_flags |= 8); + check(q, |service| service.block.flags |= 4); + check(q, |service| service.attributes = 1); + check(q, |service| service.host_os = 1); + check(q, |service| service.compression_info = 1); + check(b"RR", |service| service.service_data = Some(vec![0])); + + let mut duplicate = seed.clone(); + let service = duplicate + .blocks + .iter() + .find(|block| matches!(block, rar50::Block::Service(service) if service.name == q)) + .unwrap() + .clone(); + duplicate.blocks.push(service); + assert!(Archive::Rar50Plus(duplicate) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("unsupported derived service"))); +} + +#[test] +fn preservation_preflight_checks_comment_service_metadata_and_duplicates() { + use rars::{rar50, Archive, FileTimes, FileTimestamp}; + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .comment(Some(b"comment".to_vec())); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let Archive::Rar50Plus(seed) = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap() + else { + unreachable!() + }; + assert!(Archive::Rar50Plus(seed.clone()) + .rewrite_preservation_issues() + .is_empty()); + let check = |mutate: fn(&mut rar50::FileHeader)| { + let mut candidate = seed.clone(); + let service = candidate + .blocks + .iter_mut() + .find_map(|block| match block { + rar50::Block::Service(service) if service.name == b"CMT" => Some(service), + _ => None, + }) + .unwrap(); + mutate(service); + let issues = Archive::Rar50Plus(candidate).rewrite_preservation_issues(); + assert!( + issues.iter().any(|issue| issue.contains("service record")), + "{issues:?}" + ); + }; + check(|service| service.name = b"OTHER".to_vec()); + check(|service| service.mtime = Some(1)); + check(|service| { + service.file_times = Some(FileTimes { + modified: Some(FileTimestamp::Unix { + seconds: 1, + nanoseconds: 0, + }), + ..FileTimes::default() + }); + }); + check(|service| service.file_flags |= 8); + check(|service| service.block.flags |= 4); + check(|service| service.attributes = 1); + check(|service| service.host_os = 1); + check(|service| service.compression_info |= 1 << 15); + check(|service| service.compression_info |= 1); + + let mut duplicate = seed; + let index = duplicate + .blocks + .iter() + .position(|block| matches!(block, rar50::Block::Service(service) if service.name == b"CMT")) + .unwrap(); + let service = duplicate.blocks[index].clone(); + duplicate.blocks.insert(index + 1, service); + assert!(Archive::Rar50Plus(duplicate) + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("service record"))); +} + +#[test] +fn legacy_rewrite_helpers_leave_regular_files_without_link_or_modern_metadata() { + let mut builder = Builder::new(ArchiveVersion::Rar14).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert_eq!(archive.member_comment_encryption(), [false]); + assert_eq!(archive.legacy_symlink_targets(None).unwrap(), [None]); + let member = archive.members().next().unwrap(); + assert!(member.file_times().unwrap().is_none()); + assert!(member.unix_symlink().is_none()); +} + +#[test] +fn link_target_scan_skips_corrupt_regular_payloads_in_every_family() { + const PAYLOAD: &[u8] = b"a distinct stored payload for the link scan"; + for version in [ + ArchiveVersion::Rar14, + ArchiveVersion::Rar20, + ArchiveVersion::Rar50, + ] { + let mut builder = Builder::new(version).store(true); + builder + .add_directory(b"directory".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), PAYLOAD.to_vec(), None, None) + .unwrap(); + let mut bytes = builder.to_bytes().unwrap(); + let offset = bytes + .windows(PAYLOAD.len()) + .position(|window| window == PAYLOAD) + .unwrap(); + bytes[offset] ^= 1; + let archive = ArchiveReader::read_owned(bytes).unwrap(); + assert!(archive.read_member_at(1, None).is_err(), "{version:?}"); + assert_eq!(archive.legacy_symlink_targets(None).unwrap(), [None, None]); + } +} + +#[test] +fn preserving_builder_retains_encrypted_rar5_archive_comment_settings() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .comment(Some(b"private comment".to_vec())) + .archive_comment_password(Some(b"secret".to_vec())); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!(archive.rewrite_preservation_issues().is_empty()); + assert!(matches!( + archive.preserving_builder(None), + Err(rars::Error::NeedPassword) + )); + let mut preserving = archive + .preserving_builder(Some(b"secret")) + .unwrap() + .comment(Some(b"private comment".to_vec())); + preserving + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let output = ArchiveReader::read_owned(preserving.to_bytes().unwrap()).unwrap(); + assert!(matches!( + output.comment(None), + Err(rars::Error::NeedPassword) + )); + assert_eq!( + output.comment(Some(b"secret")).unwrap(), + Some(b"private comment".to_vec()) + ); +} + +#[test] +fn preservation_preflight_refuses_volume_layout() { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .volume_size(Some(64)); + builder + .add_bytes(b"file".to_vec(), vec![42; 1024], None, None) + .unwrap(); + let parts = builder.build_volumes(None).unwrap(); + assert!(parts.len() > 1); + let first = ArchiveReader::read_owned(parts[0].clone()).unwrap(); + assert!(first + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("volume layout"))); + assert!(first.preserving_builder(None).is_err()); +} + +#[test] +fn mixed_data_and_comment_encryption_are_independent() { + for solid in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar50) + .solid(solid) + .password(Some(b"secret".to_vec())); + for name in [b"plain".as_slice(), b"encrypted"] { + builder + .add_bytes(name.to_vec(), b"payload".repeat(100), None, None) + .unwrap(); + builder + .set_file_comment(name, Some(b"comment".to_vec())) + .unwrap(); + } + builder + .set_entry_encryption(b"plain", None, Some(b"secret".to_vec())) + .unwrap(); + builder + .set_entry_encryption(b"encrypted", Some(b"secret".to_vec()), None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert!(!members[0].meta.is_encrypted); + assert!(members[1].meta.is_encrypted); + assert_eq!(archive.member_comment_encryption(), [true, false]); + assert!(archive.rewrite_preservation_issues().is_empty()); + assert!(archive.preserving_builder(None).is_err()); + assert!(archive.preserving_builder(Some(b"secret")).is_ok()); + assert_eq!( + archive.read_member_at(1, Some(b"secret")).unwrap().unwrap(), + b"payload".repeat(100) + ); + assert_eq!( + archive.member_comments(Some(b"secret")).unwrap(), + [Some(b"comment".to_vec()), Some(b"comment".to_vec())] + ); + } +} + +#[test] +fn metadata_lock_indexes_and_recovery_are_retained_and_regenerated() { + use rars::rar50::{ + ArchiveEntry, ArchiveMetadataEntry, MainExtraRecord, Rar50Writer, WriterOptions, + }; + let bytes = Rar50Writer::new(WriterOptions::new( + ArchiveVersion::Rar50, + rars::FeatureSet::store_only(), + )) + .entries([ArchiveEntry::new( + b"file".to_vec(), + rars::EntrySource::from_bytes(b"payload".as_slice()), + ) + .with_attributes(0x20)]) + .archive_metadata(Some(ArchiveMetadataEntry { + name: Some(b"original.rar"), + creation_time: Some(123), + })) + .finish() + .unwrap(); + let seed = ArchiveReader::read_owned(bytes).unwrap(); + let rars::Archive::Rar50Plus(seed) = seed else { + unreachable!() + }; + let metadata = seed + .main + .extras + .into_iter() + .find_map(|extra| match extra { + MainExtraRecord::ArchiveMetadata(value) => Some(value), + _ => None, + }) + .unwrap(); + for flags in [3, 7, 15] { + let mut metadata = metadata.clone(); + metadata.flags = flags; + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .recovery_percent(Some(5)) + .archive_metadata(Some(metadata.clone()), true, true) + .unwrap(); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let source = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + assert!( + source.rewrite_preservation_issues().is_empty(), + "{:?}", + source.rewrite_preservation_issues() + ); + if flags == 3 { + let mut invalid = source.clone(); + if let rars::Archive::Rar50Plus(archive) = &mut invalid { + for extra in &mut archive.main.extras { + if let MainExtraRecord::ArchiveMetadata(metadata) = extra { + metadata.flags |= 1 << 20; + } + } + } + assert!(invalid + .rewrite_preservation_issues() + .iter() + .any(|issue| issue.contains("unsupported archive metadata"))); + } + let mut rewritten = source.preserving_builder(None).unwrap(); + rewritten + .add_bytes(b"renamed".to_vec(), b"new payload".to_vec(), None, None) + .unwrap(); + let output = ArchiveReader::read_owned(rewritten.to_bytes().unwrap()).unwrap(); + assert!( + output.rewrite_preservation_issues().is_empty(), + "{:?}", + output.rewrite_preservation_issues() + ); + let rars::Archive::Rar50Plus(output) = output else { + unreachable!() + }; + assert!(output.main.is_locked()); + assert!(output.main.has_recovery_record()); + assert!(output.main.extras.iter().any( + |extra| matches!(extra, MainExtraRecord::ArchiveMetadata(value) if value == &metadata) + )); + assert!(output.blocks.iter().any( + |block| matches!(block, rars::rar50::Block::Service(service) if service.name == b"QO") + )); + } +} diff --git a/arca-rar/vendor/rars/tests/rewrite_staging.rs b/arca-rar/vendor/rars/tests/rewrite_staging.rs new file mode 100644 index 00000000..f4ddb042 --- /dev/null +++ b/arca-rar/vendor/rars/tests/rewrite_staging.rs @@ -0,0 +1,788 @@ +#![cfg(feature = "write")] +#![cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + +use rars::{ + ArchiveReadOptions, ArchiveReader, ArchiveVersion, Builder, Error, ErrorKind, + ExtractionDecision, RewriteStaging, +}; +use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +#[path = "support/scratch.rs"] +mod scratch; + +#[test] +fn staging_finished_events_require_verified_payloads() { + use rars::{WriteOperation, WriteProgressEvent}; + use std::sync::Mutex; + let root = scratch::case("rewrite-progress-integrity"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"first".to_vec(), b"first payload".to_vec(), None, None) + .unwrap(); + builder + .add_bytes(b"fault".to_vec(), b"fault payload".to_vec(), None, None) + .unwrap(); + let mut bytes = builder.to_bytes().unwrap(); + let offset = bytes + .windows(13) + .position(|bytes| bytes == b"fault payload") + .unwrap(); + bytes[offset] ^= 1; + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let finished = Arc::new(Mutex::new(Vec::new())); + let recorded = finished.clone(); + let progress = Arc::new(move |event: WriteProgressEvent<'_>| match event { + WriteProgressEvent::EntryFinished { + operation: WriteOperation::Staging, + name, + .. + } => { + recorded.lock().unwrap().push(name.to_vec()); + } + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Staging, + .. + } => { + panic!("failed staging must not finish"); + } + _ => {} + }); + let error = archive + .stage_rewrite_sources_with_progress( + &[0, 1], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 26, + }, + Some(progress), + ) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ChecksumMismatch); + assert_eq!(*finished.lock().unwrap(), vec![b"first".to_vec()]); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn staging_finishes_selected_member_before_a_trailing_archive_member() { + use rars::{WriteOperation, WriteProgressEvent}; + use std::sync::Mutex; + + let root = scratch::case("rewrite-progress-trailing"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + for name in [b"selected".as_slice(), b"trailing".as_slice()] { + builder + .add_bytes(name.to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + } + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let events = Arc::new(Mutex::new(Vec::new())); + let recorded = events.clone(); + let progress = Arc::new(move |event: WriteProgressEvent<'_>| match event { + WriteProgressEvent::EntryFinished { + operation: WriteOperation::Staging, + name, + .. + } => recorded.lock().unwrap().push(name.to_vec()), + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Staging, + .. + } => recorded.lock().unwrap().push(b"finished".to_vec()), + _ => {} + }); + let sources = archive + .stage_rewrite_sources_with_progress( + &[0], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 7, + }, + Some(progress), + ) + .unwrap(); + assert_eq!( + &*events.lock().unwrap(), + &[b"selected".to_vec(), b"finished".to_vec()] + ); + assert_eq!(sources.len(), 1); +} + +#[test] +fn staging_progress_cancellation_prevents_source_creation() { + use rars::{WriteProgress, WriteProgressEvent}; + use std::sync::atomic::AtomicBool; + + struct CancelAtStart { + cancelled: AtomicBool, + cancel_on_report: bool, + } + impl WriteProgress for CancelAtStart { + fn report(&self, _: WriteProgressEvent<'_>) { + if self.cancel_on_report { + self.cancelled.store(true, Ordering::SeqCst); + } + } + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } + } + + let root = scratch::case("rewrite-progress-cancel"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 7, + }; + for cancel_on_report in [false, true] { + let progress: Arc = Arc::new(CancelAtStart { + cancelled: AtomicBool::new(!cancel_on_report), + cancel_on_report, + }); + assert_eq!( + archive + .stage_rewrite_sources_with_progress( + &[0], + ArchiveReadOptions::default(), + &staging, + Some(progress), + ) + .unwrap_err(), + Error::Cancelled + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} + +struct Counted { + data: Cursor>, + count: Arc, +} + +impl Read for Counted { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + let read = self.data.read(bytes)?; + self.count.fetch_add(read as u64, Ordering::Relaxed); + Ok(read) + } +} + +impl Seek for Counted { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.data.seek(from) + } +} + +#[test] +fn solid_staging_reads_once_and_reopens_without_touching_archive() { + let root = scratch::case("rewrite-once"); + for format in ArchiveVersion::ALL { + let mut builder = Builder::new(format).solid(true); + for index in 0..3 { + builder + .add_bytes( + format!("file{index}").into_bytes(), + vec![b'a' + index; 2048], + None, + None, + ) + .unwrap(); + } + let count = Arc::new(AtomicU64::new(0)); + let archive = ArchiveReader::read_reader(Counted { + data: Cursor::new(builder.to_bytes().unwrap()), + count: count.clone(), + }) + .unwrap(); + count.store(0, Ordering::Relaxed); + archive + .extract_with_control(ArchiveReadOptions::default(), |_| { + Ok(ExtractionDecision::Extract(Box::new(std::io::sink()))) + }) + .unwrap(); + let one_pass = count.swap(0, Ordering::Relaxed); + let sources = archive + .stage_rewrite_sources( + &[2, 0], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 4096, + }, + ) + .unwrap(); + assert_eq!(count.load(Ordering::Relaxed), one_pass, "{format}"); + for _ in 0..2 { + for (source, expected) in sources.iter().zip(*b"ca") { + let mut data = Vec::new(); + source.open().unwrap().read_to_end(&mut data).unwrap(); + assert_eq!(data, vec![expected; 2048]); + } + } + assert_eq!(count.load(Ordering::Relaxed), one_pass); + drop(sources); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} + +#[test] +fn sources_and_independent_readers_own_the_staged_files() { + let root = scratch::case("rewrite-lifetime"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"abcdef".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let sources = archive + .stage_rewrite_sources( + &[0], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 6, + }, + ) + .unwrap(); + let clone = sources[0].clone(); + let mut a = clone.open().unwrap(); + let mut b = clone.open().unwrap(); + a.seek(SeekFrom::Start(3)).unwrap(); + drop(sources); + drop(clone); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 1); + let mut data = Vec::new(); + a.read_to_end(&mut data).unwrap(); + assert_eq!(data, b"def"); + data.clear(); + b.read_to_end(&mut data).unwrap(); + assert_eq!(data, b"abcdef"); + drop(a); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 1); + drop(b); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn encrypted_sources_require_password_and_respect_cancellation() { + let root = scratch::case("rewrite-password"); + for format in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + let mut builder = Builder::new(format) + .store(true) + .password(Some(b"secret".to_vec())); + builder + .add_bytes(b"file".to_vec(), b"secret content".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 14, + }; + assert!(archive + .stage_rewrite_sources(&[0], ArchiveReadOptions::default(), &staging) + .is_err()); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + let options = ArchiveReadOptions::with_password(b"secret"); + let sources = archive + .stage_rewrite_sources(&[0], options, &staging) + .unwrap(); + let mut data = Vec::new(); + sources[0].open().unwrap().read_to_end(&mut data).unwrap(); + assert_eq!(data, b"secret content"); + drop(sources); + let token = rars::ReadCancellation::new(); + token.cancel(); + assert_eq!( + archive + .stage_rewrite_sources(&[0], options.with_cancellation(&token), &staging) + .unwrap_err(), + Error::Cancelled + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} + +#[test] +fn admission_fails_before_io_and_indices_include_directories() { + let root = scratch::case("rewrite-admission"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_bytes(b"file".to_vec(), b"abc".to_vec(), None, None) + .unwrap(); + let count = Arc::new(AtomicU64::new(0)); + let archive = ArchiveReader::read_reader(Counted { + data: Cursor::new(builder.to_bytes().unwrap()), + count: count.clone(), + }) + .unwrap(); + count.store(0, Ordering::Relaxed); + let staging = RewriteStaging { + directory: root.join("does-not-exist"), + max_staged_bytes: 2, + }; + assert!(matches!( + archive.stage_rewrite_sources(&[1], ArchiveReadOptions::default(), &staging), + Err(Error::RewriteStagingLimitExceeded { + limit: 2, + required: 3 + }) + )); + assert!(matches!( + archive.stage_rewrite_sources(&[1, 1], ArchiveReadOptions::default(), &staging), + Err(Error::DuplicateEntry) + )); + assert!(matches!( + archive.stage_rewrite_sources(&[2], ArchiveReadOptions::default(), &staging), + Err(Error::EntryNotFound) + )); + assert!(matches!( + archive.stage_rewrite_sources(&[0], ArchiveReadOptions::default(), &staging), + Err(Error::InvalidArgument(_)) + )); + assert!(archive + .stage_rewrite_sources(&[], ArchiveReadOptions::default(), &staging) + .unwrap() + .is_empty()); + assert_eq!(count.load(Ordering::Relaxed), 0); + let sources = archive + .stage_rewrite_sources( + &[1], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 3, + }, + ) + .unwrap(); + let mut output = Builder::new(ArchiveVersion::Rar50).store(true); + output + .add_source(b"renamed".to_vec(), sources[0].clone(), None, None) + .unwrap(); + let path = root.join("output.rar"); + output.write_to_path(&path, None).unwrap(); + let rewritten = ArchiveReader::read_owned(std::fs::read(path).unwrap()).unwrap(); + assert_eq!( + rewritten.read_member(b"renamed", None).unwrap().unwrap(), + b"abc" + ); +} + +#[test] +fn staging_rejects_redirections_and_split_volume_fragments() { + let root = scratch::case("rewrite-special-admission"); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 1024, + }; + let mut link = Builder::new(ArchiveVersion::Rar50).store(true); + link.add_unix_symlink(b"link".to_vec(), b"target".to_vec(), false, None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(link.to_bytes().unwrap()).unwrap(); + assert!(archive.members().next().unwrap().meta.is_redirection); + assert!(matches!( + archive.stage_rewrite_sources(&[0], ArchiveReadOptions::default(), &staging), + Err(Error::InvalidArgument(_)) + )); + + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .volume_size(Some(4)); + builder + .add_bytes(b"file".to_vec(), b"a split payload".to_vec(), None, None) + .unwrap(); + let volumes = builder.build_volumes(None).unwrap(); + assert!(volumes.len() >= 2); + for (index, bytes) in volumes.iter().take(2).enumerate() { + let archive = ArchiveReader::read_owned(bytes.clone()).unwrap(); + let member = archive.members().next().unwrap(); + assert!(if index == 0 { + member.meta.is_split_after + } else { + member.meta.is_split_before + }); + assert!(matches!( + archive.stage_rewrite_sources(&[0], ArchiveReadOptions::default(), &staging), + Err(Error::InvalidArgument(_)) + )); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn solid_staging_skips_directory_and_link_between_payload_dependencies() { + let root = scratch::case("rewrite-solid-special-dependencies"); + let mut builder = Builder::new(ArchiveVersion::Rar50).solid(true); + builder + .add_bytes(b"first".to_vec(), b"first payload".to_vec(), None, None) + .unwrap(); + builder.add_directory(b"dir".to_vec(), None, None).unwrap(); + builder + .add_unix_symlink(b"link".to_vec(), b"first".to_vec(), false, None, None) + .unwrap(); + builder + .add_bytes(b"last".to_vec(), b"last payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let members: Vec<_> = archive.members().collect(); + assert!(members[1].meta.is_directory); + assert!(members[2].meta.is_redirection); + + let sources = archive + .stage_rewrite_sources( + &[3], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 64, + }, + ) + .unwrap(); + let mut data = Vec::new(); + sources[0].open().unwrap().read_to_end(&mut data).unwrap(); + assert_eq!(data, b"last payload"); + drop(sources); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn corrupt_dependencies_fail_cleanly_but_independent_omissions_are_skipped() { + let root = scratch::case("rewrite-corruption"); + for solid in [false, true] { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true).solid(solid); + for name in [b"first", b"fault", b"final"] { + builder + .add_bytes(name.to_vec(), name.repeat(5), None, None) + .unwrap(); + } + let mut bytes = builder.to_bytes().unwrap(); + let offset = bytes + .windows(25) + .position(|bytes| bytes == b"fault".repeat(5)) + .unwrap(); + bytes[offset] ^= 1; + let archive = ArchiveReader::read_owned(bytes).unwrap(); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 75, + }; + let error = archive + .stage_rewrite_sources(&[0, 1], ArchiveReadOptions::default(), &staging) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ChecksumMismatch); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + let prefix = archive + .stage_rewrite_sources(&[0], ArchiveReadOptions::default(), &staging) + .unwrap(); + drop(prefix); + let suffix = archive.stage_rewrite_sources(&[2], ArchiveReadOptions::default(), &staging); + if solid { + assert_eq!(suffix.unwrap_err().kind(), ErrorKind::ChecksumMismatch); + } else { + drop(suffix.unwrap()); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} + +#[test] +fn incremental_legacy_sources_fit_staging_preparation_windows() { + let run = || { + let root = scratch::case("rewrite-incremental-legacy"); + for sizes in [[2048, 2048, 2048], [512, 2048, 1024]] { + let mut original = Builder::new(ArchiveVersion::Rar50).store(true); + for (index, size) in sizes.into_iter().enumerate() { + original + .add_bytes( + format!("f{index}").into_bytes(), + vec![b'a' + index as u8; size], + None, + None, + ) + .unwrap(); + } + let archive = ArchiveReader::read_owned(original.to_bytes().unwrap()).unwrap(); + for format in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + for (store, solid, encrypted) in [ + (true, false, false), + (false, false, false), + (false, true, false), + (true, false, true), + ] { + if encrypted && !cfg!(feature = "encryption") { + continue; + } + let password = encrypted.then_some(b"secret".as_slice()); + for max_staged_bytes in [2048, 4096] { + let bytes = archive.with_rewrite_sources(&[0, 1, 2], ArchiveReadOptions::default(), + &RewriteStaging { directory: root.to_path_buf(), max_staged_bytes }, None, + |sources| { + let mut output = Builder::new(format).store(store).solid(solid).password(password.map(<[u8]>::to_vec)); + for (index, source) in sources.into_iter().enumerate() { + output.add_source(format!("f{index}").into_bytes(), source, None, None)?; + } + output.to_bytes() + }).unwrap_or_else(|error| panic!("{format:?} store={store} solid={solid} encrypted={encrypted} quota={max_staged_bytes}: {error}")); + let output = ArchiveReader::read_owned(bytes).unwrap(); + for (index, size) in sizes.into_iter().enumerate() { + assert_eq!( + output.read_member_at(index, password).unwrap().unwrap(), + vec![b'a' + index as u8; size] + ); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + } + } + } + }; + #[cfg(feature = "parallel")] + rayon::ThreadPoolBuilder::new() + .num_threads(2) + .build() + .unwrap() + .install(run); + #[cfg(not(feature = "parallel"))] + run(); +} + +#[test] +fn incremental_streaming_fallback_fits_one_payload_of_staging() { + let root = scratch::case("rewrite-incremental-stream"); + let size = 2 * 1024 * 1024; + let mut original = Builder::new(ArchiveVersion::Rar50).store(true); + for index in 0..3 { + original + .add_bytes( + format!("f{index}").into_bytes(), + vec![b'a' + index; size], + None, + None, + ) + .unwrap(); + } + let archive = ArchiveReader::read_owned(original.to_bytes().unwrap()).unwrap(); + let bytes = archive + .with_rewrite_sources( + &[0, 1, 2], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: size as u64, + }, + None, + |sources| { + let mut output = Builder::new(ArchiveVersion::Rar50).compression_level(Some(1)); + for (index, source) in sources.into_iter().enumerate() { + output.add_source(format!("f{index}").into_bytes(), source, None, None)?; + } + let mut bytes = Vec::new(); + output.write_to( + &mut bytes, + &rars::WriterResources::new(70 * 1024 * 1024).with_temp_dir(&*root), + None, + )?; + Ok(bytes) + }, + ) + .unwrap(); + let output = ArchiveReader::read_owned(bytes).unwrap(); + for index in 0..3 { + assert_eq!( + output.read_member_at(index, None).unwrap().unwrap(), + vec![b'a' + index as u8; size] + ); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn incremental_sources_decode_once_and_join_on_early_failure_or_panic() { + let root = scratch::case("rewrite-incremental-once"); + let mut builder = Builder::new(ArchiveVersion::Rar50).solid(true); + for index in 0..3 { + builder + .add_bytes( + format!("f{index}").into_bytes(), + vec![b'a' + index; 2048], + None, + None, + ) + .unwrap(); + } + let count = Arc::new(AtomicU64::new(0)); + let archive = ArchiveReader::read_reader(Counted { + data: Cursor::new(builder.to_bytes().unwrap()), + count: count.clone(), + }) + .unwrap(); + count.store(0, Ordering::Relaxed); + archive + .extract_with_control(ArchiveReadOptions::default(), |_| { + Ok(ExtractionDecision::Extract(Box::new(std::io::sink()))) + }) + .unwrap(); + let one_pass = count.swap(0, Ordering::Relaxed); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 4096, + }; + archive + .with_rewrite_sources( + &[2, 0], + ArchiveReadOptions::default(), + &staging, + None, + |sources| { + // Out-of-order requests retain required earlier payloads without replaying decoding. + for _ in 0..2 { + for (source, byte) in sources.iter().zip(*b"ca") { + let mut bytes = Vec::new(); + source.open()?.read_to_end(&mut bytes)?; + assert_eq!(bytes, vec![byte; 2048]); + } + } + Ok(()) + }, + ) + .unwrap(); + assert_eq!(count.load(Ordering::Relaxed), one_pass); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + let token = rars::ReadCancellation::new(); + let result: rars::Result<()> = archive.with_rewrite_sources( + &[0, 2], + ArchiveReadOptions::default().with_cancellation(&token), + &staging, + None, + |sources| { + let _reader = sources[0].open()?; + Err(Error::InvalidArgument("writer stopped")) + }, + ); + assert!(matches!( + result, + Err(Error::InvalidArgument("writer stopped")) + )); + assert!(!token.is_cancelled()); + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _: rars::Result<()> = archive.with_rewrite_sources( + &[0, 2], + ArchiveReadOptions::default(), + &staging, + None, + |sources| { + let _reader = sources[0].open()?; + panic!("writer panic"); + }, + ); + })); + assert!(panic.is_err()); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn incremental_decoder_callback_panic_wakes_the_consumer() { + let root = scratch::case("rewrite-decoder-panic"); + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(builder.to_bytes().unwrap()).unwrap(); + let error = archive + .with_rewrite_sources( + &[0], + ArchiveReadOptions::default(), + &RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 7, + }, + Some(Arc::new(|_: rars::WriteProgressEvent<'_>| { + panic!("decoder callback") + })), + |sources| { + sources[0].open()?; + Ok(()) + }, + ) + .unwrap_err(); + assert_eq!( + error, + Error::WriterFailure("rewrite decoder thread panicked") + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn incremental_rewrites_match_eager_staging_bytes() { + let root = scratch::case("rewrite-incremental-bytes"); + let mut input = Builder::new(ArchiveVersion::Rar50).store(true); + input + .add_bytes( + b"a".to_vec(), + b"compressible text\n".repeat(1000), + None, + None, + ) + .unwrap(); + input + .add_bytes(b"b".to_vec(), b"small".to_vec(), None, None) + .unwrap(); + let archive = ArchiveReader::read_owned(input.to_bytes().unwrap()).unwrap(); + let staging = RewriteStaging { + directory: root.to_path_buf(), + max_staged_bytes: 20000, + }; + for format in [ + ArchiveVersion::Rar29, + ArchiveVersion::Rar50, + ArchiveVersion::Rar70, + ] { + for solid in [false, true] { + let encode = |sources: Vec| { + let mut builder = Builder::new(format).solid(solid).compression_level(Some(1)); + for (index, source) in sources.into_iter().enumerate() { + builder.add_source(format!("f{index}").into_bytes(), source, None, None)?; + } + builder.to_bytes() + }; + let eager = encode( + archive + .stage_rewrite_sources(&[0, 1], ArchiveReadOptions::default(), &staging) + .unwrap(), + ) + .unwrap(); + let incremental = archive + .with_rewrite_sources( + &[0, 1], + ArchiveReadOptions::default(), + &staging, + None, + encode, + ) + .unwrap(); + assert_eq!(incremental, eager, "{format:?}, solid={solid}"); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + } +} diff --git a/arca-rar/vendor/rars/tests/source_consistency.rs b/arca-rar/vendor/rars/tests/source_consistency.rs new file mode 100644 index 00000000..f2069315 --- /dev/null +++ b/arca-rar/vendor/rars/tests/source_consistency.rs @@ -0,0 +1,362 @@ +#![cfg(feature = "write")] + +use rars::{rar50, ArchiveVersion, EntrySource, FeatureSet, WriterResources}; +use std::io::Cursor; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn changing_source(len: usize, change_at: usize, changed_len: usize) -> EntrySource { + let opens = Arc::new(AtomicUsize::new(0)); + EntrySource::from_opener(len as u64, move || { + let changed = opens.fetch_add(1, Ordering::Relaxed) >= change_at; + Ok(Box::new(Cursor::new(if changed { + vec![b'B'; changed_len] + } else { + vec![b'A'; len] + }))) + }) +} + +fn write_rar50( + source: EntrySource, + encrypted: bool, + volumes: bool, + level: u8, +) -> rars::Result>> { + let entry = rar50::ArchiveEntry::new(b"file".to_vec(), source); + let entry = if encrypted { + entry.with_password(b"secret".to_vec()) + } else { + entry + }; + let options = rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::default()) + .with_compression_level(level); + if volumes { + let mut sink = rar50::CollectedVolumes::new(); + rar50::write_streaming_volumes_to( + &[entry], + options, + rar50::ArchiveExtras::default(), + 1024, + &mut sink, + &WriterResources::default(), + )?; + Ok(sink.take()) + } else { + rar50::Rar50Writer::new(options) + .entry(entry) + .finish() + .map(|bytes| vec![bytes]) + } +} + +#[test] +fn rar50_stored_emission_rejects_changed_sources() { + for encrypted in [false, true] { + for volumes in [false, true] { + for changed_len in [4095, 4096, 4097] { + let result = + write_rar50(changing_source(4096, 1, changed_len), encrypted, volumes, 0); + assert!( + result.is_err(), + "encrypted={encrypted}, volumes={volumes}, size={changed_len}" + ); + } + } + } +} + +#[test] +fn rar50_fragment_checksums_are_verified_against_the_emission_read() { + // First open prepares whole-member integrity, second prepares the first + // fragment's header. Mutation on the third open changes its emitted bytes. + let error = write_rar50(changing_source(4096, 2, 4096), false, true, 0).unwrap_err(); + assert!(error.to_string().contains("contents changed")); + assert_eq!(error.kind(), rars::ErrorKind::SourceChanged); + assert_eq!(error.entry_context().unwrap().0, b"file"); +} + +// Equal-length CRC32 collision, independently constructed with zlib's CRC32. +// Repeating each block preserves the collision for every 1024-byte fragment. +fn crc_collision_payloads() -> (Vec, Vec) { + let original = vec![b'A'; 64]; + let mut changed = vec![b'B'; 60]; + changed.extend_from_slice(&[224, 101, 112, 255]); + assert_ne!(original, changed); + assert_eq!(rars::crc32::crc32(&original), 0x414c623c); + assert_eq!(rars::crc32::crc32(&changed), 0x414c623c); + let original = original.repeat(64); + let changed = changed.repeat(64); + assert_eq!(rars::crc32::crc32(&original), rars::crc32::crc32(&changed)); + (original, changed) +} + +#[test] +fn rar50_stored_emission_rejects_crc_collisions_using_the_payload_hash() { + for (encrypted, volumes, change_at) in [ + (false, false, 1), + (true, false, 1), + (false, true, 1), // Fragment checksums match the changed bytes; whole-member hash refuses. + (false, true, 2), // First fragment's CRC matches, but its hash changed after header preparation. + (true, true, 1), // Encrypted volume staging must verify the source before retaining ciphertext. + ] { + let (original, changed) = crc_collision_payloads(); + let opens = Arc::new(AtomicUsize::new(0)); + let source = EntrySource::from_opener(original.len() as u64, move || { + let data = if opens.fetch_add(1, Ordering::Relaxed) >= change_at { + changed.clone() + } else { + original.clone() + }; + Ok(Box::new(Cursor::new(data))) + }); + let error = write_rar50(source, encrypted, volumes, 0).unwrap_err(); + assert_eq!( + error.kind(), + rars::ErrorKind::SourceChanged, + "encrypted={encrypted}, volumes={volumes}, change_at={change_at}: {error}" + ); + assert_eq!(error.entry_context().unwrap().0, b"file"); + assert!(error.to_string().contains("contents changed")); + } +} + +#[test] +fn rar50_split_header_preparation_rejects_a_truncated_source_reread() { + let error = write_rar50(changing_source(4096, 1, 1), false, true, 0).unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::SourceChanged); + assert_eq!(error.entry_context().unwrap().0, b"file"); + assert!(error.to_string().contains("size changed")); +} + +#[test] +fn rar50_empty_sources_and_compression_store_fallback_are_verified() { + for encrypted in [false, true] { + for volumes in [false, true] { + assert!(write_rar50(changing_source(0, 1, 1), encrypted, volumes, 0).is_err()); + } + } + assert!(write_rar50(changing_source(1, 1, 1), false, false, 3).is_err()); +} + +#[test] +fn unchanged_rar50_sources_produce_identical_plain_archive_and_volume_bytes() { + for volumes in [false, true] { + let expected = + write_rar50(EntrySource::from_bytes(vec![b'A'; 4096]), false, volumes, 0).unwrap(); + let actual = + write_rar50(changing_source(4096, usize::MAX, 4096), false, volumes, 0).unwrap(); + assert_eq!(actual, expected); + } +} + +#[test] +fn unchanged_encrypted_sources_pass_extraction_integrity_checks() { + for volumes in [false, true] { + let bytes = write_rar50(changing_source(4096, usize::MAX, 4096), true, volumes, 0).unwrap(); + let archives: Vec<_> = bytes + .into_iter() + .map(|bytes| rar50::Archive::parse_owned(bytes).unwrap()) + .collect(); + rar50::extract_volumes_to( + &archives, + rars::ArchiveReadOptions::with_password(b"secret"), + |_| Ok(Box::new(std::io::sink())), + ) + .unwrap(); + } +} + +#[test] +fn legacy_stored_emission_rejects_same_size_content_changes() { + let resources = WriterResources::default(); + let options13 = + rars::rar13::WriterOptions::new(ArchiveVersion::Rar14, FeatureSet::store_only()); + let error = rars::rar13::write_streaming_archive_to( + &[rars::rar13::StreamingEntry::new( + b"FILE".to_vec(), + changing_source(4096, 1, 4096), + )], + options13, + rars::MemberCoding::Stored, + None, + &resources, + None, + &mut Vec::new(), + ) + .unwrap_err(); + assert!(error.to_string().contains("contents changed")); + for version in [ + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ] { + let options = rars::rar15_40::WriterOptions::new(version, FeatureSet::store_only()); + let error = rars::rar15_40::write_streaming_archive_to( + &[rars::rar15_40::StreamingEntry::new( + b"file".to_vec(), + changing_source(4096, 1, 4096), + )], + options, + rars::MemberCoding::Stored, + None, + &resources, + None, + &mut Vec::new(), + ) + .unwrap_err(); + assert!(error.to_string().contains("contents changed")); + } +} + +#[test] +fn member_io_failures_keep_identity_through_compression_routes() { + for (level, solid, filter) in [ + (0, false, rar50::FilterPolicy::None), + (1, false, rar50::FilterPolicy::None), + (1, true, rar50::FilterPolicy::None), + (1, false, rar50::FilterPolicy::Auto), + ] { + for fail_in_read in [false, true] { + struct Broken; + impl std::io::Read for Broken { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected read failure", + )) + } + } + impl std::io::Seek for Broken { + fn seek(&mut self, _: std::io::SeekFrom) -> std::io::Result { + Ok(0) + } + } + let mut entries: Vec<_> = (0..8) + .map(|index| { + rar50::ArchiveEntry::new( + format!("member-{index}").into_bytes(), + EntrySource::from_bytes(vec![b'a'; 128]), + ) + }) + .collect(); + entries.push(rar50::ArchiveEntry::new( + b"failed-member".to_vec(), + EntrySource::from_opener(128, move || { + if fail_in_read { + Ok(Box::new(Broken)) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected open failure", + ) + .into()) + } + }), + )); + let mut features = FeatureSet::default(); + features.solid = solid; + let result = rar50::write_streaming_archive_to( + &entries, + rar50::WriterOptions::new(ArchiveVersion::Rar50, features) + .with_compression_level(level), + rar50::ArchiveExtras::default().with_filter_policy(filter.clone()), + &WriterResources::default(), + &mut Vec::new(), + ); + let error = result.unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((b"failed-member".as_slice(), "compressing")) + ); + assert!( + matches!(error.root_cause(), rars::Error::Io(source) if source.kind == std::io::ErrorKind::PermissionDenied) + ); + } + } +} + +#[test] +fn volume_compression_source_failure_keeps_member_identity() { + let entry = rar50::ArchiveEntry::new( + b"failed-member".to_vec(), + EntrySource::from_opener(128, || { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected open failure", + ) + .into()) + }), + ); + let mut sink = rar50::CollectedVolumes::new(); + let error = rar50::write_streaming_volumes_to( + &[entry], + rar50::WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::default()) + .with_compression_level(1), + rar50::ArchiveExtras::default(), + 64, + &mut sink, + &WriterResources::default(), + ) + .unwrap_err(); + assert_eq!(error.kind(), rars::ErrorKind::Io); + assert_eq!( + error.entry_context(), + Some((b"failed-member".as_slice(), "compressing")) + ); + assert!(sink.take().is_empty()); +} + +#[test] +fn rar50_writer_rejects_aggregate_source_size_overflow_before_reading_or_emitting() { + for version in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for volumes in [false, true] { + let entries: Vec<_> = [(b"huge".as_slice(), u64::MAX), (b"overflow", 1)] + .into_iter() + .map(|(name, len)| { + rar50::ArchiveEntry::new( + name.to_vec(), + EntrySource::from_opener(len, || { + panic!("overflowing input opened a source") + }), + ) + }) + .collect(); + let options = + rar50::WriterOptions::new(version, FeatureSet::default()).with_compression_level(0); + let mut output = Vec::new(); + let mut sink = rar50::CollectedVolumes::new(); + let error = if volumes { + rar50::write_streaming_volumes_to( + &entries, + options, + rar50::ArchiveExtras::default(), + 1024, + &mut sink, + &WriterResources::default(), + ) + .unwrap_err() + } else { + rar50::write_streaming_archive_to( + &entries, + options, + rar50::ArchiveExtras::default(), + &WriterResources::default(), + &mut output, + ) + .unwrap_err() + }; + assert_eq!(error.kind(), rars::ErrorKind::InvalidArgument); + assert_eq!( + error.entry_context().unwrap(), + (b"overflow".as_slice(), "preparing") + ); + assert!(error.to_string().contains("total input size overflows")); + assert!(output.is_empty()); + assert!(sink.take().is_empty()); + } + } +} diff --git a/arca-rar/vendor/rars/tests/support/read_errors.rs b/arca-rar/vendor/rars/tests/support/read_errors.rs new file mode 100644 index 00000000..c50d8361 --- /dev/null +++ b/arca-rar/vendor/rars/tests/support/read_errors.rs @@ -0,0 +1,37 @@ +use std::io::{self, Cursor, Read}; + +/// Short reads with one source error at an exact byte offset. +pub(crate) struct ErrorOnceReader { + inner: Cursor>, + fail_at: u64, + kind: io::ErrorKind, + failed: bool, +} + +impl ErrorOnceReader { + pub(crate) fn new(data: Vec, fail_at: u64, kind: io::ErrorKind) -> Self { + Self { + inner: Cursor::new(data), + fail_at, + kind, + failed: false, + } + } +} + +impl Read for ErrorOnceReader { + fn read(&mut self, out: &mut [u8]) -> io::Result { + if out.is_empty() { + return Ok(0); + } + if !self.failed && self.inner.position() == self.fail_at { + self.failed = true; + return Err(io::Error::new(self.kind, "source read failed")); + } + let mut count = out.len().min(5); + if !self.failed { + count = count.min((self.fail_at - self.inner.position()) as usize); + } + self.inner.read(&mut out[..count]) + } +} diff --git a/arca-rar/vendor/rars/tests/support/scratch.rs b/arca-rar/vendor/rars/tests/support/scratch.rs new file mode 100644 index 00000000..7629e5bc --- /dev/null +++ b/arca-rar/vendor/rars/tests/support/scratch.rs @@ -0,0 +1,152 @@ +//! Where the tests put the files they write. +//! +//! Not `std::env::temp_dir()`. On a stock Linux install that is `/tmp`, and on +//! a stock systemd install `/tmp` is a tmpfs, so it spends RAM. This suite +//! writes hundreds of megabytes of archives, and a few of the streaming tests +//! write a single member larger than the memory budget they are proving, +//! which is the whole point of them. Sending that to RAM is the opposite of +//! what those tests measure, and on a loaded machine it wakes the OOM killer. +//! +//! Nor does it leave anything behind. Tests used to remove their directory on +//! the last line, so every failure and every interrupted run leaked one; a few +//! thousand of them had piled up by the time anyone looked. [`case`] hands out +//! a directory that removes itself when the test drops it, which happens on +//! the way out of a panic too. +//! +//! Included with `#[path]` by each test target rather than shared through a +//! crate boundary, so it stays a test detail and does not become API. + +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// A directory of a test's own, removed when the test drops it. +/// +/// Derefs to [`Path`], so it goes wherever a `&Path` goes. +/// +/// A test that panics keeps its directory and prints the path, since that is +/// the one run whose files are worth reading. +pub struct Scratch { + path: PathBuf, +} + +impl Deref for Scratch { + type Target = Path; + + fn deref(&self) -> &Path { + &self.path + } +} + +impl AsRef for Scratch { + fn as_ref(&self) -> &Path { + &self.path + } +} + +// `Command::arg` is generic, so it takes no deref coercion. +impl AsRef for Scratch { + fn as_ref(&self) -> &std::ffi::OsStr { + self.path.as_os_str() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + if std::thread::panicking() { + eprintln!("scratch kept for inspection: {}", self.path.display()); + return; + } + let _ = std::fs::remove_dir_all(&self.path); + } +} + +/// An empty directory named after the test asking for it. +/// +/// The name only has to be readable; the process id and a counter make it +/// unique, so two tests running at once cannot collide. +pub fn case(name: &str) -> Scratch { + static NEXT: AtomicU64 = AtomicU64::new(0); + let serial = NEXT.fetch_add(1, Ordering::Relaxed); + let path = root().join(format!("{name}-{}-{serial}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path) + .unwrap_or_else(|err| panic!("cannot create {}: {err}", path.display())); + Scratch { path } +} + +/// The directory the scratch directories live in, created if it is missing. +/// +/// Integration tests and benchmarks get `CARGO_TARGET_TMPDIR` from cargo, +/// which already follows `CARGO_TARGET_DIR`. Unit tests inside `src/` get no +/// such variable, so they use the location of the running test executable. +fn root() -> PathBuf { + let path = match option_env!("CARGO_TARGET_TMPDIR") { + Some(path) => PathBuf::from(path), + None => target_dir().join("tmp"), + }; + std::fs::create_dir_all(&path) + .unwrap_or_else(|err| panic!("cannot create {}: {err}", path.display())); + path +} + +fn target_dir() -> PathBuf { + if let Some(dir) = std::env::var_os("CARGO_TARGET_DIR") { + let dir = PathBuf::from(dir); + if dir.is_absolute() { + return dir.canonicalize().unwrap_or(dir); + } + } + // Cargo resolves relative target directories before changing to each + // crate's directory to run its tests. Resolve from the executable rather + // than interpreting that relative path a second time in the new cwd. + // This also follows target-dir settings in Cargo configuration files. + if let Ok(executable) = std::env::current_exe() { + if let Some(dir) = executable_target_dir(&executable) { + return dir; + } + } + // Fallback for runners that relocate test executables outside Cargo's + // profile/deps layout. Every crate lives at /crates/. + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); + manifest + .parent() + .and_then(Path::parent) + .unwrap_or(manifest) + .join("target") +} + +fn executable_target_dir(executable: &Path) -> Option { + if !executable.is_absolute() { + return None; + } + let deps = executable.parent()?; + if deps.file_name()? != "deps" { + return None; + } + Some(deps.parent()?.parent()?.to_path_buf()) +} + +#[cfg(test)] +mod target_directory_tests { + use super::*; + + #[test] + fn test_executable_layouts_produce_absolute_scratch_roots() { + let base = Path::new(env!("CARGO_MANIFEST_DIR")).join("rars-scratch-layout-probe"); + for profile in ["debug", "release", "custom-profile"] { + let executable = base.join(profile).join("deps").join("rars-tests"); + assert_eq!(executable_target_dir(&executable), Some(base.clone())); + } + let cross_target = base.join("aarch64-apple-darwin"); + assert_eq!( + executable_target_dir(&cross_target.join("debug/deps/rars-tests")), + Some(cross_target), + ); + assert_eq!( + executable_target_dir(Path::new("target/debug/deps/test")), + None + ); + assert_eq!(executable_target_dir(&base.join("relocated-tests")), None); + } +} diff --git a/arca-rar/vendor/rars/tests/write_progress.rs b/arca-rar/vendor/rars/tests/write_progress.rs new file mode 100644 index 00000000..781adb0b --- /dev/null +++ b/arca-rar/vendor/rars/tests/write_progress.rs @@ -0,0 +1,446 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::rar50::{self, ArchiveEntry, ArchiveExtras, FilterPolicy, WriterOptions}; +use rars::{ + ArchiveVersion, EntrySource, FeatureSet, WriteOperation, WriteProgressEvent, WriterResources, +}; +use std::io::{Cursor, Write}; +use std::sync::{Arc, Mutex}; + +#[test] +fn member_events_surround_work_and_emission_surrounds_output() { + for (level, solid, policy) in [ + (0, false, FilterPolicy::None), + (1, false, FilterPolicy::None), + (1, true, FilterPolicy::None), + (1, false, FilterPolicy::Auto), + ] { + let events = Arc::new(Mutex::new(Vec::new())); + let entries: Vec<_> = (0..8) + .map(|index| { + let events = events.clone(); + let data = if index == 3 { + Vec::new() + } else { + vec![b'a' + index as u8; 1024] + }; + ArchiveEntry::new( + format!("member-{index}").into_bytes(), + EntrySource::from_opener(data.len() as u64, move || { + events.lock().unwrap().push(('r', index)); + Ok(Box::new(Cursor::new(data.clone()))) + }), + ) + }) + .collect(); + let report = |event: WriteProgressEvent<'_>| { + let mut events = events.lock().unwrap(); + match event { + WriteProgressEvent::EntryStarted { index, .. } => events.push(('s', index)), + WriteProgressEvent::EntryFinished { index, .. } => events.push(('f', index)), + WriteProgressEvent::Advanced { + operation: WriteOperation::Compression, + completed_bytes, + total_bytes, + .. + } => { + assert!(completed_bytes <= total_bytes); + if let Some((_, previous)) = events.iter().rev().find(|(kind, _)| *kind == 'a') + { + assert!(completed_bytes as usize >= *previous); + } + events.push(('a', completed_bytes as usize)); + } + WriteProgressEvent::OperationStarted { + operation: WriteOperation::Emission, + .. + } => events.push(('e', 0)), + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Emission, + .. + } => events.push(('z', 0)), + _ => {} + } + }; + struct Output<'a>(&'a Mutex>); + impl Write for Output<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let mut events = self.0.lock().unwrap(); + assert!(events.contains(&('e', 0))); + assert!(!events.contains(&('z', 0))); + events.push(('w', 0)); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let scratch = scratch::case("writer-progress"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let mut features = FeatureSet::store_only(); + features.solid = solid; + rar50::write_streaming_archive_with_progress( + &entries, + WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(level), + ArchiveExtras::default().with_filter_policy(policy), + &resources, + Some(&report), + &mut Output(&events), + ) + .unwrap(); + let events = events.lock().unwrap(); + let position = |event| events.iter().position(|value| *value == event).unwrap(); + for index in 0..entries.len() { + assert_eq!( + events + .iter() + .filter(|value| **value == ('s', index)) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|value| **value == ('f', index)) + .count(), + 1 + ); + assert!(position(('s', index)) < position(('r', index))); + assert!(position(('r', index)) < position(('f', index))); + assert!(position(('f', index)) < position(('e', 0))); + } + assert!(position(('w', 0)) < position(('z', 0))); + } +} + +#[test] +fn cancellation_token_stops_input_preparation_without_a_reporter() { + use rars::{Error, WriteCancellation}; + use std::io::{Read, Seek, SeekFrom}; + struct Input { + data: Cursor>, + cancel: WriteCancellation, + } + impl Read for Input { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + let len = bytes.len().min(64 * 1024); + let count = self.data.read(&mut bytes[..len])?; + self.cancel.cancel(); + Ok(count) + } + } + impl Seek for Input { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.data.seek(from) + } + } + for (level, solid, policy) in [ + (0, false, FilterPolicy::None), + (1, false, FilterPolicy::None), + (1, true, FilterPolicy::None), + (1, false, FilterPolicy::Auto), + ] { + let scratch = scratch::case("cancel-preparation"); + let cancel = WriteCancellation::new(); + let resources = WriterResources::default() + .with_temp_dir(&*scratch) + .with_cancellation(cancel.clone()); + let entry = ArchiveEntry::new( + b"input".to_vec(), + EntrySource::from_opener(256 * 1024, move || { + Ok(Box::new(Input { + data: Cursor::new(vec![b'a'; 256 * 1024]), + cancel: cancel.clone(), + })) + }), + ); + let mut features = FeatureSet::store_only(); + features.solid = solid; + let mut output = Vec::new(); + let result = rar50::write_streaming_archive_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, features).with_compression_level(level), + ArchiveExtras::default().with_filter_policy(policy), + &resources, + &mut output, + ); + assert_eq!(result, Err(Error::Cancelled)); + assert!(output.is_empty()); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } +} + +#[test] +fn cancellation_during_output_preserves_partial_output_and_omits_completion() { + use rars::{Error, WriteCancellation, WriteProgress}; + use std::sync::atomic::{AtomicBool, Ordering}; + for via_callback in [false, true] { + let scratch = scratch::case("cancel-output"); + let cancel = WriteCancellation::new(); + let resources = WriterResources::default() + .with_temp_dir(&*scratch) + .with_cancellation(if via_callback { + WriteCancellation::new() + } else { + cancel.clone() + }); + struct Reporter { + cancel: WriteCancellation, + finished: AtomicBool, + } + impl WriteProgress for Reporter { + fn report(&self, event: WriteProgressEvent<'_>) { + if matches!( + event, + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Emission, + .. + } + ) { + self.finished.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.cancel.is_cancelled() + } + } + struct Output { + cancel: WriteCancellation, + bytes: Vec, + } + impl Write for Output { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.bytes.extend_from_slice(bytes); + self.cancel.cancel(); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let reporter = Reporter { + cancel: cancel.clone(), + finished: AtomicBool::new(false), + }; + let mut output = Output { + cancel, + bytes: Vec::new(), + }; + let entry = ArchiveEntry::new( + b"input".to_vec(), + EntrySource::from_bytes(vec![b'a'; 128 * 1024]), + ); + let result = rar50::write_streaming_archive_with_progress( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ArchiveExtras::default(), + &resources, + Some(&reporter), + &mut output, + ); + assert_eq!(result, Err(Error::Cancelled)); + assert!(!output.bytes.is_empty()); + assert!(output.bytes.len() < 128 * 1024); + assert!(!reporter.finished.load(Ordering::Relaxed)); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } +} + +#[test] +fn cancellation_interrupts_encrypted_volume_preparation() { + use rars::{Error, WriteCancellation}; + use std::io::{Read, Seek, SeekFrom}; + use std::sync::atomic::{AtomicUsize, Ordering}; + struct Input { + data: Cursor>, + cancel: Option, + } + impl Read for Input { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + let count = self.data.read(bytes)?; + if let Some(cancel) = &self.cancel { + cancel.cancel(); + } + Ok(count) + } + } + impl Seek for Input { + fn seek(&mut self, from: SeekFrom) -> std::io::Result { + self.data.seek(from) + } + } + struct NoVolumes; + impl rar50::VolumeSink for NoVolumes { + fn start_volume(&mut self, _: u64) -> rars::Result> { + panic!("cancelled encryption must not publish a volume"); + } + } + let scratch = scratch::case("cancel-volume-encryption"); + let cancel = WriteCancellation::new(); + let resources = WriterResources::default() + .with_temp_dir(&*scratch) + .with_cancellation(cancel.clone()); + let opens = AtomicUsize::new(0); + let entry = ArchiveEntry::new( + b"input".to_vec(), + EntrySource::from_opener(256 * 1024, move || { + Ok(Box::new(Input { + data: Cursor::new(vec![b'a'; 256 * 1024]), + cancel: (opens.fetch_add(1, Ordering::Relaxed) > 0).then(|| cancel.clone()), + })) + }), + ) + .with_password(b"secret".to_vec()); + let result = rar50::write_streaming_volumes_to( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(0), + ArchiveExtras::default(), + 128 * 1024, + &mut NoVolumes, + &resources, + ); + assert_eq!(result, Err(Error::Cancelled)); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); +} + +#[test] +fn callback_cancellation_during_encoding_and_recovery_is_typed() { + use rars::{Error, WriteProgress}; + use std::sync::atomic::{AtomicBool, Ordering}; + struct Reporter { + phase: WriteOperation, + cancel: AtomicBool, + } + impl WriteProgress for Reporter { + fn report(&self, event: WriteProgressEvent<'_>) { + if let WriteProgressEvent::Advanced { operation, .. } = event { + if operation == self.phase { + self.cancel.store(true, Ordering::Relaxed); + } + } + } + fn is_cancelled(&self) -> bool { + // A callback may deliver a one-shot request; it must not be lost + // while the recovery layer translates its I/O error. + self.cancel.swap(false, Ordering::Relaxed) + } + } + for phase in [WriteOperation::Compression, WriteOperation::Recovery] { + let scratch = scratch::case("cancel-codec-recovery"); + let resources = WriterResources::default().with_temp_dir(&*scratch); + let reporter = Reporter { + phase, + cancel: AtomicBool::new(false), + }; + let entry = ArchiveEntry::new( + b"input".to_vec(), + EntrySource::from_bytes(vec![b'a'; 128 * 1024]), + ); + let result = rar50::write_streaming_archive_with_progress( + &[entry], + WriterOptions::new(ArchiveVersion::Rar50, FeatureSet::store_only()) + .with_compression_level(if phase == WriteOperation::Compression { + 1 + } else { + 0 + }), + ArchiveExtras::default().with_recovery_percent(Some(5)), + &resources, + Some(&reporter), + &mut Vec::new(), + ); + assert_eq!(result, Err(Error::Cancelled)); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } +} + +#[test] +fn cancellation_on_final_compression_update_prevents_archive_and_volume_emission() { + use rars::{Error, WriteProgress}; + use std::sync::atomic::{AtomicBool, Ordering}; + + #[derive(Default)] + struct CancelFinalUpdate { + member_finished: AtomicBool, + cancelled: AtomicBool, + } + impl WriteProgress for CancelFinalUpdate { + fn report(&self, event: WriteProgressEvent<'_>) { + match event { + WriteProgressEvent::EntryFinished { .. } => { + self.member_finished.store(true, Ordering::Relaxed); + } + WriteProgressEvent::Advanced { + operation: WriteOperation::Compression, + .. + } if self.member_finished.load(Ordering::Relaxed) => { + self.cancelled.store(true, Ordering::Relaxed); + } + WriteProgressEvent::OperationFinished { + operation: WriteOperation::Compression, + .. + } + | WriteProgressEvent::OperationStarted { + operation: WriteOperation::Emission, + .. + } => panic!("cancelled preparation reported success or began emission"), + _ => {} + } + } + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } + } + + for version in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for volumes in [false, true] { + for size in [0, 1024] { + let scratch = scratch::case("cancel-final-compression-update"); + let resources = WriterResources::default() + .with_temp_dir(&*scratch) + .with_max_memory_bytes(8 * 1024 * 1024); + let entries = [ArchiveEntry::new( + b"file".to_vec(), + EntrySource::from_bytes(vec![42; size]), + )]; + let options = + WriterOptions::new(version, FeatureSet::store_only()).with_compression_level(0); + let reporter = CancelFinalUpdate::default(); + let mut output = Vec::new(); + let mut sink = rar50::CollectedVolumes::new(); + let result = if volumes { + rar50::write_streaming_volumes_with_progress( + &entries, + options, + ArchiveExtras::default(), + 512, + &mut sink, + &resources, + Some(&reporter), + ) + } else { + rar50::write_streaming_archive_with_progress( + &entries, + options, + ArchiveExtras::default(), + &resources, + Some(&reporter), + &mut output, + ) + }; + assert_eq!(result, Err(Error::Cancelled)); + assert!(reporter.member_finished.load(Ordering::Relaxed)); + assert!(reporter.cancelled.load(Ordering::Relaxed)); + assert!(output.is_empty()); + assert!(sink.take().is_empty()); + assert_eq!(resources.managed_memory_in_use(), 0); + assert_eq!(std::fs::read_dir(&*scratch).unwrap().count(), 0); + } + } + } +} diff --git a/arca-rar/vendor/rars/tests/writer_aggregate_limit.rs b/arca-rar/vendor/rars/tests/writer_aggregate_limit.rs new file mode 100644 index 00000000..aa430012 --- /dev/null +++ b/arca-rar/vendor/rars/tests/writer_aggregate_limit.rs @@ -0,0 +1,161 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; +use rars::{ArchiveReader, ArchiveVersion, Builder, ErrorKind, WriterResources}; + +fn builder(version: ArchiveVersion) -> Builder { + let mut builder = Builder::new(version).store(true); + for index in 0..4 { + builder + .add_bytes( + format!("member-{index}").into_bytes(), + vec![42; 4096], + None, + None, + ) + .unwrap(); + } + builder +} + +#[test] +fn aggregate_limit_preserves_output_and_releases_execution_owners() { + let root = scratch::case("aggregate-parity"); + for version in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for (store, solid, recovery) in [ + (true, false, false), + (false, false, false), + (false, true, true), + ] { + let builder = builder(version) + .store(store) + .solid(solid) + .recovery_percent(recovery.then_some(10)); + let expected = builder.to_bytes().unwrap(); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_memory_bytes(256 << 20); + for _ in 0..2 { + let output = builder.to_output(&resources, None).unwrap(); + assert_eq!(output.as_bytes(), expected); + assert!(resources.managed_memory_in_use() >= output.as_bytes().len() as u64); + ArchiveReader::read(output.as_bytes()) + .unwrap() + .test(None) + .unwrap(); + let bytes = output.into_vec(); + assert_eq!(resources.managed_memory_in_use(), 0); + assert_eq!(bytes, expected); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + } +} + +#[test] +fn failed_writes_release_the_group_and_preserve_published_files() { + let root = scratch::case("aggregate-refusal"); + let path = root.join("archive.rar"); + std::fs::write(&path, b"original").unwrap(); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_memory_bytes(1024); + for _ in 0..3 { + let error = builder(ArchiveVersion::Rar50) + .write_to_path_with_resources(&path, &resources, None) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!(resources.managed_memory_in_use(), 0); + assert_eq!(std::fs::read(&path).unwrap(), b"original"); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 1); + } +} + +#[test] +fn retained_volumes_count_until_copy_and_handoff() { + let builder = builder(ArchiveVersion::Rar50).volume_size(Some(2048)); + let expected = builder.build_volumes(None).unwrap(); + let resources = WriterResources::default().with_max_memory_bytes(16 << 20); + let output = builder.to_volume_output(&resources, None).unwrap(); + let retained = resources.managed_memory_in_use(); + let payload: u64 = output + .volumes() + .iter() + .map(|part| part.as_bytes().len() as u64) + .sum(); + assert!(retained >= payload); + let copied = output + .copy_with(|parts| { + assert_eq!(resources.managed_memory_in_use(), retained + payload); + parts + .iter() + .map(|part| part.as_bytes().to_vec()) + .collect::>() + }) + .unwrap(); + assert_eq!(copied, expected); + assert_eq!(resources.managed_memory_in_use(), retained); + assert_eq!(output.into_vec().unwrap(), expected); + assert_eq!(resources.managed_memory_in_use(), 0); +} + +#[test] +fn legacy_families_refuse_before_emission_even_with_zero_budget() { + for version in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let builder = builder(version); + for resources in [ + WriterResources::default().with_max_memory_bytes(0), + WriterResources::default().with_max_preparation_bytes(0), + ] { + let mut bytes = Vec::new(); + let error = builder.write_to(&mut bytes, &resources, None).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::UnsupportedFeature); + assert!(bytes.is_empty()); + assert_eq!( + builder + .clone() + .volume_size(Some(2048)) + .build_volumes_with_resources(&resources, None) + .unwrap_err() + .kind(), + ErrorKind::UnsupportedFeature + ); + assert_eq!(resources.managed_memory_in_use(), 0); + } + } +} + +#[test] +fn managed_builder_handles_encrypted_file_comment_and_releases_its_charge() { + let mut builder = Builder::new(ArchiveVersion::Rar50).store(true); + builder + .add_bytes(b"file".to_vec(), b"payload".to_vec(), None, None) + .unwrap(); + builder + .set_file_comment(b"file", Some(b"private comment".to_vec())) + .unwrap(); + builder + .set_entry_encryption(b"file", None, Some(b"password".to_vec())) + .unwrap(); + + let resources = WriterResources::default().with_max_memory_bytes(16 << 20); + let output = builder.to_output(&resources, None).unwrap(); + assert!(resources.managed_memory_in_use() >= output.as_bytes().len() as u64); + let archive = ArchiveReader::read(output.as_bytes()).unwrap(); + assert_eq!( + archive.read_member(b"file", None).unwrap().unwrap(), + b"payload" + ); + drop(archive); + drop(output); + assert_eq!(resources.managed_memory_in_use(), 0); +} diff --git a/arca-rar/vendor/rars/tests/writer_preparation_limit.rs b/arca-rar/vendor/rars/tests/writer_preparation_limit.rs new file mode 100644 index 00000000..abc0e140 --- /dev/null +++ b/arca-rar/vendor/rars/tests/writer_preparation_limit.rs @@ -0,0 +1,193 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; +use rars::{ArchiveReader, ArchiveVersion, Builder, Error, ErrorKind, WriterResources}; + +#[test] +fn preparation_limit_preserves_plain_output_and_releases_failed_writes() { + let root = scratch::case("preparation-byte-parity"); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for (store, solid, recovery) in [ + (true, false, false), + (false, false, false), + (false, true, true), + ] { + let mut builder = Builder::new(target) + .store(store) + .solid(solid) + .comment(Some(vec![b'c'; 4096])) + .archive_metadata(None, false, true) + .unwrap() + .recovery_percent(recovery.then_some(10)); + for index in 0..8 { + builder + .add_bytes( + format!("member-{index}").into_bytes(), + vec![42; 1024], + None, + None, + ) + .unwrap(); + } + let base = WriterResources::default().with_temp_dir(&*root); + let mut expected = Vec::new(); + builder.write_to(&mut expected, &base, None).unwrap(); + let denied = base.clone().with_max_preparation_bytes(0); + let mut output = Vec::new(); + assert!(matches!( + builder + .write_to(&mut output, &denied, None) + .unwrap_err() + .root_cause(), + Error::WriterPreparationLimitExceeded { limit: 0, .. } + )); + assert!(output.is_empty()); + let resources = base.with_max_preparation_bytes(1 << 20); + assert_eq!(resources.max_preparation_bytes(), Some(1 << 20)); + let mut unavailable: &mut [u8] = &mut []; + assert_eq!( + builder + .write_to(&mut unavailable, &resources, None) + .unwrap_err() + .kind(), + ErrorKind::Io + ); + for _ in 0..2 { + output.clear(); + builder.write_to(&mut output, &resources, None).unwrap(); + assert_eq!(output, expected); + ArchiveReader::read(&output).unwrap().test(None).unwrap(); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + } +} + +#[test] +fn legacy_writers_refuse_the_unsupported_hard_policy_before_emission() { + for target in [ + ArchiveVersion::Rar13, + ArchiveVersion::Rar14, + ArchiveVersion::Rar15, + ArchiveVersion::Rar20, + ArchiveVersion::Rar29, + ArchiveVersion::Rar30, + ArchiveVersion::Rar40, + ] { + let mut builder = Builder::new(target).store(true); + builder + .add_bytes(b"member".to_vec(), b"data".to_vec(), None, None) + .unwrap(); + let mut output = Vec::new(); + let error = builder + .write_to( + &mut output, + &WriterResources::default().with_max_preparation_bytes(u64::MAX), + None, + ) + .unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::UnsupportedFamilyFeature { + feature: "preparation memory quota", + .. + } + )); + assert!(output.is_empty()); + } +} + +#[test] +fn volume_headers_and_encryption_state_obey_preparation_policy() { + use rars::rar50::{ + write_streaming_volumes_to, ArchiveEntry, ArchiveExtras, CollectedVolumes, WriterOptions, + }; + let root = scratch::case("preparation-volumes"); + for encrypted in [false, true] { + let mut entry = ArchiveEntry::new( + b"member".to_vec(), + rars::EntrySource::from_bytes(vec![42; 8192]), + ); + if encrypted { + entry.password = Some(b"secret".to_vec()); + } + let entries = [entry]; + let mut options = WriterOptions::default(); + options.compression_level = Some(0); + options.features.header_encryption = encrypted; + let mut extras = ArchiveExtras::default(); + extras.recovery_percent = Some(10); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_preparation_bytes(1 << 20); + let mut sink = CollectedVolumes::new(); + write_streaming_volumes_to(&entries, options, extras, 2048, &mut sink, &resources).unwrap(); + let archives: Vec<_> = sink + .take() + .into_iter() + .map(|bytes| { + ArchiveReader::read_owned_with_options( + bytes, + rars::ArchiveReadOptions::with_optional_password( + encrypted.then_some(b"secret".as_slice()), + ), + ) + .unwrap() + }) + .collect(); + assert!(archives.len() > 1); + assert_eq!( + rars::read_volume_member_at(&archives, 0, encrypted.then_some(b"secret".as_slice())) + .unwrap() + .unwrap(), + vec![42; 8192] + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} + +#[test] +fn cancellation_after_preparation_releases_capacity_for_reuse() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct CancelAtEmission(AtomicBool); + impl rars::WriteProgress for CancelAtEmission { + fn report(&self, event: rars::WriteProgressEvent<'_>) { + if matches!( + event, + rars::WriteProgressEvent::OperationStarted { + operation: rars::WriteOperation::Emission, + .. + } + ) { + self.0.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + let root = scratch::case("preparation-cancel"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_preparation_bytes(16384); + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .archive_metadata(None, false, true) + .unwrap(); + builder + .add_bytes(b"member".to_vec(), vec![42; 1024], None, None) + .unwrap(); + for _ in 0..32 { + let progress = CancelAtEmission(AtomicBool::new(false)); + assert_eq!( + builder + .write_to(&mut Vec::new(), &resources, Some(&progress)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + builder.write_to(&mut Vec::new(), &resources, None).unwrap(); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} diff --git a/arca-rar/vendor/rars/tests/writer_spool_limit.rs b/arca-rar/vendor/rars/tests/writer_spool_limit.rs new file mode 100644 index 00000000..a9fd6fef --- /dev/null +++ b/arca-rar/vendor/rars/tests/writer_spool_limit.rs @@ -0,0 +1,398 @@ +#![cfg(feature = "write")] + +#[path = "support/scratch.rs"] +mod scratch; + +use rars::{ArchiveReader, ArchiveVersion, Builder, Error, ErrorKind, WriterResources}; +use std::io::Write; + +#[test] +fn spool_quota_does_not_change_successful_archive_bytes() { + let root = scratch::case("writer-spool-byte-parity"); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + for (stored, solid, recovery) in [ + (true, false, false), + (false, false, false), + (false, true, false), + (true, false, true), + (false, false, true), + ] { + let mut builder = Builder::new(target) + .store(stored) + .solid(solid) + .recovery_percent(recovery.then_some(10)); + builder + .add_bytes(b"first".to_vec(), b"test data ".repeat(512), None, None) + .unwrap(); + builder + .add_bytes(b"second".to_vec(), vec![42; 2048], None, None) + .unwrap(); + let unlimited = WriterResources::default().with_temp_dir(&*root); + let limited = unlimited.clone().with_max_spool_bytes(1 << 20); + let mut expected = Vec::new(); + let mut actual = Vec::new(); + builder.write_to(&mut expected, &unlimited, None).unwrap(); + builder.write_to(&mut actual, &limited, None).unwrap(); + assert_eq!( + actual, expected, + "{target:?}, stored={stored}, solid={solid}, recovery={recovery}" + ); + ArchiveReader::read_owned(actual) + .unwrap() + .test(None) + .unwrap(); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + } +} + +#[test] +fn spool_refusal_is_typed_and_resources_can_be_reused_after_failure() { + let root = scratch::case("writer-spool-refusal"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(0); + let mut builder = Builder::new(ArchiveVersion::Rar50); + builder + .add_bytes(b"member".to_vec(), vec![42; 16384], None, None) + .unwrap(); + let error = builder + .write_to(&mut Vec::new(), &resources, None) + .unwrap_err(); + assert!( + matches!( + error.root_cause(), + Error::WriterSpoolLimitExceeded { limit: 0, .. } + ), + "{error:?}" + ); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + // Plain stored output needs no nonempty spool. The quota is not an output cap. + let stored = builder.store(true); + let mut bytes = Vec::new(); + stored.write_to(&mut bytes, &resources, None).unwrap(); + assert!(bytes.len() > 16384); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn recovery_payload_growth_preserves_spool_error_details() { + let root = scratch::case("writer-spool-recovery"); + let mut builder = Builder::new(ArchiveVersion::Rar50) + .store(true) + .recovery_percent(Some(10)); + builder + .add_bytes(b"member".to_vec(), vec![42; 1024], None, None) + .unwrap(); + // Enough for the prefix; insufficient for prefix plus the recovery payload. + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(1200); + let error = builder + .write_to(&mut Vec::new(), &resources, None) + .unwrap_err(); + assert!( + matches!(error.root_cause(), Error::WriterSpoolLimitExceeded { limit: 1200, used, .. } if *used >= 1024), + "{error:?}" + ); + assert_eq!(error.kind(), ErrorKind::ResourceLimit); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn encrypted_volume_preparation_uses_the_shared_spool_quota() { + use rars::rar50::{ + write_streaming_volumes_to, ArchiveEntry, ArchiveExtras, CollectedVolumes, WriterOptions, + }; + let root = scratch::case("writer-spool-volumes"); + let mut entry = ArchiveEntry::new( + b"member".to_vec(), + rars::EntrySource::from_bytes(vec![42; 8192]), + ); + entry.password = Some(b"secret".to_vec()); + let entries = [entry]; + let limited = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(0); + let mut options = WriterOptions::default(); + options.compression_level = Some(0); + let error = write_streaming_volumes_to( + &entries, + options, + ArchiveExtras::default(), + 4096, + &mut CollectedVolumes::new(), + &limited, + ) + .unwrap_err(); + assert!( + matches!( + error.root_cause(), + Error::WriterSpoolLimitExceeded { limit: 0, .. } + ), + "{error:?}" + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + let sufficient = limited.with_max_spool_bytes(1 << 20); + let mut sink = CollectedVolumes::new(); + write_streaming_volumes_to( + &entries, + options, + ArchiveExtras::default(), + 4096, + &mut sink, + &sufficient, + ) + .unwrap(); + let archives: Vec<_> = sink + .take() + .into_iter() + .map(|bytes| ArchiveReader::read_owned(bytes).unwrap()) + .collect(); + assert!(archives.len() > 1); + let data = rars::read_volume_member_at(&archives, 0, Some(b"secret")) + .unwrap() + .unwrap(); + assert_eq!(data, vec![42; 8192]); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn recovery_io_bridge_preserves_resource_error_context() { + let expected = Error::WriterSpoolLimitExceeded { + limit: 10, + required: 12, + used: 8, + } + .at_entry(b"\xff".to_vec(), "writing"); + let recovery = rars::recovery::rar5::Error::from(std::io::Error::other(expected.clone())); + assert_eq!(Error::from(recovery), expected); +} + +#[test] +fn sink_failure_still_cleans_up_spools_under_a_quota() { + struct Refuse; + impl Write for Refuse { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::PermissionDenied.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let root = scratch::case("writer-spool-sink-failure"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(1 << 20); + let mut builder = Builder::new(ArchiveVersion::Rar50); + builder + .add_bytes(b"member".to_vec(), vec![42; 16384], None, None) + .unwrap(); + assert_eq!( + builder + .write_to(&mut Refuse, &resources, None) + .unwrap_err() + .kind(), + ErrorKind::Io + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + builder.write_to(&mut Vec::new(), &resources, None).unwrap(); +} + +#[test] +fn cancellation_after_preparation_releases_retained_spools() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct CancelAtEmission(AtomicBool); + impl rars::WriteProgress for CancelAtEmission { + fn report(&self, event: rars::WriteProgressEvent<'_>) { + if matches!( + event, + rars::WriteProgressEvent::OperationStarted { + operation: rars::WriteOperation::Emission, + .. + } + ) { + self.0.store(true, Ordering::Relaxed); + } + } + fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } + } + let root = scratch::case("writer-spool-cancel"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(1 << 20); + let mut builder = Builder::new(ArchiveVersion::Rar50); + builder + .add_bytes(b"member".to_vec(), vec![42; 16384], None, None) + .unwrap(); + let progress = CancelAtEmission(AtomicBool::new(false)); + assert_eq!( + builder + .write_to(&mut Vec::new(), &resources, Some(&progress)) + .unwrap_err() + .kind(), + ErrorKind::Cancelled + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + builder.write_to(&mut Vec::new(), &resources, None).unwrap(); +} + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +#[test] +fn native_file_spools_do_not_consume_the_memory_payload_quota() { + let root = scratch::case("writer-spool-native-memory"); + let resources = WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_memory_bytes(0) + .with_max_spool_bytes(1 << 20); + assert_eq!(resources.max_spool_memory_bytes(), Some(0)); + assert_eq!(resources.max_spool_bytes(), Some(1 << 20)); + let mut builder = Builder::new(ArchiveVersion::Rar50); + builder + .add_bytes(b"member".to_vec(), vec![42; 16384], None, None) + .unwrap(); + let mut output = Vec::new(); + builder.write_to(&mut output, &resources, None).unwrap(); + assert_eq!( + ArchiveReader::read_owned(output) + .unwrap() + .read_member(b"member", None) + .unwrap() + .unwrap(), + vec![42; 16384] + ); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); +} + +#[test] +fn quick_open_payload_obeys_exact_spool_limit_and_cleans_up() { + let root = scratch::case("writer-spool-quick-open"); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let mut builder = Builder::new(target) + .store(true) + .comment(Some(b"indexed comment".to_vec())) + .archive_metadata(None, false, true) + .unwrap(); + for index in 0..32 { + builder + .add_bytes( + format!("member-{index}").into_bytes(), + vec![42; 16], + None, + None, + ) + .unwrap(); + } + let resources = WriterResources::default().with_temp_dir(&*root); + let mut expected = Vec::new(); + builder.write_to(&mut expected, &resources, None).unwrap(); + let archive = ArchiveReader::read(&expected).unwrap(); + let size = archive + .as_rar50() + .unwrap() + .services() + .find(|service| service.name == b"QO") + .unwrap() + .packed_size(); + assert!(size > 0); + let limited = resources.clone().with_max_spool_bytes(size - 1); + let mut output = Vec::new(); + let error = builder.write_to(&mut output, &limited, None).unwrap_err(); + assert!(matches!( + error.root_cause(), + Error::WriterSpoolLimitExceeded { .. } + )); + assert!(output.is_empty(), "quick-open is prepared before emission"); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + let exact = resources.with_max_spool_bytes(size); + for _ in 0..2 { + output.clear(); + builder.write_to(&mut output, &exact, None).unwrap(); + assert_eq!(output, expected); + ArchiveReader::read(&output).unwrap().test(None).unwrap(); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + let recovery = builder.recovery_percent(Some(10)); + let mut recovered = Vec::new(); + recovery + .write_to( + &mut recovered, + &WriterResources::default() + .with_temp_dir(&*root) + .with_max_spool_bytes(1 << 20), + None, + ) + .unwrap(); + ArchiveReader::read(&recovered).unwrap().test(None).unwrap(); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} + +#[test] +fn prepared_header_quota_covers_retained_members_services_and_quick_open() { + let root = scratch::case("writer-prepared-header-quota"); + for target in [ArchiveVersion::Rar50, ArchiveVersion::Rar70] { + let mut builder = Builder::new(target) + .store(true) + .comment(Some(b"archive comment".to_vec())) + .archive_metadata(None, false, true) + .unwrap(); + for index in 0..3 { + builder + .add_bytes( + format!("member-{index}").into_bytes(), + vec![42; 16], + None, + None, + ) + .unwrap(); + } + let resources = WriterResources::default().with_temp_dir(&*root); + let mut expected = Vec::new(); + builder.write_to(&mut expected, &resources, None).unwrap(); + // Discover the exact aggregate admission boundary, including the QO header. + let mut limit = 0; + loop { + let limited = resources.clone().with_max_prepared_header_bytes(limit); + let mut output = Vec::new(); + match builder.write_to(&mut output, &limited, None) { + Ok(()) => { + assert_eq!(output, expected); + break; + } + Err(error) => { + let Error::WriterPreparedHeaderLimitExceeded { required, used, .. } = + error.root_cause() + else { + panic!("{error:?}") + }; + assert!(required > &limit); + assert!(used <= &limit); + limit = *required; + assert!(output.is_empty()); + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } + } + } + assert!(limit > 0); + let exact = resources.with_max_prepared_header_bytes(limit); + assert_eq!(exact.max_prepared_header_bytes(), Some(limit)); + let mut unavailable: &mut [u8] = &mut []; + assert_eq!( + builder + .write_to(&mut unavailable, &exact, None) + .unwrap_err() + .kind(), + ErrorKind::Io + ); + for _ in 0..2 { + let mut output = Vec::new(); + builder.write_to(&mut output, &exact, None).unwrap(); + assert_eq!(output, expected); + } + assert_eq!(std::fs::read_dir(&root).unwrap().count(), 0); + } +} diff --git a/arca-setup/src/app.rs b/arca-setup/src/app.rs index ab78b29d..b77850af 100644 --- a/arca-setup/src/app.rs +++ b/arca-setup/src/app.rs @@ -315,6 +315,9 @@ impl Setup { Action::Begin => self.begin(), Action::Customize => self.screen = Screen::Options, Action::Back => self.screen = Screen::Welcome, + // Closing halfway would leave a folder that is neither the old + // version nor the new one, and registry entries pointing into it. + Action::Close if self.screen == Screen::Installing => {} Action::Close => ctx.send_viewport_cmd(ViewportCommand::Close), Action::Finish => self.finish(ctx), Action::Releases => ctx.open_url(egui::OpenUrl::new_tab(RELEASES)), @@ -376,6 +379,9 @@ impl Setup { impl eframe::App for Setup { fn ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { let ctx = ui.ctx().clone(); + if self.screen == Screen::Installing && ctx.input(|i| i.viewport().close_requested()) { + ctx.send_viewport_cmd(ViewportCommand::CancelClose); + } if self.events.is_some() { self.advance(Instant::now()); if self.events.is_some() { diff --git a/arca-setup/src/engine/install.rs b/arca-setup/src/engine/install.rs index cbe551e0..c189bac1 100644 --- a/arca-setup/src/engine/install.rs +++ b/arca-setup/src/engine/install.rs @@ -1,6 +1,6 @@ use super::{ overall, payload, stamp_manifest, system, warn_on, Event, Job, INNO_LEFTOVERS, MANIFEST, - SHELL_DLL, VERSION, + VERSION, }; use std::{fs, path::Path}; use system::ctx; @@ -12,7 +12,6 @@ pub fn run(job: &Job, report: &mut dyn FnMut(Event)) -> Result<(), String> { fs::create_dir_all(dir).map_err(ctx(format!("creating {}", dir.display())))?; system::sweep_leftovers(dir); - let mut shell_moved = false; let mut size = 0u64; let total = files.len().max(1) as f32; for (index, file) in files.iter().enumerate() { @@ -27,10 +26,10 @@ pub fn run(job: &Job, report: &mut dyn FnMut(Event)) -> Result<(), String> { } else { &file.bytes }; - let moved = system::replace_file(&target, bytes)?; - if moved && file.path == Path::new(SHELL_DLL) { - shell_moved = true; - } + // A shell extension Explorer has loaded cannot be overwritten but can + // be renamed: the new one takes its place for every Explorer started + // from now on, and the old copy goes the next time this runs. + system::replace_file(&target, bytes)?; size += bytes.len() as u64; report(Event::Progress(overall(0, (index + 1) as f32 / total))); } @@ -78,9 +77,6 @@ pub fn run(job: &Job, report: &mut dyn FnMut(Event)) -> Result<(), String> { if !job.files_only { system::notify_changes(); - if shell_moved && !job.quiet_update { - system::restart_explorer(); - } } report(Event::Progress(overall(3, 0.8))); if job.quiet_update && !job.files_only { diff --git a/arca-setup/src/engine/system.rs b/arca-setup/src/engine/system.rs index 207d6811..8633ae1d 100644 --- a/arca-setup/src/engine/system.rs +++ b/arca-setup/src/engine/system.rs @@ -398,17 +398,32 @@ pub fn replace_file(target: &Path, bytes: &[u8]) -> Res { } } +// Removes the copies `move_aside` left of Arca's own files, and nothing else: +// the folder may be one somebody chose that holds files of their own. pub fn sweep_leftovers(dir: &Path) { - let Ok(entries) = fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - if entry.file_name().to_string_lossy().contains(".old-") { - let _ = fs::remove_file(entry.path()); + for known in super::KNOWN_FILES.iter().copied().chain([UNINSTALLER]) { + let target = dir.join(known); + let (Some(folder), Some(name)) = (target.parent(), target.file_name()) else { + continue; + }; + let Ok(entries) = fs::read_dir(folder) else { + continue; + }; + let prefix = format!("{}.old-", name.to_string_lossy()); + for entry in entries.flatten() { + if is_aside_copy(&entry.file_name().to_string_lossy(), &prefix) { + let _ = fs::remove_file(entry.path()); + } } } } +fn is_aside_copy(found: &str, prefix: &str) -> bool { + found + .strip_prefix(prefix) + .is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) +} + pub fn install_uninstaller(dir: &Path) -> Res<()> { let own = std::env::current_exe().map_err(ctx("the installer's own path"))?; let target = dir.join(UNINSTALLER); @@ -419,31 +434,6 @@ pub fn install_uninstaller(dir: &Path) -> Res<()> { replace_file(&target, &bytes).map(|_| ()) } -fn explorer_running() -> bool { - Command::new(system32("tasklist.exe")) - .args(["/FI", "IMAGENAME eq explorer.exe", "/NH"]) - .stdin(Stdio::null()) - .stderr(Stdio::null()) - .creation_flags(CREATE_NO_WINDOW) - .output() - .is_ok_and(|out| String::from_utf8_lossy(&out.stdout).contains("explorer.exe")) -} - -pub fn restart_explorer() { - let mut kill = Command::new(system32("taskkill.exe")); - kill.args(["/F", "/IM", "explorer.exe"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .creation_flags(CREATE_NO_WINDOW); - let _ = wait_for(kill, Duration::from_secs(15)); - thread::sleep(Duration::from_millis(1500)); - if !explorer_running() { - let root = std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into()); - let _ = Command::new(PathBuf::from(root).join("explorer.exe")).spawn(); - } -} - pub fn launch(exe: &Path, dir: &Path) { let _ = Command::new(exe).current_dir(dir).spawn(); } @@ -482,3 +472,37 @@ pub fn notify_changes() { arca_notify::associations_changed(); arca_notify::environment_changed(); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_numbered_copies_of_known_files_are_swept() { + let room = std::env::temp_dir().join(format!("arca-sweep-{}", std::process::id())); + fs::create_dir_all(room.join("Assets")).unwrap(); + let swept = [ + "arca.exe.old-1", + "unins000.exe.old-12", + "Assets/StoreLogo.png.old-3", + ]; + let kept = [ + "arca.exe", + "notes.old-1", + "report.old-2023.txt", + "arca.exe.old-", + "arca.exe.old-1a", + ]; + for name in swept.iter().chain(&kept) { + fs::write(room.join(name), b"x").unwrap(); + } + sweep_leftovers(&room); + for name in swept { + assert!(!room.join(name).exists(), "{name} should be gone"); + } + for name in kept { + assert!(room.join(name).exists(), "{name} should stay"); + } + fs::remove_dir_all(&room).unwrap(); + } +} diff --git a/arca-setup/src/engine/uninstall.rs b/arca-setup/src/engine/uninstall.rs index abd36f8f..72ecddaa 100644 --- a/arca-setup/src/engine/uninstall.rs +++ b/arca-setup/src/engine/uninstall.rs @@ -1,6 +1,5 @@ use super::{ - leftovers_error, overall, system, warn_on, Event, Job, INNO_LEFTOVERS, KNOWN_FILES, SHELL_DLL, - UNINSTALLER, + leftovers_error, overall, system, warn_on, Event, Job, INNO_LEFTOVERS, KNOWN_FILES, UNINSTALLER, }; use std::{fs, path::Path}; @@ -25,7 +24,7 @@ pub fn run(job: &Job, report: &mut dyn FnMut(Event)) -> Result<(), String> { } report(Event::Progress(overall(1, 1.))); - remove_files(dir, job.files_only, report)?; + remove_files(dir, report)?; report(Event::Progress(overall(2, 1.))); if !job.files_only { @@ -37,7 +36,7 @@ pub fn run(job: &Job, report: &mut dyn FnMut(Event)) -> Result<(), String> { Ok(()) } -fn remove_files(dir: &Path, files_only: bool, report: &mut dyn FnMut(Event)) -> Result<(), String> { +fn remove_files(dir: &Path, report: &mut dyn FnMut(Event)) -> Result<(), String> { let names: Vec<&str> = KNOWN_FILES .iter() .copied() @@ -45,18 +44,11 @@ fn remove_files(dir: &Path, files_only: bool, report: &mut dyn FnMut(Event)) -> .chain(INNO_LEFTOVERS) .collect(); let total = names.len() as f32; - let mut explorer_restarted = false; let mut left = Vec::new(); for (index, name) in names.iter().enumerate() { let path = dir.join(name); - if path.exists() && fs::remove_file(&path).is_err() { - if *name == SHELL_DLL && !files_only && !explorer_restarted { - system::restart_explorer(); - explorer_restarted = true; - } - if fs::remove_file(&path).is_err() && system::move_aside(&path).is_err() { - left.push(path); - } + if path.exists() && fs::remove_file(&path).is_err() && system::move_aside(&path).is_err() { + left.push(path); } report(Event::Progress(overall(2, (index + 1) as f32 / total))); } diff --git a/arca-tar/src/lib.rs b/arca-tar/src/lib.rs index 46b3f32c..caa22e58 100644 --- a/arca-tar/src/lib.rs +++ b/arca-tar/src/lib.rs @@ -28,6 +28,106 @@ fn octal(field: &[u8], what: &str) -> Result { Ok(v) } +/// A numeric field, in octal or, with the top bit of its first byte set, in the +/// base-256 GNU tar uses for sizes over 8 GiB and times before 1970. +fn number(field: &[u8], what: &str) -> Result { + let overflow = || Error::Limit(format!("{what}: overflow")); + match field.first() { + Some(&first) if first & 0x80 != 0 => { + // Big-endian, in two's complement over the whole field when the + // second bit says negative, with the marker bit aside otherwise. + if field.len() > 15 { + return Err(overflow()); + } + let negative = first & 0x40 != 0; + let mut v: i128 = 0; + for (i, &byte) in field.iter().enumerate() { + let byte = if i == 0 && !negative { + byte & 0x7f + } else { + byte + }; + v = v * 256 + i128::from(byte); + } + if negative { + v -= 1i128 << (8 * field.len()); + } + i64::try_from(v).map_err(|_| overflow()) + } + _ => i64::try_from(octal(field, what)?).map_err(|_| overflow()), + } +} + +fn unsigned(field: &[u8], what: &str) -> Result { + u64::try_from(number(field, what)?).map_err(|_| Error::Format(format!("{what}: negative"))) +} + +/// The most a pax header may hold. Real ones are a few hundred bytes; this is +/// room for a long path and every attribute tar knows, and no more. +const PAX_LIMIT: u64 = 64 * 1024; + +/// What the extension headers in front of an entry say about it. +#[derive(Default)] +struct Longer { + name: Option, + size: Option, + mtime: Option, +} + +impl Longer { + fn is_empty(&self) -> bool { + self.name.is_none() && self.size.is_none() && self.mtime.is_none() + } + + /// Reads pax records, each ` =\n` where the length + /// counts the whole record, itself included. + fn apply_pax(&mut self, mut data: &[u8]) -> Result<()> { + let bad = || Error::Format("a malformed pax record".into()); + while !data.is_empty() { + let space = data.iter().position(|&b| b == b' ').ok_or_else(bad)?; + let length: usize = std::str::from_utf8(&data[..space]) + .ok() + .and_then(|s| s.parse().ok()) + .ok_or_else(bad)?; + if length <= space + 1 || length > data.len() || data[length - 1] != b'\n' { + return Err(bad()); + } + let record = &data[space + 1..length - 1]; + let equals = record.iter().position(|&b| b == b'=').ok_or_else(bad)?; + let (key, value) = (&record[..equals], &record[equals + 1..]); + let value = std::str::from_utf8(value).map_err(|_| bad())?; + match key { + b"path" => self.name = Some(value.to_string()), + b"size" => self.size = Some(value.parse().map_err(|_| bad())?), + b"mtime" => { + let whole = value.split('.').next().unwrap_or(value); + self.mtime = Some(whole.parse().map_err(|_| bad())?); + } + _ => {} + } + data = &data[length..]; + } + Ok(()) + } +} + +/// A name without the `./` in front of it, however many times it is there. +fn without_current(name: &str) -> &str { + let mut name = name; + while let Some(rest) = name.strip_prefix("./") { + name = rest.trim_start_matches('/'); + } + if name == "." { + "" + } else { + name + } +} + +fn dangling() -> Error { + Error::Format("an extension header with no entry after it".into()) +} + fn write_octal(dest: &mut [u8], value: u64) { let width = dest.len() - 1; let s = format!("{value:0width$o}", width = width); @@ -54,6 +154,21 @@ fn text_field(field: &[u8]) -> String { String::from_utf8_lossy(&field[..end]).into_owned() } +/// The permissions to record for a file going into a tar: its own on Unix, so +/// a program stays runnable once extracted, and an ordinary file's elsewhere. +pub fn mode_of(meta: &std::fs::Metadata) -> u32 { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + meta.permissions().mode() & 0o777 + } + #[cfg(not(unix))] + { + let _ = meta; + 0o644 + } +} + /// Whether a 512-byte block is a TAR header with a valid checksum. pub fn is_header(block: &[u8]) -> bool { let Ok(header) = <&[u8; BLOCK]>::try_from(block) else { @@ -69,6 +184,21 @@ pub fn is_header(block: &[u8]) -> bool { pub struct TarEntry { pub entry: Entry, pub data: u64, + /// The Unix permission bits, `0o755` for a program. Zero when the field + /// cannot be read, which only ever costs the execute bit. + pub mode: u32, + /// The typeflag: `0` or NUL a file, `5` a folder, `1` a hard link, `2` a + /// symbolic link, `3` to `6` devices and pipes, `V` a volume label. + pub kind: u8, +} + +impl TarEntry { + /// Whether extracting the entry means writing a file or making a folder. + /// Links, devices, pipes and volume labels have no data of their own, and + /// writing them as files would leave empty files where a tar puts links. + pub fn is_plain(&self) -> bool { + self.entry.is_dir || matches!(self.kind, b'0' | 0 | b'7') + } } pub struct TarReader { @@ -91,59 +221,120 @@ impl TarReader { self.source } + /// The next file or folder. The headers GNU tar and pax put in front of + /// one to give it a longer name, a larger size or an exact time are read + /// and applied here, never returned as entries of their own. pub fn next_entry(&mut self) -> Result> { if self.finished { return Ok(None); } - let mut header = [0u8; BLOCK]; - match self.source.read_exact(&mut header) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { - self.finished = true; - return Ok(None); - } - Err(e) => return Err(Error::Io(e)), - } - self.pos += BLOCK as u64; + let (header, longer, name) = loop { + let mut longer = Longer::default(); + let header = loop { + let mut header = [0u8; BLOCK]; + // A tar that simply stops where a header would start is read as + // ended, as tar itself does; one that stops inside a header, or + // whose decompressor reports it cut short, is not. + if fill(&mut self.source, &mut header)? == 0 { + self.finished = true; + return if longer.is_empty() { + Ok(None) + } else { + Err(dangling()) + }; + } + self.pos += BLOCK as u64; - if header.iter().all(|&b| b == 0) { - self.finished = true; - return Ok(None); - } + if header.iter().all(|&b| b == 0) { + self.finished = true; + return if longer.is_empty() { + Ok(None) + } else { + Err(dangling()) + }; + } - let declared = octal(&header[148..156], "checksum")? as u32; - let actual = checksum(&header); - if declared != actual { - return Err(Error::Format(format!( - "invalid header checksum: {declared} declared, {actual} computed" - ))); - } + let declared = octal(&header[148..156], "checksum")? as u32; + let actual = checksum(&header); + if declared != actual { + return Err(Error::Format(format!( + "invalid header checksum: {declared} declared, {actual} computed" + ))); + } + let size = unsigned(&header[124..136], "size")?; + match header[156] { + b'L' => { + let name = self.payload(size, limits::MAX_NAME as u64 + 1)?; + longer.name = Some(text_field(&name)); + } + b'K' => { + self.payload(size, limits::MAX_NAME as u64 + 1)?; + } + b'x' => { + let records = self.payload(size, PAX_LIMIT)?; + longer.apply_pax(&records)?; + } + // The global header every `git archive` tarball starts with. + // Nothing in it names a file, so it is stepped over. + b'g' => self.payload(size, PAX_LIMIT).map(|_| ())?, + _ => break header, + } + }; - let base_name = text_field(&header[0..100]); - let prefix = if &header[257..262] == b"ustar" { - text_field(&header[345..500]) - } else { - String::new() - }; - let name = if prefix.is_empty() { - base_name - } else { - format!("{prefix}/{base_name}") + let name = match longer.name.take() { + Some(name) => name, + None => { + let base_name = text_field(&header[0..100]); + let prefix = if &header[257..262] == b"ustar" { + text_field(&header[345..500]) + } else { + String::new() + }; + if prefix.is_empty() { + base_name + } else { + format!("{prefix}/{base_name}") + } + } + }; + if name.len() > limits::MAX_NAME { + return Err(Error::Limit("name too long".into())); + } + // `tar -C folder .` names everything `./…` and adds `./` for the + // folder itself. The prefix says nothing, and the folder is wherever + // the archive is extracted to, so it is not an entry of its own. + let bare = without_current(&name); + if bare.is_empty() { + if header[156] == b'5' || name.ends_with('/') || name == "." { + let size = unsigned(&header[124..136], "size")?; + self.skip(longer.size.unwrap_or(size))?; + continue; + } + return Err(Error::Format("an entry without a name".into())); + } + break (header, longer, bare.to_string()); }; - if name.len() > limits::MAX_NAME { - return Err(Error::Limit("name too long".into())); - } - let size = octal(&header[124..136], "size")?; - let mtime = octal(&header[136..148], "mtime")? as i64; + let size = match longer.size { + Some(size) => size, + None => unsigned(&header[124..136], "size")?, + }; + let mtime = match longer.mtime { + Some(mtime) => mtime, + None => number(&header[136..148], "mtime")?, + }; let kind = header[156]; - let is_dir = kind == b'5' || name.ends_with('/'); - - if kind == b'L' || kind == b'K' { - return Err(Error::Unsupported( - "GNU tar long headers are not supported yet".into(), - )); + if matches!(kind, b'S' | b'M') { + return Err(Error::Unsupported(format!( + "'{name}' is a GNU tar {} entry", + if kind == b'S' { + "sparse" + } else { + "multivolume" + } + ))); } + let is_dir = kind == b'5' || name.ends_with('/'); let entry = Entry { // A tar name is bytes too, but this reader has always taken them as @@ -169,12 +360,49 @@ impl TarReader { Ok(Some(TarEntry { entry, data: if is_dir { 0 } else { size }, + mode: octal(&header[100..108], "mode").map_or(0, |m| (m & 0o7777) as u32), + kind, })) } + /// The data of an extension header, read whole and with its padding. Its + /// size comes from the archive, so it is held to `limit` before anything + /// is set aside for it. + fn payload(&mut self, size: u64, limit: u64) -> Result> { + if size > limit { + return Err(Error::Limit(format!("an extension header of {size} bytes"))); + } + let padded = size.div_ceil(BLOCK as u64) * BLOCK as u64; + let mut data = vec![0u8; padded as usize]; + self.source.read_exact(&mut data).map_err(|e| { + if e.kind() == io::ErrorKind::UnexpectedEof { + Error::Format("an extension header is truncated".into()) + } else { + Error::Io(e) + } + })?; + self.pos += padded; + data.truncate(size as usize); + Ok(data) + } + + /// Steps over `size` bytes of data and their padding. + fn skip(&mut self, size: u64) -> Result<()> { + let padded = size.div_ceil(BLOCK as u64) * BLOCK as u64; + let n = io::copy(&mut (&mut self.source).take(padded), &mut io::sink())?; + self.pos += n; + if n < padded { + return Err(Error::Io(truncated())); + } + Ok(()) + } + pub fn copy_data(&mut self, e: &TarEntry, dest: &mut W) -> Result { let n = io::copy(&mut (&mut self.source).take(e.data), dest)?; self.pos += n; + if n != e.data { + return Err(Error::Io(truncated())); + } let padding = (BLOCK - (e.data as usize % BLOCK)) % BLOCK; if padding > 0 { let mut discard = vec![0u8; padding]; @@ -247,6 +475,24 @@ fn truncated() -> io::Error { io::Error::new(io::ErrorKind::UnexpectedEof, "the TAR entry is truncated") } +/// Reads a whole block, or nothing at all: gives back 0 when the source ends +/// before the block starts and an error when it ends, or fails, inside it. +fn fill(source: &mut R, block: &mut [u8; BLOCK]) -> Result { + let mut filled = 0; + while filled < BLOCK { + match source.read(&mut block[filled..]) { + Ok(0) if filled == 0 => return Ok(0), + Ok(0) => { + return Err(Error::Format("the TAR archive ends inside a header".into())); + } + Ok(n) => filled += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(Error::Io(e)), + } + } + Ok(filled) +} + pub struct TarWriter { out: W, } @@ -262,39 +508,58 @@ impl TarWriter { size: u64, mtime: i64, mode: u32, - mut data: R, + data: R, ) -> Result<()> { - let (prefix, base) = split_name(name)?; - let mut header = [0u8; BLOCK]; - header[..base.len()].copy_from_slice(base.as_bytes()); - write_octal(&mut header[100..108], mode as u64); - write_octal(&mut header[108..116], 0); - write_octal(&mut header[116..124], 0); - write_octal(&mut header[124..136], size); - write_octal(&mut header[136..148], mtime.max(0) as u64); - header[156] = if name.ends_with('/') { b'5' } else { b'0' }; - header[257..262].copy_from_slice(b"ustar"); - header[262] = 0; - header[263..265].copy_from_slice(b"00"); - if !prefix.is_empty() { - header[345..345 + prefix.len()].copy_from_slice(prefix.as_bytes()); + if name.len() > limits::MAX_NAME { + return Err(Error::Limit(format!("name of {} bytes", name.len()))); + } + // What the ustar header cannot hold goes in a pax header in front of + // it, which every tar since 2001 reads: a name past its 255 bytes, a + // size from 8 GiB up, a time before 1970 or past 2242. + let mut pax = Vec::new(); + let (prefix, base) = match split_name(name) { + Some(parts) => parts, + None => { + pax.extend(pax_record("path", name)); + ("", cut_name(name, 100)) + } + }; + if size > OCTAL_11 { + pax.extend(pax_record("size", &size.to_string())); + } + if !(0..=OCTAL_11 as i64).contains(&mtime) { + pax.extend(pax_record("mtime", &mtime.to_string())); + } + if !pax.is_empty() { + let header = ustar_header( + &format!("PaxHeaders/{}", cut_name(base, 89)), + "", + pax.len() as u64, + mtime, + 0o644, + b'x', + ); + self.out.write_all(&header)?; + self.out.write_all(&pax)?; + self.pad(pax.len() as u64)?; } - let sum = checksum(&header); - let s = format!("{sum:06o}"); - header[148..154].copy_from_slice(s.as_bytes()); - header[154] = 0; - header[155] = b' '; - - self.out.write_all(&header)?; - let written = io::copy(&mut data, &mut self.out)?; + let kind = if name.ends_with('/') { b'5' } else { b'0' }; + self.out + .write_all(&ustar_header(base, prefix, size, mtime, mode, kind))?; + // No more than the size just recorded, whatever the source holds now. + let written = io::copy(&mut data.take(size), &mut self.out)?; if written != size { return Err(Error::Format(format!( "'{name}': {size} bytes declared but {written} written" ))); } - let padding = (BLOCK - (size as usize % BLOCK)) % BLOCK; + self.pad(size) + } + + fn pad(&mut self, size: u64) -> Result<()> { + let padding = (BLOCK - (size % BLOCK as u64) as usize) % BLOCK; if padding > 0 { - self.out.write_all(&vec![0u8; padding])?; + self.out.write_all(&[0u8; BLOCK][..padding])?; } Ok(()) } @@ -306,19 +571,78 @@ impl TarWriter { } } -fn split_name(name: &str) -> Result<(&str, &str)> { - if name.len() <= 100 { - return Ok(("", name)); +/// The largest number eleven octal digits hold: 8 GiB less one byte. +const OCTAL_11: u64 = 0o777_7777_7777; + +/// A ustar header. Numbers too large for their octal field are written in +/// the base-256 form GNU tar reads, beside the pax record that says the same. +fn ustar_header( + base: &str, + prefix: &str, + size: u64, + mtime: i64, + mode: u32, + kind: u8, +) -> [u8; BLOCK] { + let mut header = [0u8; BLOCK]; + header[..base.len()].copy_from_slice(base.as_bytes()); + write_octal(&mut header[100..108], u64::from(mode & 0o7777)); + write_octal(&mut header[108..116], 0); + write_octal(&mut header[116..124], 0); + if size > OCTAL_11 { + header[124] = 0x80; + header[128..136].copy_from_slice(&size.to_be_bytes()); + } else { + write_octal(&mut header[124..136], size); + } + write_octal( + &mut header[136..148], + mtime.clamp(0, OCTAL_11 as i64) as u64, + ); + header[156] = kind; + header[257..262].copy_from_slice(b"ustar"); + header[263..265].copy_from_slice(b"00"); + header[345..345 + prefix.len()].copy_from_slice(prefix.as_bytes()); + let sum = checksum(&header); + header[148..154].copy_from_slice(format!("{sum:06o}").as_bytes()); + header[154] = 0; + header[155] = b' '; + header +} + +/// A name cut to at most `max` bytes without splitting a character. +fn cut_name(name: &str, max: usize) -> &str { + let mut end = name.len().min(max); + while !name.is_char_boundary(end) { + end -= 1; } - let cut = name[..name.len().min(156)] - .rfind('/') - .ok_or_else(|| Error::Limit(format!("name of {} bytes with no split point", name.len())))?; - let (p, b) = name.split_at(cut); - let b = &b[1..]; - if p.len() > 155 || b.len() > 100 { - return Err(Error::Limit(format!("name too long for ustar: {name}"))); + &name[..end] +} + +/// ` =\n`, where the length counts the whole record, +/// its own digits included. +fn pax_record(key: &str, value: &str) -> Vec { + let body = key.len() + value.len() + 3; + let mut length = body + 1; + while length != body + length.to_string().len() { + length = body + length.to_string().len(); + } + format!("{length} {key}={value}\n").into_bytes() +} + +/// The name split into ustar's prefix and name fields, at a slash, or `None` +/// when it does not fit them. The split is on bytes, so a name of any script +/// is cut only at its ASCII slash. +fn split_name(name: &str) -> Option<(&str, &str)> { + if name.len() <= 100 { + return Some(("", name)); } - Ok((p, b)) + let bytes = name.as_bytes(); + let cut = bytes[..bytes.len().min(156)] + .iter() + .rposition(|&b| b == b'/')?; + let (p, b) = (&name[..cut], &name[cut + 1..]); + (p.len() <= 155 && b.len() <= 100 && !b.is_empty()).then_some((p, b)) } #[cfg(test)] @@ -427,4 +751,284 @@ mod tests { let _ = r.next_entry(); } } + + /// A header of `kind` for `name` holding `size` bytes, as GNU tar and + /// pax write their extension headers. + fn header(name: &str, kind: u8, size: u64) -> Vec { + let mut h = [0u8; BLOCK]; + h[..name.len()].copy_from_slice(name.as_bytes()); + write_octal(&mut h[100..108], 0o644); + write_octal(&mut h[124..136], size); + write_octal(&mut h[136..148], 1_700_000_000); + h[156] = kind; + h[257..262].copy_from_slice(b"ustar"); + let sum = checksum(&h); + write_octal(&mut h[148..155], u64::from(sum)); + h[155] = b' '; + h.to_vec() + } + + fn padded(data: &[u8]) -> Vec { + let mut out = data.to_vec(); + out.resize(data.len().div_ceil(BLOCK) * BLOCK, 0); + out + } + + fn pax(records: &[(&str, &str)]) -> Vec { + let mut out = Vec::new(); + for (key, value) in records { + let body = format!(" {key}={value}\n"); + let mut length = body.len() + 1; + while format!("{length}{body}").len() != length { + length += 1; + } + out.extend_from_slice(format!("{length}{body}").as_bytes()); + } + out + } + + type Listed = (String, u64, i64, Vec); + + fn names(tar: &[u8]) -> Result> { + let mut r = TarReader::new(tar); + let mut out = Vec::new(); + while let Some(e) = r.next_entry()? { + let mut data = Vec::new(); + r.copy_data(&e, &mut data)?; + out.push((e.entry.name, e.entry.size, e.entry.mtime.unwrap(), data)); + } + Ok(out) + } + + fn end() -> Vec { + vec![0u8; BLOCK * 2] + } + + #[test] + fn the_permissions_of_an_entry_are_read_and_a_bad_field_costs_only_them() { + let with_mode = |field: &[u8]| { + let mut h = header("tool", b'0', 0); + h[100..108].copy_from_slice(field); + h[148..156].copy_from_slice(b" "); + let sum = checksum(<&[u8; BLOCK]>::try_from(&h[..]).unwrap()); + write_octal(&mut h[148..155], u64::from(sum)); + h.extend(end()); + let mut r = TarReader::new(&h[..]); + r.next_entry().unwrap().unwrap().mode + }; + assert_eq!(with_mode(b"0000755\0"), 0o755); + assert_eq!(with_mode(b"0104755\0"), 0o4755); + assert_eq!(with_mode(b"07x5\0\0\0\0"), 0); + } + + #[test] + fn a_gnu_long_name_names_the_entry_after_it() { + let long = format!("{}/{}.txt", "d".repeat(120), "f".repeat(40)); + let mut tar = header("././@LongLink", b'L', long.len() as u64 + 1); + tar.extend(padded(format!("{long}\0").as_bytes())); + tar.extend(header("short-cut-name", b'0', 2)); + tar.extend(padded(b"hi")); + tar.extend(header("././@LongLink", b'K', 10)); + tar.extend(padded(b"target.txt")); + tar.extend(header("link", b'2', 0)); + tar.extend(end()); + let got = names(&tar).unwrap(); + assert_eq!(got[0].0, long); + assert_eq!(got[0].3, b"hi"); + assert_eq!(got[1].0, "link"); + assert_eq!(got.len(), 2); + } + + #[test] + fn pax_records_override_the_header_and_global_ones_are_skipped() { + let long = format!("{}/{}.bin", "p".repeat(150), "q".repeat(30)); + let global = pax(&[("comment", "made by git archive")]); + let mut tar = header("pax_global_header", b'g', global.len() as u64); + tar.extend(padded(&global)); + let local = pax(&[("path", &long), ("mtime", "1234567890.25"), ("size", "3")]); + tar.extend(header("PaxHeaders/x", b'x', local.len() as u64)); + tar.extend(padded(&local)); + tar.extend(header("cut", b'0', 0)); + tar.extend(padded(b"abc")); + tar.extend(end()); + let got = names(&tar).unwrap(); + assert_eq!(got.len(), 1); + assert_eq!(got[0].0, long); + assert_eq!((got[0].1, got[0].2), (3, 1_234_567_890)); + assert_eq!(got[0].3, b"abc"); + } + + #[test] + fn the_folder_tar_was_run_in_is_not_an_entry() { + let mut tar = header("./", b'5', 0); + tar.extend(header("./a.txt", b'0', 1)); + tar.extend(padded(b"a")); + tar.extend(header("././b/", b'5', 0)); + tar.extend(end()); + let got = names(&tar).unwrap(); + let got: Vec<&str> = got.iter().map(|e| e.0.as_str()).collect(); + assert_eq!(got, ["a.txt", "b/"]); + let mut nameless = header("", b'0', 0); + nameless.extend(end()); + assert!(names(&nameless).is_err()); + } + + #[test] + fn base_256_numbers_are_read() { + let mut field = [0u8; 12]; + field[0] = 0x80; + field[11] = 5; + field[7] = 1; + assert_eq!(number(&field, "size").unwrap(), (1i64 << 32) + 5); + let negative = [0xffu8; 12]; + assert_eq!(number(&negative, "mtime").unwrap(), -1); + assert!(unsigned(&negative, "size").is_err()); + let mut huge = [0u8; 12]; + huge[0] = 0x80; + huge[1] = 0x7f; + assert!(number(&huge, "size").is_err()); + } + + #[test] + fn broken_extension_headers_are_errors_not_panics() { + let name = b"long-name.txt\0"; + // A long name with nothing after it. + let mut alone = header("././@LongLink", b'L', name.len() as u64); + alone.extend(padded(name)); + assert!(names(&alone).is_err()); + let mut then_end = alone.clone(); + then_end.extend(end()); + assert!(names(&then_end).is_err()); + // Cut in the middle of its own data. + let mut cut = header("././@LongLink", b'L', 600); + cut.extend(vec![b'a'; 100]); + assert!(names(&cut).is_err()); + // Larger than any name may be. + let mut huge = header("././@LongLink", b'L', 100_000); + huge.extend(end()); + assert!(matches!(names(&huge), Err(Error::Limit(_)))); + // Pax records that do not add up. + for records in [ + &b"99 path=x\n"[..], + b"9 pathx\n", + b"x path=y\n", + b"10 path=xy", + b"3 a\n", + ] { + let mut tar = header("PaxHeaders/x", b'x', records.len() as u64); + tar.extend(padded(records)); + tar.extend(header("a.txt", b'0', 0)); + tar.extend(end()); + assert!(names(&tar).is_err(), "{}", String::from_utf8_lossy(records)); + } + } + + // A name of any script that is too long for ustar goes in a pax record and + // reads back whole; one that ustar can split is cut only at a slash, never + // inside a character. + #[test] + fn long_and_non_ascii_names_round_trip() { + let split = format!("{}/{}.txt", "ñ".repeat(77), "é".repeat(30)); + assert!(split.len() > 100 && split.len() < 256); + let long = format!("{}/{}", "日本".repeat(60), "ñ".repeat(80)); + let one_part = "x".repeat(300); + let mut w = TarWriter::new(Vec::new()); + for name in [&split, &long, &one_part] { + w.add(name, 3, 1_700_000_000, 0o755, &b"abc"[..]).unwrap(); + } + let tar = w.finish().unwrap(); + let got = names(&tar).unwrap(); + let got: Vec<&str> = got.iter().map(|e| e.0.as_str()).collect(); + assert_eq!(got, [split.as_str(), long.as_str(), one_part.as_str()]); + let mut r = TarReader::new(&tar[..]); + assert_eq!(r.next_entry().unwrap().unwrap().mode, 0o755); + } + + // A size from 8 GiB up does not fit eleven octal digits; it is written in + // base 256 and in a pax record, and reads back from either. + #[test] + fn sizes_and_times_past_ustar_fields_are_kept() { + let huge = (8u64 << 30) + 5; + let header = ustar_header("big", "", huge, -1, 0o644, b'0'); + assert_eq!(unsigned(&header[124..136], "size").unwrap(), huge); + assert!(is_header(&header)); + let record = pax_record("size", &huge.to_string()); + let mut longer = Longer::default(); + longer.apply_pax(&record).unwrap(); + assert_eq!(longer.size, Some(huge)); + // Lengths that cross a digit boundary still count themselves. + for value in ["x".repeat(4), "x".repeat(92), "x".repeat(993)] { + let record = pax_record("path", &value); + let digits = record.iter().position(|&b| b == b' ').unwrap(); + let length: usize = std::str::from_utf8(&record[..digits]) + .unwrap() + .parse() + .unwrap(); + assert_eq!(length, record.len()); + } + let mut w = TarWriter::new(Vec::new()); + w.add("old.txt", 1, -86_400, 0o644, &b"a"[..]).unwrap(); + let tar = w.finish().unwrap(); + assert_eq!(names(&tar).unwrap()[0].2, -86_400); + } + + // A source that grew after its size was taken writes the size recorded, + // not what it holds now, so the archive stays readable. + #[test] + fn a_source_that_grew_writes_only_its_recorded_size() { + let mut w = TarWriter::new(Vec::new()); + w.add("grew.txt", 3, 0, 0o644, &b"abcdef"[..]).unwrap(); + w.add("next.txt", 1, 0, 0o644, &b"z"[..]).unwrap(); + let tar = w.finish().unwrap(); + let got = names(&tar).unwrap(); + assert_eq!(got[0].3, b"abc"); + assert_eq!(got[1].0, "next.txt"); + let mut w = TarWriter::new(Vec::new()); + assert!(w.add("short.txt", 5, 0, 0o644, &b"ab"[..]).is_err()); + } + + // A tar that stops where a header would start is ended; one that stops + // inside a header or inside an entry's data is cut, even when the data + // is a whole number of blocks and so has no padding to trip over. + #[test] + fn truncation_is_an_error_and_a_missing_end_is_not() { + let mut w = TarWriter::new(Vec::new()); + w.add("a.bin", 1024, 0, 0o644, &[7u8; 1024][..]).unwrap(); + let tar = w.finish().unwrap(); + let entry = BLOCK + 1024; + assert_eq!(names(&tar[..entry]).unwrap().len(), 1); + assert!(names(&tar[..entry + 100]).is_err()); + assert!(names(&tar[..BLOCK + 512]).is_err()); + assert!(names(&tar[..300]).is_err()); + } + + // Links, devices and pipes are listed but are not files to write, and a + // sparse or multivolume entry is refused rather than written wrong. + #[test] + fn entry_kinds_are_told_apart() { + let mut tar = header("link", b'2', 0); + tar.extend(header("hard", b'1', 0)); + tar.extend(header("file", b'0', 0)); + tar.extend(header("dir/", b'5', 0)); + tar.extend(end()); + let mut r = TarReader::new(&tar[..]); + let mut plain = Vec::new(); + while let Some(e) = r.next_entry().unwrap() { + plain.push((e.entry.name.clone(), e.is_plain())); + } + assert_eq!( + plain, + [ + ("link".to_string(), false), + ("hard".to_string(), false), + ("file".to_string(), true), + ("dir/".to_string(), true), + ] + ); + for &kind in b"SM" { + let mut tar = header("odd", kind, 0); + tar.extend(end()); + assert!(matches!(names(&tar), Err(Error::Unsupported(_)))); + } + } } diff --git a/arca-xz/src/lib.rs b/arca-xz/src/lib.rs index e6411f63..e49059ce 100644 --- a/arca-xz/src/lib.rs +++ b/arca-xz/src/lib.rs @@ -340,7 +340,9 @@ fn dictionary(preset: u32) -> u64 { u64::from(LzmaOptions::with_preset(preset).dict_size) } -fn block_size(preset: u32) -> u64 { +/// How much uncompressed data each thread of a multithreaded LZMA2 encoder +/// takes at a time: three dictionaries, so cutting the stream costs little. +pub fn block_size(preset: u32) -> u64 { (3 * dictionary(preset)).max(1 << 20) } diff --git a/arca-zip/src/lib.rs b/arca-zip/src/lib.rs index 30dd0d0c..0f33d347 100644 --- a/arca-zip/src/lib.rs +++ b/arca-zip/src/lib.rs @@ -17,6 +17,7 @@ const SIG_EOCD64: u32 = 0x0606_4b50; const SIG_LOC64: u32 = 0x0706_4b50; const SIG_CD: u32 = 0x0201_4b50; const SIG_LFH: u32 = 0x0403_4b50; +const SIG_DESCRIPTOR: u32 = 0x0807_4b50; const EOCD_MIN: usize = 22; const CD_FIXED: usize = 46; @@ -59,6 +60,53 @@ impl Write for CrcWriter { pub struct ZipArchive { source: R, entries_list: Vec, + kept: Vec, +} + +/// What the central directory says about an entry beyond what `Entry` holds, +/// carried through a rewrite so an entry nobody touched comes out the way it +/// went in: its attributes, its flags, and every extra field except the Zip64 +/// one, whose offset stops being true the moment anything moves. +#[derive(Clone, Debug, Default)] +struct Kept { + made_by: u16, + needed: u16, + flags: u16, + method: u16, + time: u16, + date: u16, + internal: u16, + external: u32, + extra: Vec, + comment: Vec, +} + +const EXTRA_ZIP64: u16 = 0x0001; +const EXTRA_UNICODE_PATH: u16 = 0x7075; + +/// The extra fields of `extra` without the ones `drop` names. A field whose +/// declared length runs past the end is where the parse stops, and it is not +/// kept: no reader could have made sense of it either. +fn strip_extra(extra: &[u8], drop: &[u16]) -> Vec { + let mut out = Vec::with_capacity(extra.len()); + let mut at = 0usize; + while extra.len() - at >= 4 { + let id = u16::from_le_bytes([extra[at], extra[at + 1]]); + let len = u16::from_le_bytes([extra[at + 2], extra[at + 3]]) as usize; + let end = at + 4 + len; + if end > extra.len() { + break; + } + if !drop.contains(&id) { + out.extend_from_slice(&extra[at..end]); + } + at = end; + } + out +} + +fn has_extra(extra: &[u8], id: u16) -> bool { + strip_extra(extra, &[id]).len() != strip_extra(extra, &[]).len() } impl ZipArchive { @@ -126,13 +174,22 @@ impl ZipArchive { source.read_exact(&mut cd)?; let mut entries_list = Vec::with_capacity(n_entries.min(4096) as usize); + let mut kept = Vec::with_capacity(entries_list.capacity()); let mut c = Cursor::new(&cd); - for i in 0..n_entries { - if c.remaining() < CD_FIXED { - break; + // Every header the directory holds, not only as many as the end record + // counts: a writer that stores more than 65,535 entries without Zip64 + // records the count modulo 65,536, and an archive listed short would + // lose the rest the first time it is rewritten. + while c.remaining() >= CD_FIXED { + let i = entries_list.len(); + if i as u64 >= limits::MAX_ENTRIES { + return Err(Error::Limit(format!("more than {i} entries"))); } - match read_central_header(&mut c) { - Ok(e) => entries_list.push(e), + match read_central(&mut c) { + Ok((e, k)) => { + entries_list.push(e); + kept.push(k); + } Err(e) => { return Err(Error::Format(format!( "entry {i} of the central directory: {e}" @@ -140,10 +197,18 @@ impl ZipArchive { } } } + let found = entries_list.len() as u64; + if found != n_entries && found % 0x1_0000 != n_entries % 0x1_0000 { + return Err(Error::Format(format!( + "the central directory holds {found} entries where its end record says {n_entries}" + ))); + } + reject_overlaps(&entries_list)?; Ok(ZipArchive { source, entries_list, + kept, }) } @@ -413,11 +478,11 @@ fn extract_inner( if let Crypt::Zip { check } = crypt { let pw = need_password(e, password)?; let reader = zipcrypto::open(&mut bounded, pw, check, &e.name)?; - decompress_into(reader, e.method, &mut cw)?; + decompress_into(reader, e.method, &mut cw, e.size)?; } else if matches!(crypt, Crypt::Aes) { let (keys, cipher_len) = unlock(&mut bounded, e, password)?; let reader = aes::AesReader::new(&mut bounded, &keys, cipher_len)?; - let reader = decompress_into(reader, e.method, &mut cw)?; + let reader = decompress_into(reader, e.method, &mut cw, e.size)?; let computed = reader.finish()?; // The authentication code sits after the ciphertext, so it can only be @@ -432,21 +497,53 @@ fn extract_inner( }); } } else { - decompress_into(bounded, e.method, &mut cw)?; + decompress_into(bounded, e.method, &mut cw, e.size)?; } let (_, crc_val, written) = cw.finalize(); Ok((crc_val, written)) } +/// Refuses entries whose data share bytes. No writer produces them; they are +/// how a few kilobytes are made to extract to terabytes, every entry naming +/// the same compressed data. Each entry spans at least its fixed local header +/// and its compressed size from where it starts. +fn reject_overlaps(entries: &[Entry]) -> Result<()> { + let mut spans: Vec<(u64, u64)> = entries + .iter() + .map(|e| { + let end = e + .offset + .saturating_add(LFH_FIXED as u64) + .saturating_add(e.compressed_size); + (e.offset, end) + }) + .collect(); + spans.sort_unstable(); + for pair in spans.windows(2) { + if pair[1].0 < pair[0].1 { + return Err(Error::Format(format!( + "two entries share the data at offset {}", + pair[1].0 + ))); + } + } + Ok(()) +} + // Hands the reader back afterwards, which is what lets the encrypted path get // its AES reader out to ask it for the authentication code. A decompressor may // stop before the end of its input, so the reader is not necessarily drained. +// Nothing past one byte more than the entry declares is decompressed: that +// byte is enough to tell the entry lied, and a bomb stops there instead of +// when the disk is full. fn decompress_into( src: Rd, method_code: Method, cw: &mut CrcWriter, + declared: u64, ) -> Result { + let limit = declared.saturating_add(1); match method_code { Method::Lzma | Method::Lzma2 | Method::Bzip2 | Method::Other7z => { Err(Error::Unsupported(format!("{} in ZIP", method_code.name()))) @@ -456,19 +553,19 @@ fn decompress_into( )), Method::Store => { let mut a = src; - io::copy(&mut a, cw)?; + io::copy(&mut (&mut a).take(limit), cw)?; Ok(a) } Method::Deflate => { let mut dec = flate2::read::DeflateDecoder::new(src); - io::copy(&mut dec, cw)?; + io::copy(&mut (&mut dec).take(limit), cw)?; Ok(dec.into_inner()) } Method::Zstd => { #[cfg(feature = "codecs-native")] { let mut dec = zstd::stream::read::Decoder::new(src).map_err(Error::Io)?; - io::copy(&mut dec, cw)?; + io::copy(&mut (&mut dec).take(limit), cw)?; Ok(dec.finish().into_inner()) } #[cfg(not(feature = "codecs-native"))] @@ -641,11 +738,17 @@ fn read_times(extra: &[u8]) -> Times { out } +#[cfg(test)] fn read_central_header(c: &mut Cursor<'_>) -> Result { + read_central(c).map(|(e, _)| e) +} + +fn read_central(c: &mut Cursor<'_>) -> Result<(Entry, Kept)> { if c.u32le("central directory signature")? != SIG_CD { return Err(Error::Format("invalid entry signature".into())); } - c.skip(4, "versions")?; + let made_by = c.u16le("version made by")?; + let needed = c.u16le("version needed")?; let flags = c.u16le("flags")?; let method_code = c.u16le("method")?; let time_val = c.u16le("time")?; @@ -656,7 +759,8 @@ fn read_central_header(c: &mut Cursor<'_>) -> Result { let n_len = c.u16le("name length")? as usize; let x_len = c.u16le("extra field length")? as usize; let k_len = c.u16le("comment length")? as usize; - c.skip(4, "disk and internal attributes")?; + c.skip(2, "disk number")?; + let internal = c.u16le("internal attributes")?; let ext_attr = c.u32le("external attributes")?; let mut offset = c.u32le("local offset")? as u64; @@ -669,7 +773,7 @@ fn read_central_header(c: &mut Cursor<'_>) -> Result { let name_bytes = c.bytes(n_len, "name")?; let extra = c.bytes(x_len, "extra field")?; - c.skip(k_len, "comment")?; + let comment = c.bytes(k_len, "comment")?; if uncompressed == 0xFFFF_FFFF || comp_size == 0xFFFF_FFFF || offset == 0xFFFF_FFFF { let mut x = Cursor::new(extra); @@ -740,7 +844,7 @@ fn read_central_header(c: &mut Cursor<'_>) -> Result { let is_dir = name.ends_with('/') || name.ends_with('\\'); let times = read_times(extra); - Ok(Entry { + let entry = Entry { name, raw_name: name_bytes.to_vec(), utf8, @@ -761,7 +865,20 @@ fn read_central_header(c: &mut Cursor<'_>) -> Result { offset, encrypted, zipcrypto, - }) + }; + let kept = Kept { + made_by, + needed, + flags, + method: method_code, + time: time_val, + date: date_val, + internal, + external: ext_attr, + extra: strip_extra(extra, &[EXTRA_ZIP64]), + comment: comment.to_vec(), + }; + Ok((entry, kept)) } struct Record { @@ -776,6 +893,9 @@ struct Record { is_directory: bool, encrypted: bool, real_method: u16, + // Set when the entry came out of another archive, and then it wins over + // everything above that the central directory would otherwise make up. + kept: Option, } pub struct ZipWriter { @@ -998,6 +1118,125 @@ impl ZipWriter { ) } + // Copies an entry of another archive across as it stands: its local header, + // its compressed and possibly encrypted bytes, and what the central + // directory said about it. Nothing is decrypted, so no password is needed + // and an entry under the old PKWARE cipher stays under it. With `rename` + // the name is the one thing that changes. + // + // An entry streamed with a data descriptor keeps flag bit 3: when it is set + // the old cipher checks a password against the time instead of the CRC. + // The descriptor is written afresh from the central directory's values + // rather than parsed out of the source. + fn copy_raw( + &mut self, + source: &mut R, + e: &Entry, + kept: &Kept, + rename: Option<&str>, + ) -> Result<()> { + source.seek(SeekFrom::Start(e.offset))?; + let mut lfh = [0u8; LFH_FIXED]; + source.read_exact(&mut lfh)?; + let mut c = Cursor::new(&lfh); + if c.u32le("local header signature")? != SIG_LFH { + return Err(Error::Format(format!( + "'{}': no local header at offset {}", + e.name, e.offset + ))); + } + c.skip(2, "version")?; + let local_flags = c.u16le("flags")?; + c.skip(18, "method, time, checksum and sizes")?; + let n_len = c.u16le("name length")? as usize; + let x_len = c.u16le("extra field length")? as usize; + let mut tail = vec![0u8; n_len + x_len]; + source.read_exact(&mut tail)?; + let local_extra = &tail[n_len..]; + + let (header, name_bytes, flags, cd_extra) = match rename { + None => { + let mut h = lfh.to_vec(); + h.extend_from_slice(&tail); + (h, e.raw_name.clone(), kept.flags, kept.extra.clone()) + } + Some(n) => { + let name_bytes = check_name(n)?; + let x = strip_extra(local_extra, &[EXTRA_UNICODE_PATH]); + let mut h = lfh[..26].to_vec(); + h[6..8].copy_from_slice(&(local_flags | 0x800).to_le_bytes()); + h.extend_from_slice(&(name_bytes.len() as u16).to_le_bytes()); + h.extend_from_slice(&(x.len() as u16).to_le_bytes()); + h.extend_from_slice(&name_bytes); + h.extend_from_slice(&x); + let cd_extra = strip_extra(&kept.extra, &[EXTRA_UNICODE_PATH]); + (h, name_bytes, kept.flags | 0x800, cd_extra) + } + }; + + let offset = self.pos; + self.out.write_all(&header)?; + let copied = io::copy(&mut source.take(e.compressed_size), &mut self.out)?; + if copied != e.compressed_size { + return Err(Error::Format(format!( + "'{}': the archive ends inside this entry", + e.name + ))); + } + let crc_val = e.crc32.unwrap_or(0); + let mut written = header.len() as u64 + copied; + if local_flags & 8 != 0 { + let mut d = Vec::with_capacity(24); + d.extend_from_slice(&SIG_DESCRIPTOR.to_le_bytes()); + d.extend_from_slice(&crc_val.to_le_bytes()); + if has_extra(local_extra, EXTRA_ZIP64) { + d.extend_from_slice(&e.compressed_size.to_le_bytes()); + d.extend_from_slice(&e.size.to_le_bytes()); + } else { + d.extend_from_slice(&(e.compressed_size as u32).to_le_bytes()); + d.extend_from_slice(&(e.size as u32).to_le_bytes()); + } + self.out.write_all(&d)?; + written += d.len() as u64; + } + self.pos = offset + written; + self.registros.push(Record { + name_str: name_bytes, + crc_val, + comp_size: e.compressed_size, + uncompressed: e.size, + offset, + method_code: kept.method, + date_val: kept.date, + time_val: kept.time, + is_directory: e.is_dir, + encrypted: false, + real_method: kept.method, + kept: Some(Kept { + flags, + extra: cd_extra, + ..kept.clone() + }), + }); + Ok(()) + } + + // Gives the entry just written the attributes, extra fields and comment + // `kept` carried in another archive, for an entry that had to be copied + // through `copy_entry` because its encryption changed. Its flags and its + // AES field are the new ones. + fn keep_metadata(&mut self, kept: &Kept) { + if let Some(r) = self.registros.last_mut() { + r.kept = Some(Kept { + needed: 45, + flags: if r.encrypted { 0x0801 } else { 0x0800 }, + method: r.method_code, + extra: strip_extra(&kept.extra, &[EXTRA_ZIP64, aes::EXTRA_AE]), + ..kept.clone() + }); + } + } + #[allow(clippy::too_many_arguments)] fn put_block( &mut self, @@ -1099,6 +1338,7 @@ impl ZipWriter { is_directory, encrypted, real_method: method_code.code()?, + kept: None, }); Ok(()) } @@ -1169,15 +1409,41 @@ impl ZipWriter { // The reader looks the AES field up in the central directory, so it // has to be here too and not only in the local header. let mut extra = extra; + let zip64 = !extra.is_empty(); if r.encrypted { extra.extend_from_slice(&aes::extra_field(r.real_method)); } - let flags: u16 = if r.encrypted { 0x0801 } else { 0x0800 }; + let (made_by, needed, flags, internal, external, comment) = match &r.kept { + Some(k) => { + extra.extend_from_slice(&k.extra); + let needed = if zip64 { k.needed.max(45) } else { k.needed }; + ( + k.made_by, + needed, + k.flags, + k.internal, + k.external, + &k.comment[..], + ) + } + None => { + let mode: u32 = if r.is_directory { 0o040755 } else { 0o100644 }; + let external = (mode << 16) | if r.is_directory { 0x10 } else { 0 }; + let flags: u16 = if r.encrypted { 0x0801 } else { 0x0800 }; + (0x031E, 45, flags, 0, external, &[][..]) + } + }; + if extra.len() > u16::MAX as usize { + return Err(Error::Limit(format!( + "extra field of {} bytes", + extra.len() + ))); + } let mut h = Vec::with_capacity(CD_FIXED + r.name_str.len() + extra.len()); h.extend_from_slice(&SIG_CD.to_le_bytes()); - h.extend_from_slice(&(0x031Eu16).to_le_bytes()); - h.extend_from_slice(&45u16.to_le_bytes()); + h.extend_from_slice(&made_by.to_le_bytes()); + h.extend_from_slice(&needed.to_le_bytes()); h.extend_from_slice(&flags.to_le_bytes()); h.extend_from_slice(&r.method_code.to_le_bytes()); h.extend_from_slice(&r.time_val.to_le_bytes()); @@ -1187,16 +1453,14 @@ impl ZipWriter { h.extend_from_slice(&(r.uncompressed.min(0xFFFF_FFFF) as u32).to_le_bytes()); h.extend_from_slice(&(r.name_str.len() as u16).to_le_bytes()); h.extend_from_slice(&(extra.len() as u16).to_le_bytes()); + h.extend_from_slice(&(comment.len() as u16).to_le_bytes()); h.extend_from_slice(&0u16.to_le_bytes()); - h.extend_from_slice(&0u16.to_le_bytes()); - h.extend_from_slice(&0u16.to_le_bytes()); - let mode: u32 = if r.is_directory { 0o040755 } else { 0o100644 }; - h.extend_from_slice( - &((mode << 16) | if r.is_directory { 0x10 } else { 0 }).to_le_bytes(), - ); + h.extend_from_slice(&internal.to_le_bytes()); + h.extend_from_slice(&external.to_le_bytes()); h.extend_from_slice(&(r.offset.min(0xFFFF_FFFF) as u32).to_le_bytes()); h.extend_from_slice(&r.name_str); h.extend_from_slice(&extra); + h.extend_from_slice(comment); self.out.write_all(&h)?; cd_size += h.len() as u64; @@ -1204,7 +1468,7 @@ impl ZipWriter { let n = self.registros.len() as u64; let necesita_z64 = - n > u16::MAX as u64 || cd_offset >= 0xFFFF_FFFF || cd_size >= 0xFFFF_FFFF; + n >= u16::MAX as u64 || cd_offset >= 0xFFFF_FFFF || cd_size >= 0xFFFF_FFFF; if necesita_z64 { let z_off = cd_offset + cd_size; @@ -1343,8 +1607,8 @@ fn compress_stream( fn check_name(name_str: &str) -> Result> { let b = name_str.as_bytes().to_vec(); - if b.len() > u16::MAX as usize { - return Err(Error::Limit("name too long for ZIP".into())); + if b.len() > limits::MAX_NAME { + return Err(Error::Limit(format!("name of {} bytes", b.len()))); } Ok(b) } @@ -1548,20 +1812,38 @@ fn rewrite( use std::fs::File; use std::io::{BufReader, BufWriter}; - let entries: Vec = ZipArchive::open(File::open(archive)?)? - .entries() - .iter() - .filter(|e| keep(e)) - .cloned() + let source = ZipArchive::open(File::open(archive)?)?; + // When the password stays what it was, every entry that survives is copied + // as it stands, encrypted or not: a plain entry in an archive with a + // password stays plain, an old PKWARE one is not turned into AES, and + // nothing has to be decrypted to move it. The password, if there is one, + // is only for the files being added, so it has to be the right one. + let raw = current == new; + if let (true, Some(pw)) = (raw, current) { + check_password(&mut File::open(archive)?, &source.entries_list, pw)?; + } + let entries: Vec<(Entry, Kept)> = source + .entries_list + .into_iter() + .zip(source.kept) + .filter(|(e, _)| keep(e)) .collect(); let total = entries.len() + extra.len(); let mut w = ZipWriter::new(BufWriter::with_capacity(STREAM_BUF, File::create(out)?)); let mut bytes = 0u64; + let mut src = BufReader::with_capacity(STREAM_BUF, File::open(archive)?); - for (i, e) in entries.iter().enumerate() { + for (i, (e, k)) in entries.iter().enumerate() { if !notify(i, total, &e.name) { return Err(Error::Cancelled); } + if raw { + let renamed = name(e); + let rename = (renamed != e.name).then_some(renamed.as_str()); + w.copy_raw(&mut src, e, k, rename)?; + bytes += e.size; + continue; + } // Directories hold nothing, and no other tool encrypts them either. let pw = if e.is_dir { None } else { new }; let crc = if e.encrypted && e.crc32 == Some(0) { @@ -1571,11 +1853,11 @@ fn rewrite( e.crc32 .ok_or_else(|| Error::Format("missing ZIP checksum".into()))? }; - let mut src = BufReader::with_capacity(STREAM_BUF, File::open(archive)?); let stamp = dos_stamp(&mut src, e)?; w.copy_entry(&name(e), crc, e.size, e.method, stamp, pw, |sink| { copy_compressed(&mut src, e, sink, current) })?; + w.keep_metadata(k); bytes += e.size; } @@ -1633,8 +1915,18 @@ fn rewrite( .filter(|e| !e.is_dir) .cloned() .collect(); + // An encrypted entry copied across without a password cannot be opened to + // check; what can be checked is that its header is where the directory + // says and that all of its bytes are there. written.par_iter().try_for_each(|e| -> Result<()> { let mut source = BufReader::with_capacity(STREAM_BUF, File::open(out)?); + if e.encrypted && new.is_none() { + let (mut bounded, _) = open_data(&mut source, e)?; + if io::copy(&mut bounded, &mut io::sink())? != e.compressed_size { + return Err(Error::Format(format!("'{}' came out short", e.name))); + } + return Ok(()); + } extract_entry_with(&mut source, e, io::sink(), new)?; Ok(()) })?; @@ -2182,6 +2474,246 @@ mod tests { w.finish().unwrap().into_inner() } + #[test] + fn an_entry_that_inflates_past_its_size_stops_one_byte_later() { + let body = vec![0u8; 1 << 20]; + let mut w = ZipWriter::new(IoCursor::new(Vec::new())); + w.add("big.bin", &body[..], Codec::Deflate, Level::NORMAL, None) + .unwrap(); + let buf = w.finish().unwrap().into_inner(); + let a = ZipArchive::open(IoCursor::new(buf.clone())).unwrap(); + let mut e = a.entries()[0].clone(); + e.size = 1000; + let mut out = Vec::new(); + let r = extract_entry(&mut IoCursor::new(buf), &e, &mut out); + assert!(r.is_err(), "{r:?}"); + assert_eq!(out.len(), 1001); + } + + fn eocd_field(buf: &[u8], at: usize) -> usize { + let p = buf.len() - EOCD_MIN + at; + u32::from_le_bytes(buf[p..p + 4].try_into().unwrap()) as usize + } + + #[test] + fn entries_sharing_their_data_are_rejected() { + let mut w = ZipWriter::new(IoCursor::new(Vec::new())); + w.add("a.txt", &b"payload"[..], Codec::Store, Level::STORE, None) + .unwrap(); + let buf = w.finish().unwrap().into_inner(); + let cd_size = eocd_field(&buf, 12); + let cd_offset = eocd_field(&buf, 16); + let header = &buf[cd_offset..cd_offset + cd_size]; + + let mut bomb = buf[..cd_offset].to_vec(); + bomb.extend_from_slice(header); + bomb.extend_from_slice(header); + let mut eocd = buf[buf.len() - EOCD_MIN..].to_vec(); + eocd[8..10].copy_from_slice(&2u16.to_le_bytes()); + eocd[10..12].copy_from_slice(&2u16.to_le_bytes()); + eocd[12..16].copy_from_slice(&(2 * cd_size as u32).to_le_bytes()); + bomb.extend_from_slice(&eocd); + + let r = ZipArchive::open(IoCursor::new(bomb)); + assert!(matches!(r, Err(Error::Format(_))), "{:?}", r.err()); + } + + #[test] + fn a_count_that_disagrees_with_the_directory_is_rejected() { + let mut w = ZipWriter::new(IoCursor::new(Vec::new())); + for name in ["a.txt", "b.txt"] { + w.add(name, &b"x"[..], Codec::Store, Level::STORE, None) + .unwrap(); + } + let buf = w.finish().unwrap().into_inner(); + let end = buf.len() - EOCD_MIN; + for claimed in [1u16, 3] { + let mut bad = buf.clone(); + bad[end + 8..end + 10].copy_from_slice(&claimed.to_le_bytes()); + bad[end + 10..end + 12].copy_from_slice(&claimed.to_le_bytes()); + let r = ZipArchive::open(IoCursor::new(bad)); + assert!( + matches!(r, Err(Error::Format(_))), + "{claimed}: {:?}", + r.err() + ); + } + } + + #[test] + fn names_the_reader_would_refuse_are_not_written() { + let mut w = ZipWriter::new(IoCursor::new(Vec::new())); + let long = "a".repeat(limits::MAX_NAME + 1); + let r = w.add(&long, &b"x"[..], Codec::Store, Level::STORE, None); + assert!(matches!(r, Err(Error::Limit(_))), "{r:?}"); + } + + fn through_files( + tag: &str, + buf: &[u8], + run: impl FnOnce(&std::path::Path, &std::path::Path) -> Result, + ) -> Vec { + let room = std::env::temp_dir().join(format!("arca-{tag}-{}", std::process::id())); + std::fs::create_dir_all(&room).unwrap(); + let (from, to) = (room.join("a.zip"), room.join("b.zip")); + std::fs::write(&from, buf).unwrap(); + run(&from, &to).unwrap(); + let out = std::fs::read(&to).unwrap(); + std::fs::remove_dir_all(&room).unwrap(); + out + } + + const EXEC: u32 = 0o100755 << 16; + + // An archive made somewhere else: a Unix host byte, an executable mode, a + // name in code page 437 without the UTF-8 flag, a time extra field, a + // Unicode path field, and a comment on the entry. + fn foreign_archive() -> Vec { + let mut w = ZipWriter::new(IoCursor::new(Vec::new())); + let mut extra = vec![0x55, 0x54, 5, 0, 1]; + extra.extend_from_slice(&1_600_000_000u32.to_le_bytes()); + extra.extend_from_slice(&[0x75, 0x70, 3, 0, 1, 0, 0]); + for name in ["run.sh", "gone.txt"] { + w.add( + name, + &b"#!/bin/sh\n"[..], + Codec::Deflate, + Level::NORMAL, + None, + ) + .unwrap(); + let r = w.registros.last_mut().unwrap(); + if name == "run.sh" { + r.name_str = b"caf\x82.sh".to_vec(); + } + r.kept = Some(Kept { + made_by: 0x0314, + needed: 20, + flags: 0, + method: r.method_code, + time: r.time_val, + date: r.date_val, + internal: 1, + external: EXEC, + extra: extra.clone(), + comment: b"keep me".to_vec(), + }); + } + w.finish().unwrap().into_inner() + } + + #[test] + fn removing_and_renaming_keep_what_the_survivors_carried() { + let buf = foreign_archive(); + let removed = through_files("keep-remove", &buf, |a, b| { + remove_entries(a, b, None, &|e| e.name != "gone.txt", &|_, _, _| true) + }); + let mut a = ZipArchive::open(IoCursor::new(removed)).unwrap(); + assert_eq!(a.len(), 1); + let (e, k) = (a.entries_list[0].clone(), a.kept[0].clone()); + assert_eq!(e.raw_name, b"caf\x82.sh"); + assert!(!e.utf8); + assert_eq!((k.made_by, k.internal, k.external), (0x0314, 1, EXEC)); + assert_eq!(k.comment, b"keep me"); + assert!(has_extra(&k.extra, 0x5455)); + assert!(has_extra(&k.extra, EXTRA_UNICODE_PATH)); + let mut out = Vec::new(); + a.extract_to(0, &mut out).unwrap(); + assert_eq!(out, b"#!/bin/sh\n"); + + let renamed = through_files("keep-rename", &buf, |a, b| { + rename_entries(a, b, None, &|e| format!("bin/{}", e.name), &|_, _, _| true) + }); + let a = ZipArchive::open(IoCursor::new(renamed)).unwrap(); + let (e, k) = (&a.entries_list[0], &a.kept[0]); + assert_eq!(e.name, "bin/caf\u{e9}.sh"); + assert!(e.utf8); + assert_eq!(k.external, EXEC); + assert!(has_extra(&k.extra, 0x5455)); + assert!(!has_extra(&k.extra, EXTRA_UNICODE_PATH)); + } + + #[test] + fn a_streamed_zipcrypto_entry_survives_a_rename_without_its_password() { + let plain = b"old tools wrote this to a pipe".repeat(10); + let buf = zipcrypto_archive_with("old.txt", &plain, "secret", true); + assert_eq!(only_entry(&buf, Some("secret")).0, plain); + + let renamed = through_files("zipcrypto-rename", &buf, |a, b| { + rename_entries(a, b, None, &|_| "new.txt".into(), &|_, _, _| true) + }); + let a = ZipArchive::open(IoCursor::new(renamed.clone())).unwrap(); + assert!(a.entries()[0].zipcrypto); + assert_eq!(a.entries()[0].name, "new.txt"); + assert_eq!(only_entry(&renamed, Some("secret")).0, plain); + } + + #[test] + fn removing_from_a_mixed_archive_leaves_plain_entries_plain() { + let mut w = ZipWriter::new(IoCursor::new(Vec::new())); + w.add( + "plain.txt", + &b"open"[..], + Codec::Deflate, + Level::NORMAL, + None, + ) + .unwrap(); + for name in ["secret.txt", "gone.txt"] { + w.add_with_password( + name, + &b"closed"[..], + Codec::Deflate, + Level::NORMAL, + None, + Some("pw"), + ) + .unwrap(); + } + let buf = w.finish().unwrap().into_inner(); + for password in [Some("pw"), None] { + let out = through_files("mixed", &buf, |a, b| { + remove_entries(a, b, password, &|e| e.name != "gone.txt", &|_, _, _| true) + }); + let mut a = ZipArchive::open(IoCursor::new(out)).unwrap(); + let flags: Vec<(&str, bool)> = a + .entries() + .iter() + .map(|e| (e.name.as_str(), e.encrypted)) + .collect(); + assert_eq!(flags, [("plain.txt", false), ("secret.txt", true)]); + let mut got = Vec::new(); + a.extract_to_with(1, &mut got, Some("pw")).unwrap(); + assert_eq!(got, b"closed"); + } + let wrong = std::env::temp_dir().join(format!("arca-wrong-{}", std::process::id())); + std::fs::create_dir_all(&wrong).unwrap(); + std::fs::write(wrong.join("a.zip"), &buf).unwrap(); + let r = remove_entries( + &wrong.join("a.zip"), + &wrong.join("b.zip"), + Some("nope"), + &|_| true, + &|_, _, _| true, + ); + std::fs::remove_dir_all(&wrong).unwrap(); + assert!(r.is_err(), "a wrong password must not add files under it"); + } + + #[test] + fn extra_fields_that_run_off_the_end_are_dropped_without_a_panic() { + let whole = [0x55, 0x54, 1, 0, 9]; + assert_eq!(strip_extra(&whole, &[]), whole); + assert!(strip_extra(&whole, &[0x5455]).is_empty()); + let mut cut = whole.to_vec(); + cut.extend_from_slice(&[0x01, 0x00, 0xFF, 0xFF, 1, 2]); + assert_eq!(strip_extra(&cut, &[]), whole); + for n in 0..cut.len() { + let _ = strip_extra(&cut[..n], &[EXTRA_ZIP64]); + let _ = has_extra(&cut[..n], EXTRA_ZIP64); + } + } + // Taking entries out is the one operation with no undo, so whatever // survives has to survive byte for byte. #[test] @@ -3130,26 +3662,50 @@ mod tests { // the bytes out by hand the way the old tools do: a stored entry whose data // is the 12 byte header and the plaintext, encrypted as one run. fn zipcrypto_archive(name: &str, plain: &[u8], password: &str) -> Vec { + zipcrypto_archive_with(name, plain, password, false) + } + + // `streamed` lays the entry out the way a tool writing to a pipe does: flag + // bit 3, zeros in the local header, a data descriptor after the data, and + // the password check byte taken from the time instead of the CRC. + fn zipcrypto_archive_with(name: &str, plain: &[u8], password: &str, streamed: bool) -> Vec { let crc = crc32fast::hash(plain); + let time: u16 = if streamed { 0x6B21 } else { 0 }; let mut body = vec![0u8; zipcrypto::HEADER]; - body[zipcrypto::HEADER - 1] = (crc >> 24) as u8; + body[zipcrypto::HEADER - 1] = if streamed { + (time >> 8) as u8 + } else { + (crc >> 24) as u8 + }; body.extend_from_slice(plain); zipcrypto::Keys::new(password.as_bytes()).encrypt(&mut body); - let flags: u16 = 0x0801; + let flags: u16 = if streamed { 0x0809 } else { 0x0801 }; + let (lfh_crc, lfh_comp, lfh_size) = if streamed { + (0, 0, 0) + } else { + (crc, body.len() as u32, plain.len() as u32) + }; let mut out = Vec::new(); out.extend_from_slice(&SIG_LFH.to_le_bytes()); out.extend_from_slice(&20u16.to_le_bytes()); out.extend_from_slice(&flags.to_le_bytes()); out.extend_from_slice(&0u16.to_le_bytes()); - out.extend_from_slice(&0u32.to_le_bytes()); - out.extend_from_slice(&crc.to_le_bytes()); - out.extend_from_slice(&(body.len() as u32).to_le_bytes()); - out.extend_from_slice(&(plain.len() as u32).to_le_bytes()); + out.extend_from_slice(&time.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(&lfh_crc.to_le_bytes()); + out.extend_from_slice(&lfh_comp.to_le_bytes()); + out.extend_from_slice(&lfh_size.to_le_bytes()); out.extend_from_slice(&(name.len() as u16).to_le_bytes()); out.extend_from_slice(&0u16.to_le_bytes()); out.extend_from_slice(name.as_bytes()); out.extend_from_slice(&body); + if streamed { + out.extend_from_slice(&SIG_DESCRIPTOR.to_le_bytes()); + out.extend_from_slice(&crc.to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(&(plain.len() as u32).to_le_bytes()); + } let cd_at = out.len(); out.extend_from_slice(&SIG_CD.to_le_bytes()); @@ -3157,7 +3713,8 @@ mod tests { out.extend_from_slice(&20u16.to_le_bytes()); out.extend_from_slice(&flags.to_le_bytes()); out.extend_from_slice(&0u16.to_le_bytes()); - out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&time.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); out.extend_from_slice(&crc.to_le_bytes()); out.extend_from_slice(&(body.len() as u32).to_le_bytes()); out.extend_from_slice(&(plain.len() as u32).to_le_bytes()); diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 28d05964..255870f3 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -42,7 +42,7 @@ Create an archive. The format is taken from the extension of the output file. | `...` | required | Files or directories to include. | | `-l, --level ` | normal | store · fast · normal · best. | | `-c, --codec ` | auto | auto · store · deflate · zstd · lzma2. auto uses Deflate in .zip, LZMA2 in .7z, .xz and .tar.xz and the RAR compressor in .rar (auto or store only). | -| `-j, --threads ` | 0 | ZIP and XZ threads; 0 means every core. XZ uses as many as fit in 2 GiB. 7z and RAR creation is sequential; .rar refuses N above 1. | +| `-j, --threads ` | 0 | ZIP and XZ threads; 0 means every core. XZ uses as many as fit in 2 GiB; 7z and RAR use every core their memory budget allows, whatever this says. | | `-p, --password ` | none | Encrypt with AES-256 (.zip, .7z). Other tools will ask for it to open the archive. Refused for .rar. | | `--hide-names` | off | 7z only: encrypt headers; requires a nonempty password. | diff --git a/docs/guide/creating-archives.md b/docs/guide/creating-archives.md index 217861a0..e3512983 100644 --- a/docs/guide/creating-archives.md +++ b/docs/guide/creating-archives.md @@ -15,7 +15,7 @@ arca create ... [-l LEVEL] [-c CODEC] [-j THREADS] [-p PASSWORD] [ | --- | --- | --- | --- | | `-l, --level` | store · fast · normal · best | normal | Compression level. | | `-c, --codec` | auto · store · deflate · zstd · lzma2 | auto | Deflate in .zip, LZMA2 in .7z, .xz and .tar.xz, the RAR compressor in .rar (auto or store only). | -| `-j, --threads` | a number | 0 | ZIP and XZ thread limit. 0 means every core; 7z and RAR are sequential. | +| `-j, --threads` | a number | 0 | ZIP, XZ and RAR thread limit. 0 means every core; 7z uses every core its memory budget allows. | | `-p, --password` | text | none | Encrypt ZIP or 7z with AES-256. Refused for .rar. | | `--hide-names` | flag | off | Encrypt 7z headers too; requires a nonempty password. | @@ -65,8 +65,11 @@ and 9. `-c store` overrides the level. Only `auto`, `store` and `lzma2` are accepted for 7z; explicit Deflate/Zstandard codec selection applies to ZIP, not 7z. TAR ignores those codec choices and `.tar.gz` always uses gzip; `lzma2` is rejected outside 7z. -Creation is sequential regardless of `-j`, with one independent block per file, -not solid compression. Reading existing solid archives is supported. +Files are compressed together in solid blocks of up to 256 MiB, grouped by +extension the way 7-Zip does; x86 executables (`.exe`, `.dll` and the like) go +in blocks of their own behind the BCJ filter, which makes machine code smaller. +LZMA2 runs on as many cores as fit in 2 GiB (every core at `fast` and `normal`, +one at `best`), and entries are listed in that grouped order. 7z includes empty files and directories and rejects symlink/reparse-point and special-file inputs rather than following them. Creation writes a temporary @@ -84,7 +87,7 @@ arca create stored.rar big.iso -c store Creates a new single-volume, non-solid, unencrypted RAR5 archive that official UnRAR and 7-Zip open. The four levels select the RAR compressor strength; `-c store` equals `-l store` and any other explicit codec is refused. Creation -is sequential (`-j` above 1 is refused), stores empty files and directories, +uses every core by default, or as many threads as `-j` says, stores empty files and directories, and rejects symbolic links and special files instead of following them. RAR output is never encrypted (`-p` and `--hide-names` are refused), never a diff --git a/docs/guide/desktop-app.md b/docs/guide/desktop-app.md index a54180e1..d0758d71 100644 --- a/docs/guide/desktop-app.md +++ b/docs/guide/desktop-app.md @@ -63,8 +63,8 @@ actions stay disabled for it. CBR cannot be created. See [RAR](rar.md#creating-r ## 7z -Create 7z using Store or LZMA2 and the existing levels. Creation and extraction -are sequential. Selected members of solid archives share one decoder per block +Create 7z using Store or LZMA2 and the existing levels, in solid blocks and on as +many cores as fit in 2 GiB; extraction decodes on one. Selected members of solid archives share one decoder per block per pass, rather than reopening it for each member. Preview, open-file, Copy and Extract use the same validated read path. See [Encryption](encryption.md) and [resource limits](architecture.md#7z-boundaries). diff --git a/docs/guide/es/cli-reference.md b/docs/guide/es/cli-reference.md index c2e2cd47..9245aa6d 100644 --- a/docs/guide/es/cli-reference.md +++ b/docs/guide/es/cli-reference.md @@ -42,7 +42,7 @@ Crea un archivo. El formato se toma de la extensión del archivo de salida. | `...` | obligatorio | Archivos o directorios a incluir. | | `-l, --level ` | normal | store · fast · normal · best. | | `-c, --codec ` | auto | auto · store · deflate · zstd · lzma2. auto usa Deflate en .zip, LZMA2 en .7z, .xz y .tar.xz y el compresor RAR en .rar (solo auto o store). | -| `-j, --threads ` | 0 | Hilos ZIP y XZ; 0 usa todos los núcleos. XZ usa tantos como quepan en 2 GiB. La creación 7z y RAR es secuencial; .rar rechaza N mayor que 1. | +| `-j, --threads ` | 0 | Hilos ZIP y XZ; 0 usa todos los núcleos. XZ usa tantos como quepan en 2 GiB; 7z y RAR usan todos los núcleos que permite su presupuesto de memoria, diga lo que diga esta opción. | | `-p, --password ` | ninguna | Cifra con AES-256 (.zip, .7z). Otras herramientas la pedirán para abrir el archivo. Se rechaza en .rar. | | `--hide-names` | desactivado | Solo 7z: cifra cabeceras; exige una contraseña no vacía. | diff --git a/docs/guide/es/creating-archives.md b/docs/guide/es/creating-archives.md index e51a9d41..e3a6441b 100644 --- a/docs/guide/es/creating-archives.md +++ b/docs/guide/es/creating-archives.md @@ -15,7 +15,7 @@ arca create ... [-l LEVEL] [-c CODEC] [-j THREADS] [-p PASSWORD] [ | --- | --- | --- | --- | | `-l, --level` | store · fast · normal · best | normal | Nivel de compresión. | | `-c, --codec` | auto · store · deflate · zstd · lzma2 | auto | Deflate en .zip, LZMA2 en .7z, .xz y .tar.xz, el compresor RAR en .rar (solo auto o store). | -| `-j, --threads` | un número | 0 | Límite de hilos ZIP y XZ. 0 usa todos los núcleos; 7z y RAR son secuenciales. | +| `-j, --threads` | un número | 0 | Límite de hilos ZIP y XZ. 0 usa todos los núcleos; 7z y RAR usan todos los núcleos que permite su presupuesto de memoria. | | `-p, --password` | texto | ninguna | Cifra ZIP o 7z con AES-256. Se rechaza en .rar. | | `--hide-names` | indicador | desactivado | Cifra también las cabeceras 7z; exige una contraseña no vacía. | @@ -64,8 +64,11 @@ Los niveles `store`, `fast`, `normal` y `best` seleccionan Copy o LZMA2 con nive 1, 6 y 9. `-c store` prevalece sobre el nivel. 7z solo admite `auto`, `store` y `lzma2`; la selección explícita Deflate/Zstandard se aplica a ZIP, no a 7z. TAR ignora esas opciones y `.tar.gz` siempre usa gzip; `lzma2` se rechaza fuera de 7z. -La creación es secuencial independientemente de `-j`, con un bloque independiente -por fichero, sin compresión sólida. Sí se pueden leer archivos sólidos existentes. +Los ficheros se comprimen juntos en bloques sólidos de hasta 256 MiB, agrupados +por extensión como hace 7-Zip; los ejecutables x86 (`.exe`, `.dll` y similares) +van en bloques propios con el filtro BCJ, que reduce el código máquina. LZMA2 usa +tantos núcleos como quepan en 2 GiB (todos en `fast` y `normal`, uno en `best`), y +las entradas se listan en ese orden agrupado. 7z incluye ficheros y directorios vacíos y rechaza enlaces simbólicos, puntos de reanálisis y ficheros especiales. La creación usa un temporal junto al destino y @@ -83,7 +86,7 @@ arca create stored.rar big.iso -c store Crea un archivo RAR5 nuevo, de un solo volumen, no sólido y sin cifrar, que UnRAR y 7-Zip oficiales abren. Los cuatro niveles eligen la fuerza del compresor RAR; `-c store` equivale a `-l store` y cualquier otro códec explícito se -rechaza. La creación es secuencial (`-j` mayor que 1 se rechaza), guarda +rechaza. La creación usa todos los núcleos que permite su presupuesto de memoria, diga lo que diga `-j`, guarda ficheros y directorios vacíos y rechaza enlaces simbólicos y ficheros especiales en lugar de seguirlos. diff --git a/docs/guide/es/desktop-app.md b/docs/guide/es/desktop-app.md index a6aa6855..37b915c0 100644 --- a/docs/guide/es/desktop-app.md +++ b/docs/guide/es/desktop-app.md @@ -64,8 +64,8 @@ No se puede crear CBR. Consulta [RAR](rar.md#crear-archivos-rar5). ## 7z -Crea 7z con Store o LZMA2 y los niveles existentes. La creación y la extracción -son secuenciales. Las entradas seleccionadas de archivos sólidos comparten un +Crea 7z con Store o LZMA2 y los niveles existentes, en bloques sólidos y con tantos +núcleos como quepan en 2 GiB; la extracción descomprime con uno. Las entradas seleccionadas de archivos sólidos comparten un decodificador por bloque y pasada, sin reabrirlo por cada entrada. La vista previa, abrir, Copiar y Extraer usan la misma ruta de lectura validada. Consulta [Cifrado](encryption.md) y los [límites de recursos](architecture.md#7z-boundaries). diff --git a/docs/guide/es/keyboard-shortcuts.md b/docs/guide/es/keyboard-shortcuts.md index 22a389f0..7a7330f5 100644 --- a/docs/guide/es/keyboard-shortcuts.md +++ b/docs/guide/es/keyboard-shortcuts.md @@ -13,18 +13,19 @@ Las órdenes de lista actúan sobre el panel activo. Los campos de texto y los d | --- | --- | | Arriba Abajo RePág AvPág Inicio Fin | Mover el cursor | | Derecha | Entrar en la carpeta del cursor | -| Izquierda Retroceso | Subir a la carpeta superior | +| Izquierda Retroceso Alt+Arriba | Subir a la carpeta superior | | Intro | Entrar en una carpeta o abrir explícitamente un archivo con la aplicación del sistema | | Espacio | Alternar la selección de la fila del cursor | | Mayús+Arriba Mayús+Abajo | Ampliar la selección | | Ctrl+A | Seleccionar filas visibles del panel activo | | a…z | Saltar por prefijo del nombre | -| Alt+Izquierda Alt+Derecha | Atrás y adelante en el historial | +| Alt+Izquierda Alt+Derecha | Atrás y adelante en el historial; los botones laterales del ratón hacen lo mismo | | Ctrl+O Ctrl+N Ctrl+E | Abrir archivo, crear archivo, extraer todo | -| Ctrl+C Ctrl+X Ctrl+V | Copiar, cortar y pegar archivos (integración con Windows) | +| Ctrl+C Ctrl+X Ctrl+V | Copiar, cortar y pegar archivos, compartidos con el gestor de archivos del sistema en Windows y Linux | | Ctrl+Mayús+C | Copiar los nombres seleccionados como texto | | Ctrl+F | Mostrar y enfocar el filtro del panel activo | -| F2 | Renombrar la entrada seleccionada en un ZIP modificable | +| Ctrl+L Alt+D F4 | Escribir una ruta en la barra de ruta | +| F2 | Renombrar el archivo o la carpeta seleccionados, en el disco o en un ZIP modificable | | F3 | Activar la vista previa integrada del archivo del cursor | | Ctrl+T | Comprobar el archivo | | Ctrl+I | Invertir la selección visible del panel activo | @@ -32,12 +33,18 @@ Las órdenes de lista actúan sobre el panel activo. Los campos de texto y los d | Supr | Confirmar la eliminación de entradas seleccionadas de un ZIP modificable | | Ctrl+Z | Deshacer la última modificación compatible | | Alt+W | Extraer aquí | +| Ctrl+Mayús+N | Carpeta nueva | +| Ctrl+D | Duplicar los archivos y carpetas seleccionados (en el disco) | +| Alt+Intro | Propiedades de la fila del cursor, o de la carpeta si no hay ninguna (en el disco) | +| Alt+P | Mostrar u ocultar el panel de vista previa | +| Ctrl+W | Cerrar la ventana | | + - | Seleccionar o deseleccionar por máscara de nombre | | Tab Mayús+Tab | Mover el foco entre controles y paneles | +| Mayús+F10 | Abrir el menú de la fila del cursor, o el de la carpeta si no hay ninguna | | F1 | Ayuda de atajos | -| Esc | Cerrar el diálogo o la edición en línea; en otro caso, quitar la selección | +| Esc | Cerrar el diálogo o la edición en línea; si no, deshacer un corte pendiente y después quitar la selección | -Ctrl+clic alterna una fila; Mayús+clic selecciona un intervalo. Un clic en una carpeta abre su hijo en Columnas; Detalles usa doble clic. Haz clic derecho en una fila o espacio vacío del panel para abrir su menú. Arrastra los divisores para ajustar el ancho y desplázate horizontalmente para llegar a los antecesores. +Ctrl+clic alterna una fila; Mayús+clic selecciona un intervalo. Un clic selecciona una fila y un doble clic o Intro la abre; con **Abrir las carpetas en el panel de al lado** activado, un clic en una carpeta la muestra en el panel de al lado. Haz clic derecho en una fila o espacio vacío del panel para abrir su menú. Arrastra los divisores para ajustar el ancho y desplázate horizontalmente para llegar a los antecesores. El botón de cierre de vista previa devuelve el foco. En ventanas estrechas puede quedar suspendida temporalmente hasta que haya sitio. El botón lateral alterna entre panel ampliado, iconos y oculto. diff --git a/docs/guide/es/rar.md b/docs/guide/es/rar.md index f9792e69..43ffd23b 100644 --- a/docs/guide/es/rar.md +++ b/docs/guide/es/rar.md @@ -75,7 +75,7 @@ La creación rechaza, antes de leer ninguna entrada o escribir ningún fichero: - Una contraseña u **Ocultar nombres**: la salida RAR nunca se cifra. Usa ZIP o 7z para eso. -- `-c deflate|zstd|lzma2`, `-j` mayor que 1 (la creación RAR es secuencial), un +- `-c deflate|zstd|lzma2`, un destino `.cbr` o cualquier otro sufijo distinto de `.rar`. - Un destino que ya exista, incluso como enlace roto. Arca solo crea archivos RAR nuevos y nunca reemplaza un fichero, tampoco uno que aparezca mientras se diff --git a/docs/guide/es/roadmap.md b/docs/guide/es/roadmap.md index 63fc45b4..9590a67e 100644 --- a/docs/guide/es/roadmap.md +++ b/docs/guide/es/roadmap.md @@ -32,8 +32,6 @@ y escritorio. Consulta el - **UDF:** las imágenes ISO se leen por ISO 9660, así que los ficheros que solo están en UDF (como `install.wim` en los medios de Windows) todavía no aparecen. - **Enlaces simbólicos:** se omiten al crear ZIP/TAR y se rechazan en 7z. -- **Nombres largos de GNU tar.** -- **Crear 7z sólidos:** se leen los existentes, pero se crean bloques independientes. - **Cambiar contraseñas y modificar 7z:** añadir, borrar y renombrar puede exigir reescribir bloques sólidos y queda fuera del alcance. - **Integración de escritorio en Linux y macOS:** el menú del Explorador es solo para Windows por ahora. - **Modificar fuera de ZIP:** ZIP ya se edita en la ventana; 7z solo se crea y se lee. diff --git a/docs/guide/keyboard-shortcuts.md b/docs/guide/keyboard-shortcuts.md index 1f6143ae..e862688e 100644 --- a/docs/guide/keyboard-shortcuts.md +++ b/docs/guide/keyboard-shortcuts.md @@ -34,6 +34,7 @@ List commands apply to the active pane. Text fields and dialogs retain their edi | Ctrl+Z | Undo the last supported archive modification | | Alt+W | Extract here | | Ctrl+Shift+N | New folder | +| Ctrl+D | Duplicate the selected files and folders (on disk) | | Alt+Enter | Properties of the cursor row, or of the folder when there is none (on disk) | | Alt+P | Show or hide the preview panel | | Ctrl+W | Close the window | diff --git a/docs/guide/rar.md b/docs/guide/rar.md index 762abbd8..bd2fab56 100644 --- a/docs/guide/rar.md +++ b/docs/guide/rar.md @@ -74,7 +74,7 @@ What creation refuses, before any input is read or any file is written: - A password or **Hide file names**: RAR output is never encrypted. Use ZIP or 7z for that. -- `-c deflate|zstd|lzma2`, `-j` above 1 (RAR creation is sequential), a `.cbr` +- `-c deflate|zstd|lzma2`, a `.cbr` output name or any other suffix than `.rar`. - An output path that already exists, even as a dangling link. Arca creates new RAR archives only and never replaces a file, including one that appears diff --git a/docs/guide/roadmap.md b/docs/guide/roadmap.md index 06ef0e9f..415b3b50 100644 --- a/docs/guide/roadmap.md +++ b/docs/guide/roadmap.md @@ -31,8 +31,6 @@ extracted in CLI and desktop. See - **UDF:** ISO images are read through ISO 9660, so files stored only in UDF (such as `install.wim` on Windows media) are not listed yet. - **Symbolic links:** skipped for ZIP/TAR creation, rejected for 7z. -- **GNU tar long names.** -- **Solid 7z creation:** existing solid archives can be read, but new ones use independent blocks. - **7z password changes and mutation:** adding, deleting and renaming can require solid-block rewrites and are out of scope. - **Desktop integration on Linux and macOS:** the Explorer menu is Windows-only for now. - **Mutation outside ZIP:** ZIP desktop editing exists; 7z remains create/read only. diff --git a/docs/plans/rars-0.10.0-distribution.md b/docs/plans/rars-0.10.0-distribution.md index 995acebb..b3315638 100644 --- a/docs/plans/rars-0.10.0-distribution.md +++ b/docs/plans/rars-0.10.0-distribution.md @@ -326,3 +326,23 @@ reinterpreted as writer evidence. WinRAR-equivalent. The creation validation record ([RAR_CREATE_VALIDATION.md](../../arca-rar/RAR_CREATE_VALIDATION.md)) states which interoperability checks were actually run and with which tool versions. + +## Scope update: vendored writer and parallel feature (2026-11-16) + +This section records a later change of technical scope. Everything above still +stands as written. + +- `rars 0.10.2` is now vendored in `arca-rar/vendor/rars` from the crates.io + archive (SHA-256 recorded in `arca-rar/vendor/PATCHES.md`) and applied with + `[patch.crates-io]`. The patches touch only the RAR 5 writer's speed and are + listed there: the binary-tree finder is kept for a whole member, and levels 2 + and 3 search and price less. The reader, the format code and the integrity + checks are unchanged, and no RARLAB source was consulted for the changes. +- The `parallel` feature is now on, so `rars` encodes members and runs on + several threads. It adds `rayon`, `rayon-core`, `crossbeam-deque`, + `crossbeam-epoch`, `crossbeam-utils` and `either`, all MIT OR Apache-2.0; + `python3 packaging/rar-notices.py` lists them and records `rars` as vendored, + and `python3 arca-rar/tests/check-features.py` asserts + `encryption,parallel,write` on the vendored copy. `recovery` stays off. +- `arca-rar/fuzz` keeps its own lockfile on the crates.io copy, whose reader is + the one the vendored copy carries unchanged. diff --git a/interop.sh b/interop.sh index f82b9e99..4278b82c 100644 --- a/interop.sh +++ b/interop.sh @@ -66,6 +66,37 @@ for f in t.tar t.tar.gz; do H=$(cd y/src && find . -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) [ "$H" = "$REF" ] && ok "Arca extracts tar's $f without losing a byte" || ko "Arca mis-extracts $f" done +# Names past ustar's 100 bytes, in both ways tar spells them, archived from +# inside the folder so every name starts with ./ as GitHub release tarballs +# do, and a .tar.gz split across two gzip members as pigz writes them. +LONG="long/$(printf 'd%.0s' {1..120})/$(printf 'e%.0s' {1..90})" +mkdir -p "$LONG" && cp -r src "$LONG/" +LREF=$(cd long && find . -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) +tar --format=gnu -czf out/long-gnu.tar.gz -C long . +tar --format=pax -czf out/long-pax.tar.gz -C long . +tar --format=gnu -cf out/long.tar -C long . +head -c 4096 out/long.tar | gzip -c > out/long-split.tar.gz +tail -c +4097 out/long.tar | gzip -c >> out/long-split.tar.gz +for f in long-gnu long-pax long-split; do + rm -rf y; mkdir y + $ARCA extract out/$f.tar.gz -o y >/dev/null 2>&1 || { ko "arca extract $f.tar.gz"; continue; } + H=$(cd y && find . -type f | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) + [ "$H" = "$LREF" ] && ok "Arca extracts $f.tar.gz with its long names" || ko "Arca mis-extracts $f.tar.gz" +done +rm -rf long +# A program in a tarball comes out runnable, and a plain file does not. +mkdir -p perm && printf '#!/bin/sh\necho ok\n' > perm/run.sh && echo data > perm/plain.txt +chmod 755 perm/run.sh && chmod 644 perm/plain.txt +tar czf out/perm.tar.gz perm && rm -rf perm y && mkdir y +$ARCA extract out/perm.tar.gz -o y >/dev/null 2>&1 +[ -x y/perm/run.sh ] && [ ! -x y/perm/plain.txt ] && [ "$(y/perm/run.sh)" = ok ] && + ok "Arca keeps the execute bit of a tar's program" || ko "Arca loses or invents execute bits" +# And the other way: a tarball Arca makes keeps it for tar to restore. +rm -rf z && mkdir z && mv y/perm . && $ARCA create out/arca-perm.tar.gz perm >/dev/null 2>&1 +tar xzf out/arca-perm.tar.gz -C z 2>/dev/null +[ -x z/perm/run.sh ] && [ ! -x z/perm/plain.txt ] && + ok "Arca's .tar.gz keeps the execute bit for tar" || ko "Arca's .tar.gz loses execute bits" +rm -rf perm z 7z a -tzip -mx5 out/s7.zip src >/dev/null 2>&1 rm -rf y; mkdir y; $ARCA extract out/s7.zip -o y >/dev/null 2>&1 H=$(cd y/src && find . -type f|sort|xargs sha256sum|sha256sum|cut -d' ' -f1) @@ -164,7 +195,8 @@ fi echo echo "G) 7z: Arca writes, 7-Zip verifies and extracts" -python3 - <<'PY' +python3 - "$ARCA" <<'PY' +import sys from pathlib import Path root = Path('src7') (root / 'nested' / 'empty-dir').mkdir(parents=True) @@ -172,6 +204,15 @@ root = Path('src7') (root / 'first.txt').write_bytes(b'first solid contents\n' * 2000) (root / 'second.txt').write_bytes(b'second solid contents\n' * 2000) (root / 'nested' / '\u00f1-\U0001f680.txt').write_bytes(bytes(range(256)) * 16) +# Executables go in a block of their own behind the BCJ filter, so 7-Zip has +# to undo that filter too; the program is the binary under test, the library +# made-up x86 calls. +(root / 'bin').mkdir() +(root / 'bin' / 'arca.exe').write_bytes(Path(sys.argv[1]).read_bytes()) +calls = bytearray() +for i in range(40000): + calls += b'\xe8' + ((0x1000 + 16 * i - len(calls)) & 0xffffffff).to_bytes(4, 'little') + b'\x48\x89\xc7\x90' +(root / 'bin' / 'helper.dll').write_bytes(bytes(calls)) PY compare_trees() { python3 - "$1" "$2" <<'PY' diff --git a/packaging/rar-notices.py b/packaging/rar-notices.py index 27a81e0e..53253a7f 100644 --- a/packaging/rar-notices.py +++ b/packaging/rar-notices.py @@ -17,6 +17,14 @@ "aarch64-apple-darwin", ) LEGAL_NAMES = ("LICENSE", "LICENCE", "COPYING", "COPYRIGHT", "NOTICE") +# Third-party crates copied into the tree and patched; their notices are +# published like any other dependency's. +VENDOR = ROOT / "arca-rar" / "vendor" +VENDORED = "vendored in arca-rar/vendor from registry+https://github.com/rust-lang/crates.io-index" + + +def vendored(package): + return Path(package["manifest_path"]).resolve().is_relative_to(VENDOR.resolve()) def cargo(*args): @@ -37,7 +45,7 @@ def render(): packages = sorted( (p for p in metadata["packages"] - if p["source"] and (p["name"], p["version"]) in selected), + if (p["source"] or vendored(p)) and (p["name"], p["version"]) in selected), key=lambda p: (p["name"], p["version"]), ) assert any(p["name"] == "rars" for p in packages), "missing RAR backend" @@ -66,7 +74,7 @@ def render(): files = sorted(set(files) | {source / package["license_file"]}) assert files, f"no published license files for {name}" output += [name, f"Declared license: {package['license']}", - f"Source: {package['source']}"] + f"Source: {package['source'] or VENDORED}"] for path in files: content = path.read_bytes() digest = hashlib.sha256(content).hexdigest()