From a5c33f1f4db01877447ba67c0810a6d0381f3ec0 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sat, 10 Oct 2026 23:55:14 -0500 Subject: [PATCH 1/8] fix(tar): budget files-from storage before allocation --- crates/bashkit/docs/threat-model.md | 2 +- crates/bashkit/src/builtins/archive.rs | 100 +++++++-- crates/bashkit/tests/integration/main.rs | 1 + .../tar_files_from_budget_tests.rs | 199 ++++++++++++++++++ .../tests/tar_files_from_allocation_tests.rs | 76 +++++++ 5 files changed, 356 insertions(+), 22 deletions(-) create mode 100644 crates/bashkit/tests/integration/tar_files_from_budget_tests.rs create mode 100644 crates/bashkit/tests/tar_files_from_allocation_tests.rs diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index bf8c23d3e..7bb773642 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -190,7 +190,7 @@ let bash = Bash::builder() | jq control normalization amplification (TM-DOS-100) | Literal controls expand sixfold as `\u00XX` | Charge single-pass work and lease live bytes before allocation growth | MITIGATED | | yq structured-data amplification (TM-DOS-101) | Deep/multi-document YAML or JSON, runaway filters, expanded output | Parser depth and 4096-document caps, aggregate budgets, shared jaq work/deadline/output limits, final render cap | MITIGATED | | Archive decoder pre-allocation (TM-DOS-102) | Compressed output grows before memory checks | Validate and charge each decoder chunk before reserve/copy | MITIGATED | -| Post-allocation charging (TM-DOS-103) | Archive/compression buffers grow before their live-byte check | Owning budget-aware string/vector/byte builders charge before reserve, roll back errors, and release on drop | MITIGATED | +| Post-allocation charging (TM-DOS-103) | Archive/compression buffers or tar file-list names grow before their live-byte check | Owning budget-aware string/vector/byte builders charge before reserve, roll back errors, and release on drop; tar file lists also lease raw reads and charge per-line work | MITIGATED | | Pipeline stderr aggregation (TM-DOS-104) | Many stderr-producing pipeline stages accumulate past the output cap | Stop appending at `max_stderr_bytes` and propagate the truncation flag to the result | MITIGATED | | Zip creation pre-allocation (TM-DOS-106) | Recursive inputs and encoded output allocate before live-memory checks | Collect paths only, lease file sizes before VFS reads, and encode through a budget-aware byte buffer | MITIGATED | | Analysis command-name validation (TM-DOS-107) | A huge comment before thousands of commands makes validation rescan the source per command | Index source character positions once, then binary-search each name | MITIGATED | diff --git a/crates/bashkit/src/builtins/archive.rs b/crates/bashkit/src/builtins/archive.rs index 5f74d6c44..955cdf75c 100644 --- a/crates/bashkit/src/builtins/archive.rs +++ b/crates/bashkit/src/builtins/archive.rs @@ -10,6 +10,9 @@ //! normalize it to the regular short-option path instead of maintaining a //! second parser. //! +//! File lists retain budget-owned names, never an eager `Vec`; raw +//! reads are admitted by metadata size and every line checks shared work. +//! //! bzip2 0.6 uses the maintained pure-Rust libbz2-rs backend by default. Its //! fixed codec state plus bounded, pre-charged output avoids native-library //! deployment differences while retaining standard bzip2 CRC validation. @@ -77,6 +80,29 @@ fn budgeted_vec(ctx: &Context<'_>) -> Result> { with_execution_budget(ctx, |budget| BudgetedVec::new(budget).map_err(Into::into)) } +// THREAT[TM-DOS-103]: Decode directly into admitted storage, including UTF-8 +// replacement expansion. `from_utf8_lossy(...).into_owned()` allocates first. +fn tar_list_name(ctx: &Context<'_>, mut bytes: &[u8]) -> Result { + let mut name = budgeted_string(ctx)?; + while !bytes.is_empty() { + ctx.consume_budget_work(1)?; + match std::str::from_utf8(bytes) { + Ok(text) => { + name.try_push_str(text)?; + break; + } + Err(error) => { + let valid = error.valid_up_to(); + name.try_push_str(std::str::from_utf8(&bytes[..valid]).unwrap())?; + name.try_push('\u{fffd}')?; + let invalid = error.error_len().unwrap_or(bytes.len() - valid); + bytes = &bytes[valid + invalid..]; + } + } + } + Ok(name) +} + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] enum ArchiveCompression { #[default] @@ -397,34 +423,66 @@ impl Builtin for Tar { // `-T FILE` / `--files-from=FILE`: one member name per line, blank // lines skipped, `-` reads stdin. - let mut listed_names: Vec = Vec::new(); + // THREAT[TM-DOS-103]: Both name capacity and vector slots must be + // charged before allocation, even when the members do not exist. + let mut listed_names = budgeted_vec(&ctx)?; for list in &files_from { + let missing_list = || { + ExecResult::err( + format!( + "tar: {list}: Cannot stat: No such file or directory\ntar: Error is not recoverable: exiting now\n" + ), + 2, + ) + }; + let bytes; + let _read_lease; + let _extra_lease; let content = if list == "-" { - String::from_utf8_lossy(ctx.stdin_bytes().unwrap_or_default()).into_owned() + let input = ctx.stdin_bytes().unwrap_or_default(); + ctx.consume_budget_input(input.len())?; + input } else { let path = resolve_path(ctx.cwd, list); - match ctx.fs.read_file(&path).await { - Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(), - Err(_) => { - return Ok(ExecResult::err( - format!( - "tar: {list}: Cannot stat: No such file or directory\ntar: Error is not recoverable: exiting now\n" - ), - 2, - )); + let metadata = match ctx.fs.stat(&path).await { + Ok(metadata) => metadata, + Err(error @ (crate::Error::ResourceLimit(_) | crate::Error::Cancelled)) => { + return Err(error); } - } + Err(_) => return Ok(missing_list()), + }; + let size = usize::try_from(metadata.size).unwrap_or(usize::MAX); + ctx.consume_budget_input(size)?; + _read_lease = ctx.lease_budget_bytes(size)?; + bytes = match ctx.fs.read_file(&path).await { + Ok(bytes) => bytes, + Err(error @ (crate::Error::ResourceLimit(_) | crate::Error::Cancelled)) => { + return Err(error); + } + Err(_) => return Ok(missing_list()), + }; + // Host-backed files may grow between stat/read; the backend + // owns read-allocation admission, as for other VFS consumers. + ctx.consume_budget_input(bytes.len().saturating_sub(size))?; + _extra_lease = ctx.lease_budget_bytes(bytes.capacity().saturating_sub(size))?; + bytes.as_slice() }; - ctx.consume_budget_input(content.len())?; - listed_names.extend( - content - .lines() - .filter(|l| !l.is_empty()) - .map(str::to_string), - ); + ctx.consume_budget_work(u64::try_from(content.len().div_ceil(64)).unwrap_or(u64::MAX))?; + for line in content.split_inclusive(|byte| *byte == b'\n') { + ctx.consume_budget_work(1)?; + // Match str::lines(): strip CR only as part of CRLF. + let line = if let Some(line) = line.strip_suffix(b"\n") { + line.strip_suffix(b"\r").unwrap_or(line) + } else { + line + }; + if !line.is_empty() { + listed_names.try_push(tar_list_name(&ctx, line)?)?; + } + } } - for name in &listed_names { - files.try_push(name.as_str())?; + for name in &*listed_names { + files.try_push(&**name)?; } // Check for exactly one of -c, -x, -t diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 28019d11d..43f973d15 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -53,6 +53,7 @@ pub mod custom_fs_tests; pub mod function_metadata_security_tests; pub mod interactive_builtins_tests; pub mod memory_growth_security_tests; +pub mod tar_files_from_budget_tests; // Both assert named-IANA-zone behavior, which only exists with `tzdata`. // `date_timezone_no_tzdata_tests` covers the closed-to-UTC side. pub mod cpython_capability_tests; diff --git a/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs b/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs new file mode 100644 index 000000000..4f07596c2 --- /dev/null +++ b/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs @@ -0,0 +1,199 @@ +use bashkit::{ + Bash, Error, ExecOptions, ExecutionBudgetExceeded, ExecutionLimits, LimitExceeded, StreamData, +}; +use std::path::Path; + +#[tokio::test] +async fn tar_files_from_large_content_obeys_live_budget() { + for content in [vec![b'x'; 20_000], vec![0xff; 4000]] { + let mut bash = Bash::builder() + .limits(ExecutionLimits::new().max_live_intermediate_bytes(8192)) + .build(); + bash.fs() + .write_file(Path::new("/list"), &content) + .await + .unwrap(); + let result = bash.exec("tar -cf /out.tar -T /list").await; + assert!( + matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget( + ExecutionBudgetExceeded::LiveBytes { .. } + ))) + ), + "expected budget rejection" + ); + assert_eq!( + bash.exec("echo recovered").await.unwrap().stdout, + "recovered\n" + ); + } +} + +#[tokio::test] +async fn tar_files_from_dense_lines_obey_work_budget() { + for content in ["x\n".repeat(2000), "\n".repeat(2000), "x".repeat(20_000)] { + let mut bash = Bash::builder() + .limits(ExecutionLimits::new().max_work_units(100)) + .build(); + bash.fs() + .write_file(Path::new("/list"), content.as_bytes()) + .await + .unwrap(); + let result = bash.exec("tar -cf /out.tar -T /list").await; + assert!( + matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget( + ExecutionBudgetExceeded::WorkUnits { .. } + ))) + ), + "expected budget rejection" + ); + assert_eq!( + bash.exec("echo recovered").await.unwrap().stdout, + "recovered\n" + ); + } +} + +#[tokio::test] +async fn tar_files_from_stdin_and_repeated_lists_obey_live_budget() { + for (script, content) in [ + ("tar -cf /out.tar -T -".to_string(), "x".repeat(20_000)), + ( + format!("tar -cf /out.tar {}", "-T /list ".repeat(40)), + format!("{}\n", "x".repeat(256)), + ), + ] { + let mut bash = Bash::builder() + .limits(ExecutionLimits::new().max_live_intermediate_bytes(8192)) + .build(); + bash.fs() + .write_file(Path::new("/list"), content.as_bytes()) + .await + .unwrap(); + let result = bash + .exec_with_options(&script, ExecOptions::new().stdin(StreamData::from(content))) + .await; + assert!( + matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget( + ExecutionBudgetExceeded::LiveBytes { .. } + ))) + ), + "expected budget rejection" + ); + assert!(!bash.fs().exists(Path::new("/out.tar")).await.unwrap()); + assert_eq!( + bash.exec("echo recovered").await.unwrap().stdout, + "recovered\n" + ); + } +} + +#[tokio::test] +async fn tar_files_from_content_boundary_and_empty_lists() { + for (bytes, succeeds) in [(8000, true), (8192, false), (8193, false)] { + let mut bash = Bash::builder() + .limits(ExecutionLimits::new().max_live_intermediate_bytes(8192)) + .build(); + bash.fs() + .write_file(Path::new("/list"), &vec![b'\n'; bytes]) + .await + .unwrap(); + let result = bash.exec("tar -cf /out.tar -T /list").await; + if succeeds { + let result = result.unwrap(); + assert_eq!(result.exit_code, 2); + assert!(result.stderr.contains("empty archive")); + } else { + assert!( + matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget( + ExecutionBudgetExceeded::LiveBytes { .. } + ))) + ), + "expected budget rejection" + ); + } + } +} + +#[tokio::test] +async fn tar_files_from_preserves_order_duplicates_crlf_and_lossy_names() { + let mut bash = Bash::new(); + for name in ["a", "b", "\u{fffd}"] { + bash.fs() + .write_file(Path::new(&format!("/{name}")), b"data") + .await + .unwrap(); + } + bash.fs() + .write_file(Path::new("/list"), b"a\r\n\nb\na\n\xff") + .await + .unwrap(); + let result = bash + .exec("cd /; tar -cf /out.tar -T /list; tar -tf /out.tar") + .await + .unwrap(); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!(result.stdout, "a\nb\na\n\u{fffd}\n"); +} + +#[tokio::test] +async fn tar_files_from_matches_real_bash_tar() { + let script = "printf a > a; printf b > b; printf 'a\\n\\nb\\na' > list; cp list list2; tar -cf out.tar -T list --files-from=list2; tar -tf out.tar; printf 'b\\n' | tar -cf stdin.tar -T -; tar -tf stdin.tar"; + let mut bash = Bash::new(); + let actual = bash.exec(script).await.unwrap(); + let temp = tempfile::tempdir().unwrap(); + let host_script = if std::process::Command::new("gtar") + .arg("--version") + .output() + .is_ok() + { + script + .replace("; tar ", "; gtar ") + .replace("| tar ", "| gtar ") + } else { + script.to_string() + }; + let expected = std::process::Command::new("bash") + .args(["-c", &host_script]) + .env("COPYFILE_DISABLE", "1") + .current_dir(temp.path()) + .output() + .unwrap(); + assert!( + expected.status.success(), + "{}", + String::from_utf8_lossy(&expected.stderr) + ); + assert_eq!(actual.exit_code, 0, "{}", actual.stderr); + assert_eq!(actual.stdout.as_bytes(), expected.stdout); +} + +#[tokio::test] +async fn tar_files_from_repeated_reads_share_input_budget() { + let mut bash = Bash::builder() + .limits(ExecutionLimits::new().max_aggregate_input_bytes(100)) + .build(); + bash.fs() + .write_file(Path::new("/list"), "a\n".repeat(20).as_bytes()) + .await + .unwrap(); + let result = bash.exec("tar -cf /out.tar -T /list -T /list").await; + assert!(matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget( + ExecutionBudgetExceeded::InputBytes { .. } + ))) + )); + assert!(!bash.fs().exists(Path::new("/out.tar")).await.unwrap()); + assert_eq!( + bash.exec("echo recovered").await.unwrap().stdout, + "recovered\n" + ); +} diff --git a/crates/bashkit/tests/tar_files_from_allocation_tests.rs b/crates/bashkit/tests/tar_files_from_allocation_tests.rs new file mode 100644 index 000000000..94c6ffcd7 --- /dev/null +++ b/crates/bashkit/tests/tar_files_from_allocation_tests.rs @@ -0,0 +1,76 @@ +//! Tar file-list allocation regression. The allocator is process-wide, +//! so this test needs its own binary to exclude unrelated test allocations. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use bashkit::{Bash, Error, ExecOptions, ExecutionLimits, LimitExceeded, StreamData}; +use std::path::Path; + +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +/// Tracks live heap bytes and their high-water mark. +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { + let live = LIVE.fetch_add(layout.size(), Ordering::Relaxed) + layout.size(); + PEAK.fetch_max(live, Ordering::Relaxed); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + LIVE.fetch_sub(layout.size(), Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) }; + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +#[tokio::test(flavor = "current_thread")] +async fn tar_files_from_rejects_before_per_line_allocation_amplification() { + for options in [ + "-T /list", + "--files-from=/list", + "--files-from /list", + "-T /list -T /list", + "-T -", + ] { + let mut bash = Bash::builder() + .limits(ExecutionLimits::new().max_live_intermediate_bytes(65_536)) + .build(); + bash.fs() + .write_file(Path::new("/list"), "x\n".repeat(20_000).as_bytes()) + .await + .unwrap(); + bash.exec("true").await.unwrap(); + let stdin = ExecOptions::new().stdin(StreamData::from("x\n".repeat(20_000))); + let before = LIVE.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let result = bash + .exec_with_options(&format!("tar -cf /out.tar {options}"), stdin) + .await; + let peak = PEAK.load(Ordering::Relaxed).saturating_sub(before); + assert!( + matches!( + result, + Err(Error::ResourceLimit(LimitExceeded::ExecutionBudget(_))) + ), + "{result:?}" + ); + eprintln!("{options}: peak heap growth {peak} bytes"); + // Includes interpreter overhead and the VFS read; the old collector + // retained over 800 KiB before its first live-budget rejection. + assert!(peak < 256 * 1024, "unbudgeted list amplification: {peak}"); + assert!(!bash.fs().exists(Path::new("/out.tar")).await.unwrap()); + assert_eq!( + bash.exec("echo recovered").await.unwrap().stdout, + "recovered\n" + ); + } +} From d8bb6161fe3da9aa5fd94e5c5219fe2b6a1ec2fd Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sat, 10 Oct 2026 23:55:14 -0500 Subject: [PATCH 2/8] chore(knowledge): document tar file-list resource admission --- knowledge/foundations/builtins.md | 14 ++++++++++++++ knowledge/operations/testing.md | 7 +++++++ knowledge/security/threat-model.md | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/knowledge/foundations/builtins.md b/knowledge/foundations/builtins.md index 9549510eb..c8b90df0e 100644 --- a/knowledge/foundations/builtins.md +++ b/knowledge/foundations/builtins.md @@ -652,6 +652,20 @@ bundles (`-cjf`, `cjf`) and `--bzip2`, and detects gzip/bzip2 magic while listin or extracting so `.tar.bz2`/`.tbz2` inputs work without an explicit codec flag. `bzip2`, `bunzip2`, and `bzcat` share the same byte-native codec path. +`tar -T` / `--files-from` admits each VFS read using its metadata size, +retains the read lease through parsing, and charges any extra returned capacity. +The filesystem backend owns allocation admission if a host-backed file grows +between stat and read. Stdin is borrowed. Names are decoded directly into +`BudgetedString` owners held by a `BudgetedVec`; neither per-line copies nor +lossy UTF-8 replacement can allocate before live-budget admission. Shared work +charges cover source bytes, every line (including blanks), and decoding steps, +checking the request deadline/cancellation during dense-list scans. Repeated +sources share input/live/work budgets, preserving order, duplicates, CRLF, +blank lines, stdin, and missing-member behavior. Tests: +`tar_files_from_budget_tests` and the isolated counting-allocator +`tar_files_from_allocation_tests` (TM-DOS-103). + + The implementation uses `bzip2` 0.6 with its default pure-Rust `libbz2-rs-sys` backend. The crate is maintained by the Trifecta Tech Foundation, dual MIT/Apache-2.0; its backend retains the permissive diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 878d8da35..f98d9f233 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -94,6 +94,13 @@ bounded 1,024-entry / 1,024-byte-target request that previously retained over 1 MiB of candidate paths. It complements the shared-budget integration tests rather than substituting budget counters for an actual allocation measurement. +`tar_files_from_allocation_tests.rs` also needs its own binary for its +process-wide counting allocator. It uses a 40 KB newline-dense list and a +64 KB live budget through `Bash::exec`, rejecting the former >1 MB eager +collector peak while asserting bounded rejection and subsequent session use. +The consolidated `tar_files_from_budget_tests` covers live/input/work limits, +stdin, repeated lists, blank/CRLF/lossy names, and GNU tar differential behavior. + Filtering still works as usual: `cargo test --test integration -- foo` matches `integration::*::foo*` test paths. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index f09fb23da..fd656aaa2 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -1416,7 +1416,7 @@ This section maps former vulnerability IDs to the new threat ID scheme and track | TM-DOS-100 | jq control-character normalization amplification | A jq JSON string consisting of literal controls expands sixfold when each byte becomes `\u00XX`; an unmetered compatibility copy can exhaust memory or CPU before strict parsing | The jq-only normalizer charges input-length work before its single pass, borrows unchanged input, and acquires/grows a shared live-intermediate lease before every allocation growth (`builtins/jq/input.rs`), **MITIGATED** | | TM-DOS-101 | yq structured-data amplification | YAML aliases, YAML/JSON nesting, multi-document floods, filters, and output format expansion are all attacker-controlled | A pre-deserialization lexical pass rejects aliases without misclassifying quoted or block scalar content; real parsers with recursion/depth caps, aggregate budgets, shared jaq work/deadline/output controls, and a final rendered-output cap; integration tests cover alias bombs plus YAML+JSON depth/document/input/output bounds, proptest composes arbitrary YAML+filters, and `yq_fuzz` covers stdin/in-place format paths, **MITIGATED** | | TM-DOS-102 | Archive decoder allocation-before-check | A small gzip/bzip2 stream makes the decoder output buffer allocate beyond live-memory or filesystem quotas before the post-growth size check runs | Decoder chunks validate absolute/ratio bounds and acquire a shared execution-budget lease before `try_reserve_exact` and copy; corrupt, truncated, CRC-invalid, ratio-bomb, and live-budget tests fail closed, **MITIGATED** | -| TM-DOS-103 | Post-allocation resource charging | Archive/compression and other growing buffers could call `reserve`/`extend` before taking a live-byte lease; concurrent `fetch_add` accounting could also wrap or transiently overcommit the ceiling | `BudgetedVec`/`BudgetedBytes` and `BudgetedString` own their `ExecutionBudgetLease`, charge planned capacity before `try_reserve_exact`, roll back on allocation/error, and release on drop. CAS admission rejects overflow and concurrent overcommit. Archive creation, extraction/list output, gzip/bzip2 encoders, decompression, and `od` output use the builders. `od` rejects widths above 65,536 bytes before layout arithmetic (including empty input), pads only a single <= 8-byte field on the stack, uses u64 padding products for 32-bit safety, and charges input/field rendering work; `od_resource_tests` cover huge-width rejection, boundary formatting, live/work budgets, and subsequent execution, **MITIGATED** | +| TM-DOS-103 | Post-allocation resource charging | Archive/compression and other growing buffers could call `reserve`/`extend` before taking a live-byte lease; concurrent `fetch_add` accounting could also wrap or transiently overcommit the ceiling | `BudgetedVec`/`BudgetedBytes` and `BudgetedString` own their `ExecutionBudgetLease`, charge planned capacity before `try_reserve_exact`, roll back on allocation/error, and release on drop. CAS admission rejects overflow and concurrent overcommit. Archive creation, extraction/list output, gzip/bzip2 encoders, decompression, and `od` output use the builders. Tar `-T`/`--files-from` leases raw list reads before VFS loading, retains budget-owned names/vector capacity, decodes lossy UTF-8 directly into admitted storage, and charges bytes/lines/decoding work; bounded allocation regressions include nonexistent names, repeated lists, stdin, and session recovery. `od` rejects widths above 65,536 bytes before layout arithmetic (including empty input), pads only a single <= 8-byte field on the stack, uses u64 padding products for 32-bit safety, and charges input/field rendering work; `od_resource_tests` cover huge-width rejection, boundary formatting, live/work budgets, and subsequent execution, **MITIGATED** | | TM-DOS-104 | Pipeline stderr aggregation exceeds the configured output cap before final truncation | A pipeline with many stderr-producing stages can accumulate their combined diagnostics in memory | The pipeline aggregate stops appending at `max_stderr_bytes` and propagates its truncation signal to the final result, **MITIGATED** | | TM-DOS-106 | Zip creation allocation-before-check | `zip` recursively collected complete file contents and encoded its output before acquiring live-byte leases, allowing transient allocations above the configured intermediate-memory ceiling | Zip traversal collects paths only; each VFS file size is leased before `read_file`, leases remain live with the entries, and archive encoding uses `BudgetedBytes` to charge capacity before growth — **MITIGATED** | | TM-DOS-108 | CLI host-stdin read can stall before execution | A producer keeps the one-shot CLI's stdin pipe open while idle or trickles fewer than 10 MiB, so the pre-execution read never reaches EOF or the byte cap | Read stdin on a dedicated thread and bound the wait with the selected execution timeout; timeout exits with a diagnostic suggesting `--no-stdin`, while the existing byte cap remains enforced — **MITIGATED** | From eea62b732a0b0f349b05ce53e0b9e09851eb5bd5 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 11 Oct 2026 00:16:09 -0500 Subject: [PATCH 3/8] test(tar): verify lossy decoding and malformed-input work limits --- crates/bashkit/src/builtins/archive.rs | 30 +++++++++++++++++++ .../tar_files_from_budget_tests.rs | 9 ++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/bashkit/src/builtins/archive.rs b/crates/bashkit/src/builtins/archive.rs index 955cdf75c..544ebd5bf 100644 --- a/crates/bashkit/src/builtins/archive.rs +++ b/crates/bashkit/src/builtins/archive.rs @@ -1952,6 +1952,36 @@ mod tests { // ==================== tar tests ==================== + #[tokio::test] + async fn tar_files_from_lossy_decoder_matches_std() { + let (fs, mut cwd, mut variables) = create_test_ctx().await; + let env = HashMap::new(); + let ctx = Context { + args: &[], + env: &env, + variables: &mut variables, + cwd: &mut cwd, + fs, + stdin: None, + #[cfg(feature = "http_client")] + http_client: None, + #[cfg(feature = "git")] + git_client: None, + #[cfg(feature = "ssh")] + ssh_client: None, + shell: None, + }; + // Valid prefixes, malformed multi-byte sequences, and incomplete + // tails must retain the previous standard-library decoding semantics. + for byte in 0..=255 { + let bytes = [b'A', byte, 0xf0, 0x9f, 0x92, byte, b'Z', 0xe2, 0x82]; + for input in [bytes.as_slice(), &bytes[..7]] { + let name = tar_list_name(&ctx, input).unwrap(); + assert_eq!(&*name, String::from_utf8_lossy(input).as_ref()); + } + } + } + #[tokio::test] async fn test_tar_create_and_list() { let (fs, mut cwd, mut variables) = create_test_ctx().await; diff --git a/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs b/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs index 4f07596c2..72f24067c 100644 --- a/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs +++ b/crates/bashkit/tests/integration/tar_files_from_budget_tests.rs @@ -32,12 +32,17 @@ async fn tar_files_from_large_content_obeys_live_budget() { #[tokio::test] async fn tar_files_from_dense_lines_obey_work_budget() { - for content in ["x\n".repeat(2000), "\n".repeat(2000), "x".repeat(20_000)] { + for content in [ + "x\n".repeat(2000).into_bytes(), + "\n".repeat(2000).into_bytes(), + "x".repeat(20_000).into_bytes(), + vec![0xff; 128], + ] { let mut bash = Bash::builder() .limits(ExecutionLimits::new().max_work_units(100)) .build(); bash.fs() - .write_file(Path::new("/list"), content.as_bytes()) + .write_file(Path::new("/list"), &content) .await .unwrap(); let result = bash.exec("tar -cf /out.tar -T /list").await; From 4b333e0ee67df716f1786d8766c1577ca492abac Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 11 Oct 2026 00:26:47 -0500 Subject: [PATCH 4/8] chore(knowledge): conform tar admission notes to OKF formatting --- knowledge/foundations/builtins.md | 1 - 1 file changed, 1 deletion(-) diff --git a/knowledge/foundations/builtins.md b/knowledge/foundations/builtins.md index c8b90df0e..909f869f5 100644 --- a/knowledge/foundations/builtins.md +++ b/knowledge/foundations/builtins.md @@ -665,7 +665,6 @@ blank lines, stdin, and missing-member behavior. Tests: `tar_files_from_budget_tests` and the isolated counting-allocator `tar_files_from_allocation_tests` (TM-DOS-103). - The implementation uses `bzip2` 0.6 with its default pure-Rust `libbz2-rs-sys` backend. The crate is maintained by the Trifecta Tech Foundation, dual MIT/Apache-2.0; its backend retains the permissive From 73e27ed4f50698ecea907e5f502cdac3217530a3 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sun, 11 Oct 2026 13:10:15 -0500 Subject: [PATCH 5/8] fix(parser): bound nested substitution stack frames --- crates/bashkit/docs/threat-model.md | 11 +- crates/bashkit/src/parser/mod.rs | 2071 +++++++++-------- .../integration/blackbox_security_tests.rs | 53 + 3 files changed, 1189 insertions(+), 946 deletions(-) diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index 511c233f5..9d694185f 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -111,7 +111,7 @@ through configurable limits. | Brace range OOM (TM-DOS-041) | `{1..999999999}` | Cap range size | **MITIGATED** | | Brace combinatorial (TM-DOS-042) | `{1..100}{1..100}{1..100}` | Cap total expansion | **MITIGATED** | | Compound assign overflow (TM-DOS-043) | `((x+=1))` with x=i64::MAX | `wrapping_*` ops | **MITIGATED** | -| Lexer stack overflow (TM-DOS-044) | ~50 nested `$()` in quotes | Depth tracking | **MITIGATED** | +| Lexer stack overflow (TM-DOS-044) | ~50 nested `$()` in quotes | Shared AST depth, heap-owned child parsers, outlined nonrecursive parser frames | **MITIGATED** | | parse_word_string limits (TM-DOS-050) | Parameter expansion ignores limits | Propagate limits | **MITIGATED** | | Removed YAML helper parser (TM-DOS-051) | Former custom parser recursed over indentation | Parser deleted; yq uses TM-DOS-101 controls | **REMOVED** | | Template engine recursion (TM-DOS-052) | Nested `{{#if}}`/`{{#each}}` overflow | Add depth limit | **MITIGATED** | @@ -656,15 +656,18 @@ The parser includes multiple layers of depth protection to prevent stack overflo attacks: 1. **Configurable depth limit** (`max_ast_depth`, default 100): Controls maximum nesting - of compound commands (if/for/while/case/subshell). + of compound commands (if/for/while/case/subshell) and substitutions. 2. **Hard cap** (`HARD_MAX_AST_DEPTH = 100`): Even if the caller configures a higher `max_ast_depth`, the parser clamps it to 100. This prevents misconfiguration from causing stack overflow. 3. **Child parser inheritance** (TM-DOS-021): When parsing `$(...)` or `<(...)`, - the child parser inherits the *remaining* depth budget and fuel from the parent. - This prevents attackers from bypassing depth limits through nested substitutions. + each substitution consumes one level of the enclosing AST depth. Command + substitution child parsers inherit the same ceiling, remaining fuel, execution + budget, and timeout origin, and stay heap-owned while parsing. Nonrecursive + parameter parsing and AST assembly run outside recursive frames, keeping the + unchanged hard ceiling safe on default 2 MiB Rust test-thread stacks. 4. **Arithmetic depth limit** (TM-DOS-026): The arithmetic evaluator (`$((expr))`) has its own depth limit (`MAX_ARITHMETIC_DEPTH = 50`) to prevent stack overflow diff --git a/crates/bashkit/src/parser/mod.rs b/crates/bashkit/src/parser/mod.rs index b68384495..bb44d0e52 100644 --- a/crates/bashkit/src/parser/mod.rs +++ b/crates/bashkit/src/parser/mod.rs @@ -4,6 +4,10 @@ //! //! # Design Notes //! +//! Substitution parsers share the enclosing AST depth and start time. Keep child +//! parsers heap-owned and nonrecursive AST assembly out of recursive frames so +//! the hard depth ceiling is safe on default 2 MiB test-thread stacks (TM-DOS-044). +//! //! Reserved words (like `done`, `fi`, `then`) are only treated as special in command //! position - when they would start a command. In argument position, they are regular //! words. The termination of compound commands is handled by `parse_compound_list_until` @@ -492,6 +496,12 @@ impl<'a> Parser<'a> { /// Parse the input and return the AST. pub fn parse(mut self) -> Result