From bb5092bb1528689a2fc6b0df2bfe814b10e230eb Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 03:25:54 +0000 Subject: [PATCH 01/11] fix(interpreter): [[ ]] extglob, nocasematch, and quoted extglob groups [[ == ]] always matches with extglob (bash 5.2). shopt nocasematch folds case for case, [[ ==/=~ ]] and ${v/pat/rep}. !(..) may consume an empty prefix; escaped | and quotes inside a group stay literal; an unbalanced group is literal. Extglob matching now spends a shared step budget (TM-DOS-031). --- crates/bashkit/src/interpreter/expansion.rs | 28 ++- crates/bashkit/src/interpreter/glob.rs | 227 ++++++++++++------ crates/bashkit/src/interpreter/mod.rs | 56 ++++- crates/bashkit/src/parser/lexer.rs | 47 +++- .../bash/bashbox-shopt-pattern.test.sh | 7 - 5 files changed, 274 insertions(+), 91 deletions(-) diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 8ae400e54..88fed888d 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -1920,8 +1920,10 @@ impl Interpreter { || Self::find_unescaped_char(pat, '?').is_some() || Self::find_unescaped_char(pat, '[').is_some(); let extglob = self.contains_unescaped_extglob(pat); + // `shopt -s nocasematch` folds case in `${v/pat/rep}` (not in removal). + let nocase = self.is_nocasematch(); - if !has_glob && !extglob { + if !has_glob && !extglob && !nocase { let literal = Self::unescape_pattern_literal(pat); return Some(match anchor { PatternAnchor::Start if value.starts_with(&literal) => vec![(0, literal.len())], @@ -1943,7 +1945,7 @@ impl Interpreter { }); } - if !extglob && let Some(re) = Self::glob_pattern_regex(pat, anchor) { + if !extglob && let Some(re) = Self::glob_pattern_regex(pat, anchor, nocase) { // THREAT[TM-DOS-127]: the regex crate matches in linear time and // the compiled program is size-capped, so a long value or pattern // cannot blow up the search. @@ -1967,7 +1969,11 @@ impl Interpreter { return Some(ranges); } - self.find_pattern_matches_glob(value, pat, anchor, global) + let opts = glob::PatternOpts { + nocase, + ..self.shell_pattern_opts() + }; + self.find_pattern_matches_glob(value, pat, anchor, global, opts) } /// Extglob fallback, bash's own scan: at each position take the longest @@ -1980,6 +1986,7 @@ impl Interpreter { pat: &str, anchor: PatternAnchor, global: bool, + opts: glob::PatternOpts, ) -> Option> { const MAX_GLOB_MATCH_CALLS: usize = 10_000; let bounds: Vec = value @@ -1989,6 +1996,7 @@ impl Interpreter { .collect(); let n = bounds.len() - 1; let mut calls = 0usize; + let steps = std::cell::Cell::new(Self::MAX_GLOB_STEPS); let mut ranges = Vec::new(); let mut s = 0; loop { @@ -2006,7 +2014,10 @@ impl Interpreter { if calls > MAX_GLOB_MATCH_CALLS { return None; } - if self.glob_match(&value[bounds[s]..bounds[e]], pat) { + if steps.get() == 0 { + return None; + } + if self.glob_match_steps(&value[bounds[s]..bounds[e]], pat, opts, 0, &steps) { found = Some(e); break; } @@ -2028,7 +2039,7 @@ impl Interpreter { /// Translate a bash glob (with `\\c` escapes) into an anchored-as-needed /// regex. `None` when it does not translate or compile; callers then use /// the glob fallback. - fn glob_pattern_regex(pat: &str, anchor: PatternAnchor) -> Option { + fn glob_pattern_regex(pat: &str, anchor: PatternAnchor, nocase: bool) -> Option { let chars: Vec = pat.chars().collect(); let mut re = String::from("(?s)"); if anchor == PatternAnchor::Start { @@ -2072,6 +2083,7 @@ impl Interpreter { re.push_str("\\z"); } regex::RegexBuilder::new(&re) + .case_insensitive(nocase) .size_limit(1 << 20) .dfa_size_limit(1 << 20) .build() @@ -2270,6 +2282,8 @@ impl Interpreter { const MAX_GLOB_MATCH_CALLS: usize = 10_000; let chars: Vec = value.chars().collect(); let mut calls = 0usize; + let steps = std::cell::Cell::new(Self::MAX_GLOB_STEPS); + let opts = self.shell_pattern_opts(); if prefix { // Try each prefix length; shortest = first match, longest = last match let mut last_match = None; @@ -2279,7 +2293,7 @@ impl Interpreter { break; } let candidate: String = chars[..i].iter().collect(); - if self.glob_match(&candidate, pattern) { + if self.glob_match_steps(&candidate, pattern, opts, 0, &steps) { if !longest { return chars[i..].iter().collect(); } @@ -2298,7 +2312,7 @@ impl Interpreter { break; } let candidate: String = chars[i..].iter().collect(); - if self.glob_match(&candidate, pattern) { + if self.glob_match_steps(&candidate, pattern, opts, 0, &steps) { if !longest { return chars[..i].iter().collect(); } diff --git a/crates/bashkit/src/interpreter/glob.rs b/crates/bashkit/src/interpreter/glob.rs index 8254a9f4d..5e2c550b8 100644 --- a/crates/bashkit/src/interpreter/glob.rs +++ b/crates/bashkit/src/interpreter/glob.rs @@ -61,14 +61,27 @@ pub(super) fn expand_posix_class(name: &str, out: &mut Vec) { } } +/// Matching knobs for one pattern match. Callers pick them per context: +/// `[[ == ]]` always understands extglob (bash 5.2), `case` and parameter +/// expansion follow `shopt extglob`; `nocasematch` folds case for `case`, +/// `[[ ]]` and `${v/pat/rep}` but not for `${v#pat}` removal; filename +/// expansion folds case under `nocaseglob`. +#[derive(Clone, Copy, Default)] +pub(crate) struct PatternOpts { + pub(crate) nocase: bool, + pub(crate) extglob: bool, +} + impl Interpreter { // ── Pattern matching ────────────────────────────────────────────── - /// Check if pattern contains extglob operators + /// Check if pattern contains extglob operators (only when `shopt extglob`). pub(crate) fn contains_extglob(&self, s: &str) -> bool { - if !self.is_extglob() { - return false; - } + self.is_extglob() && Self::has_extglob_op(s) + } + + /// Unescaped `@(`, `?(`, `*(`, `+(` or `!(` anywhere in `s`. + fn has_extglob_op(s: &str) -> bool { let mut escaped = false; let mut chars = s.chars().peekable(); while let Some(ch) = chars.next() { @@ -87,21 +100,57 @@ impl Interpreter { false } - /// Check if a value matches a shell pattern + /// Options for `case` patterns and parameter-expansion patterns: + /// `shopt extglob`, case-sensitive. + pub(crate) fn shell_pattern_opts(&self) -> PatternOpts { + PatternOpts { + nocase: false, + extglob: self.is_extglob(), + } + } + + /// Options for filename expansion: `shopt extglob` and `nocaseglob`. + pub(crate) fn glob_opts(&self) -> PatternOpts { + PatternOpts { + nocase: self.is_nocaseglob(), + extglob: self.is_extglob(), + } + } + + /// Check if `shopt -s nocasematch` is enabled. + pub(crate) fn is_nocasematch(&self) -> bool { + self.scoped + .variables + .get("SHOPT_nocasematch") + .map(|v| v == "1") + .unwrap_or(false) + } + + /// Check if a value matches a shell pattern (`shopt extglob`, case-sensitive). pub(crate) fn pattern_matches(&self, value: &str, pattern: &str) -> bool { + self.pattern_matches_opts(value, pattern, self.shell_pattern_opts()) + } + + /// Check if a value matches a shell pattern with explicit options. + pub(crate) fn pattern_matches_opts( + &self, + value: &str, + pattern: &str, + opts: PatternOpts, + ) -> bool { // Handle special case of * (match anything) if pattern == "*" { return true; } // Glob pattern matching with unescaped *, ?, [], and extglob support - if self.contains_glob_chars(pattern) || self.contains_extglob(pattern) { - self.glob_match(value, pattern) + if self.contains_glob_chars(pattern) || (opts.extglob && Self::has_extglob_op(pattern)) { + self.glob_match_impl(value, pattern, opts, 0) } else { // Literal match; a backslash still escapes the next character // (`\*` from quoted pattern text matches a literal `*`, and an // unquoted `a\b` pattern matches `ab`, as in bash). - if pattern.contains('\\') { + let literal = if pattern.contains('\\') { let mut literal = String::with_capacity(pattern.len()); let mut chars = pattern.chars(); while let Some(ch) = chars.next() { @@ -110,18 +159,22 @@ impl Interpreter { _ => literal.push(ch), } } - value == literal + literal + } else { + pattern.to_string() + }; + if opts.nocase { + value.chars().count() == literal.chars().count() + && value + .chars() + .zip(literal.chars()) + .all(|(a, b)| a.to_lowercase().eq(b.to_lowercase())) } else { - value == pattern + value == literal } } } - /// Simple glob pattern matching with support for *, ?, and [...] - pub(crate) fn glob_match(&self, value: &str, pattern: &str) -> bool { - self.glob_match_impl(value, pattern, false, 0) - } - /// Parse an extglob pattern-list from pattern string starting after '('. /// Returns (alternatives, rest_of_pattern) or None if malformed. fn parse_extglob_pattern_list(pattern: &str) -> Option<(Vec, String)> { @@ -140,8 +193,16 @@ impl Interpreter { let mut alts = Vec::new(); let mut current = String::new(); let mut d = 0; - for c in inner.chars() { + let mut inner_chars = inner.chars(); + while let Some(c) = inner_chars.next() { match c { + // `\|` and `\)` stay literal inside an alternative. + '\\' => { + current.push(c); + if let Some(next) = inner_chars.next() { + current.push(next); + } + } '(' => { d += 1; current.push(c); @@ -171,20 +232,52 @@ impl Interpreter { None // unclosed paren } - /// Glob match with optional case-insensitive mode + /// `rest` starts at the `(` after an extglob operator and the group + /// closes; an unbalanced `@(a` is literal text, as in bash. + fn extglob_group_closes(rest: &std::iter::Peekable>) -> bool { + let mut rest = rest.clone(); + if rest.next() != Some('(') { + return false; + } + let tail: String = rest.collect(); + Self::parse_extglob_pattern_list(&tail).is_some() + } + + /// Glob match with explicit options and a fresh step budget. pub(crate) fn glob_match_impl( &self, value: &str, pattern: &str, - nocase: bool, + opts: PatternOpts, + depth: usize, + ) -> bool { + let steps = std::cell::Cell::new(Self::MAX_GLOB_STEPS); + self.glob_match_steps(value, pattern, opts, depth, &steps) + } + + /// Glob match drawing on a caller-owned step budget, so a loop of + /// matches (pattern removal/substitution scans) shares one budget. + pub(crate) fn glob_match_steps( + &self, + value: &str, + pattern: &str, + opts: PatternOpts, depth: usize, + steps: &std::cell::Cell, ) -> bool { // THREAT[TM-DOS-031]: Bail on excessive recursion depth if depth >= Self::MAX_GLOB_DEPTH { return false; } + // THREAT[TM-DOS-031]: extglob alternation/repetition backtracks + // exponentially (`+(a|aa)` vs `aaa…b`); every recursive step spends + // from a shared budget and the match fails once it is exhausted. + if steps.get() == 0 { + return false; + } + steps.set(steps.get() - 1); - let extglob = self.is_extglob(); + let PatternOpts { nocase, extglob } = opts; // THREAT[TM-DOS-039]: Once the remaining pattern has no closing bracket, // every `[` is literal. Cache that state so an unmatched suffix is scanned // at most once instead of from every subsequent `[` start. @@ -196,7 +289,7 @@ impl Interpreter { if matches!(bytes[0], b'@' | b'?' | b'*' | b'+' | b'!') && bytes[1] == b'(' { let op = bytes[0]; if let Some((alts, rest)) = Self::parse_extglob_pattern_list(&pattern[2..]) { - return self.match_extglob(op, &alts, &rest, value, nocase, depth + 1); + return self.match_extglob(op, &alts, &rest, value, opts, depth + 1, steps); } } } @@ -254,15 +347,16 @@ impl Interpreter { // Check for extglob *(...) let mut pc_clone = pattern_chars.clone(); pc_clone.next(); - if extglob && pc_clone.peek() == Some(&'(') { + if extglob && Self::extglob_group_closes(&pc_clone) { // Extglob *(pattern-list) — collect remaining pattern let remaining_pattern: String = pattern_chars.collect(); let remaining_value: String = value_chars.collect(); - if self.glob_match_impl( + if self.glob_match_steps( &remaining_value, &remaining_pattern, - nocase, + opts, depth + 1, + steps, ) { return true; } @@ -285,14 +379,15 @@ impl Interpreter { // Check for extglob ?(...) let mut pc_clone = pattern_chars.clone(); pc_clone.next(); - if extglob && pc_clone.peek() == Some(&'(') { + if extglob && Self::extglob_group_closes(&pc_clone) { let remaining_pattern: String = pattern_chars.collect(); let remaining_value: String = value_chars.collect(); - if self.glob_match_impl( + if self.glob_match_steps( &remaining_value, &remaining_pattern, - nocase, + opts, depth + 1, + steps, ) { return true; } @@ -353,14 +448,15 @@ impl Interpreter { if extglob && matches!(p, '@' | '+' | '!') { let mut pc_clone = pattern_chars.clone(); pc_clone.next(); - if pc_clone.peek() == Some(&'(') { + if Self::extglob_group_closes(&pc_clone) { let remaining_pattern: String = pattern_chars.collect(); let remaining_value: String = value_chars.collect(); - if self.glob_match_impl( + if self.glob_match_steps( &remaining_value, &remaining_pattern, - nocase, + opts, depth + 1, + steps, ) { return true; } @@ -394,8 +490,9 @@ impl Interpreter { alts: &[String], rest: &str, value: &str, - nocase: bool, + opts: PatternOpts, depth: usize, + steps: &std::cell::Cell, ) -> bool { // THREAT[TM-DOS-031]: Bail on excessive recursion depth if depth >= Self::MAX_GLOB_DEPTH { @@ -407,7 +504,7 @@ impl Interpreter { // @(a|b) — exactly one of the alternatives for alt in alts { let full = format!("{}{}", alt, rest); - if self.glob_match_impl(value, &full, nocase, depth + 1) { + if self.glob_match_steps(value, &full, opts, depth + 1, steps) { return true; } } @@ -416,13 +513,13 @@ impl Interpreter { b'?' => { // ?(a|b) — zero or one of the alternatives // Try zero: skip the extglob entirely - if self.glob_match_impl(value, rest, nocase, depth + 1) { + if self.glob_match_steps(value, rest, opts, depth + 1, steps) { return true; } // Try one for alt in alts { let full = format!("{}{}", alt, rest); - if self.glob_match_impl(value, &full, nocase, depth + 1) { + if self.glob_match_steps(value, &full, opts, depth + 1, steps) { return true; } } @@ -437,7 +534,7 @@ impl Interpreter { .chain(std::iter::once(value.len())); for alt in alts { let full = format!("{}{}", alt, rest); - if self.glob_match_impl(value, &full, nocase, depth + 1) { + if self.glob_match_steps(value, &full, opts, depth + 1, steps) { return true; } // Try alt followed by more +(a|b)rest @@ -445,11 +542,11 @@ impl Interpreter { for split in split_points.clone() { let prefix = &value[..split]; let suffix = &value[split..]; - if self.glob_match_impl(prefix, alt, nocase, depth + 1) { + if self.glob_match_steps(prefix, alt, opts, depth + 1, steps) { // Rebuild the extglob for the suffix let inner = alts.join("|"); let re_pattern = format!("+({}){}", inner, rest); - if self.glob_match_impl(suffix, &re_pattern, nocase, depth + 1) { + if self.glob_match_steps(suffix, &re_pattern, opts, depth + 1, steps) { return true; } } @@ -460,7 +557,7 @@ impl Interpreter { b'*' => { // *(a|b) — zero or more of the alternatives // Try zero - if self.glob_match_impl(value, rest, nocase, depth + 1) { + if self.glob_match_steps(value, rest, opts, depth + 1, steps) { return true; } let split_points = value @@ -471,16 +568,16 @@ impl Interpreter { // Try one or more (same as +(...)) for alt in alts { let full = format!("{}{}", alt, rest); - if self.glob_match_impl(value, &full, nocase, depth + 1) { + if self.glob_match_steps(value, &full, opts, depth + 1, steps) { return true; } for split in split_points.clone() { let prefix = &value[..split]; let suffix = &value[split..]; - if self.glob_match_impl(prefix, alt, nocase, depth + 1) { + if self.glob_match_steps(prefix, alt, opts, depth + 1, steps) { let inner = alts.join("|"); let re_pattern = format!("*({}){}", inner, rest); - if self.glob_match_impl(suffix, &re_pattern, nocase, depth + 1) { + if self.glob_match_steps(suffix, &re_pattern, opts, depth + 1, steps) { return true; } } @@ -489,35 +586,19 @@ impl Interpreter { false } b'!' => { - // !(a|b) — match anything except one of the alternatives - // Try every possible split point: prefix must NOT match any alt, rest matches - // Actually: !(pat) matches anything that doesn't match @(pat) - let inner = alts.join("|"); - let positive = format!("@({}){}", inner, rest); - !self.glob_match_impl(value, &positive, nocase, depth + 1) - && self.glob_match_impl(value, rest, nocase, depth + 1) - || { - // !(pat) can also consume characters — try each split - for split in value - .char_indices() - .map(|(i, _)| i) - .skip(1) - .chain(std::iter::once(value.len())) - { - let prefix = &value[..split]; - let suffix = &value[split..]; - // prefix must not match any alt - let prefix_matches_any = alts - .iter() - .any(|a| self.glob_match_impl(prefix, a, nocase, depth + 1)); - if !prefix_matches_any - && self.glob_match_impl(suffix, rest, nocase, depth + 1) - { - return true; - } - } - false - } + // !(a|b) consumes some prefix (possibly empty) that matches + // none of the alternatives, and `rest` matches what remains. + value + .char_indices() + .map(|(i, _)| i) + .chain(std::iter::once(value.len())) + .any(|split| { + let (prefix, suffix) = value.split_at(split); + !alts + .iter() + .any(|a| self.glob_match_steps(prefix, a, opts, depth + 1, steps)) + && self.glob_match_steps(suffix, rest, opts, depth + 1, steps) + }) } _ => false, } @@ -831,7 +912,7 @@ impl Interpreter { } let dotglob = self.is_dotglob(); - let nocase = self.is_nocaseglob(); + let opts = self.glob_opts(); let is_absolute = pattern.starts_with('/'); // Empty components collapse `//` and drop a trailing `/`, matching the @@ -885,7 +966,7 @@ impl Interpreter { if !is_last && !entry.metadata.file_type.is_dir() { continue; } - if self.glob_match_impl(&entry.name, component, nocase, 0) { + if self.glob_match_impl(&entry.name, component, opts, 0) { matched.push(entry.name); } } @@ -936,7 +1017,7 @@ impl Interpreter { let is_absolute = pattern.starts_with('/'); let components: Vec<&str> = pattern.split('/').filter(|s| !s.is_empty()).collect(); let dotglob = self.is_dotglob(); - let nocase = self.is_nocaseglob(); + let opts = self.glob_opts(); // Find the ** component let star_star_idx = match components.iter().position(|&c| c == "**") { @@ -993,7 +1074,7 @@ impl Interpreter { if entry.name.starts_with('.') && !dotglob && !pattern_starts_with_dot { continue; } - if self.glob_match_impl(&entry.name, pat, nocase, 0) { + if self.glob_match_impl(&entry.name, pat, opts, 0) { matches.push(vfs_join(dir, &entry.name).to_string_lossy().to_string()); } } diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index a72c5abea..4920d1d79 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -1587,6 +1587,9 @@ impl Interpreter { // Decision: restored `$!` must match Bashkit-produced virtual numeric job ids. const MAX_RESTORED_LAST_BG_PID_LEN: usize = 20; const MAX_GLOB_DEPTH: usize = 50; + /// THREAT[TM-DOS-031]: recursive glob/extglob steps one match (or one + /// pattern-removal/substitution scan) may spend before failing. + const MAX_GLOB_STEPS: usize = 100_000; /// Create a new interpreter with the given filesystem. #[cfg(test)] @@ -4593,8 +4596,23 @@ impl Interpreter { 3 => { // Binary operators match args[1].as_str() { - "=" | "==" => self.pattern_matches(&args[0], &args[2]), - "!=" => !self.pattern_matches(&args[0], &args[2]), + // bash 5.2: `[[ == ]]` matches as if extglob were on. + "=" | "==" => self.pattern_matches_opts( + &args[0], + &args[2], + glob::PatternOpts { + nocase: self.is_nocasematch(), + extglob: true, + }, + ), + "!=" => !self.pattern_matches_opts( + &args[0], + &args[2], + glob::PatternOpts { + nocase: self.is_nocasematch(), + extglob: true, + }, + ), "<" => args[0] < args[2], ">" => args[0] > args[2], "-eq" => self.cond_int_cmp(&args[0], &args[2], |a, b| a == b), @@ -4687,6 +4705,9 @@ impl Interpreter { // read them as plain bracket members. let compiled = if ere_has_invalid_char_class(pattern) { None + } else if self.is_nocasematch() { + // `shopt -s nocasematch` makes `=~` case-insensitive too. + self.regex_cache.get_or_compile(&format!("(?i){pattern}")) } else { self.regex_cache.get_or_compile(pattern) }; @@ -4829,7 +4850,11 @@ impl Interpreter { let mut m = false; for pattern in &case_item.patterns { let pattern_str = self.expand_pattern_word(pattern).await?; - if self.pattern_matches(&word_value, &pattern_str) { + let opts = glob::PatternOpts { + nocase: self.is_nocasematch(), + extglob: self.is_extglob(), + }; + if self.pattern_matches_opts(&word_value, &pattern_str, opts) { m = true; break; } @@ -13672,6 +13697,31 @@ mod tests { assert_eq!(result.exit_code, 0); } + /// THREAT[TM-DOS-031]: `[[ == ]]` understands extglob without `shopt`, + /// so backtracking alternation against a near-miss value must stay + /// bounded by the per-match step budget. + #[tokio::test] + async fn test_cond_extglob_backtracking_is_bounded() { + use crate::time_compat::Instant; + use std::time::Duration; + let start = Instant::now(); + let result = run_script( + r#"x=$(printf 'a%.0s' {1..60})b +[[ $x == +(a|aa) ]] && echo m1 || echo n1 +[[ $x == +(+(a)|+(aa)) ]] && echo m2 || echo n2 +[[ $x == *(a|aa|aaa)*(a|aa)!(z) ]] && echo m3 || echo n3 +y=${x//+(a|aa)/-}; echo ${#y}"#, + ) + .await; + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_secs(10), + "extglob took too long: {:?}", + elapsed + ); + assert!(result.stdout.starts_with("n1\nn2\n"), "{}", result.stdout); + } + // Issue #425: $$ should not leak real host PID #[tokio::test] async fn test_dollar_dollar_no_host_pid_leak() { diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index 2ce770053..d4f71b486 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -1084,10 +1084,43 @@ impl<'a> Lexer<'a> { } '\\' => { if let Some(esc) = self.peek_char() { - word.push(esc); + if matches!(esc, '$' | '`') { + // Keep `\$` literal through parse_word. + word.pop(); + Self::push_extglob_literal(&mut word, esc); + } else { + word.push(esc); + } self.advance(); } } + // Quotes inside a group make their text literal: + // `@(a|'b)')` has the alternative `b)`. + '\'' | '"' => { + word.pop(); + let quote = c; + while let Some(q) = self.peek_char() { + self.advance(); + if q == quote { + break; + } + if quote == '"' + && q == '\\' + && let Some(esc) = self.peek_char() + && matches!(esc, '"' | '\\' | '$' | '`') + { + self.advance(); + Self::push_extglob_literal(&mut word, esc); + continue; + } + if quote == '"' && matches!(q, '$' | '`') { + // Expansions still run inside "...". + word.push(q); + continue; + } + Self::push_extglob_literal(&mut word, q); + } + } _ => {} } } @@ -1638,6 +1671,18 @@ impl<'a> Lexer<'a> { } /// Characters `escape_glob_metas_in_quoted_ranges` backslash-escapes. + /// Push one quoted character of an extglob group so it stays literal: + /// pattern metacharacters get a backslash, `$`/`` ` `` the NUL sentinel. + fn push_extglob_literal(word: &mut String, ch: char) { + if Self::is_glob_escape_char(ch) || matches!(ch, '"' | '\'') { + word.push('\\'); + } else if matches!(ch, '$' | '`') { + word.push('\\'); + word.push('\x00'); + } + word.push(ch); + } + fn is_glob_escape_char(ch: char) -> bool { matches!( ch, diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-pattern.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-pattern.test.sh index 58188e9cc..07558dce5 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-pattern.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-pattern.test.sh @@ -4,7 +4,6 @@ ### bashbox_shopt_pattern_always_understands_extglob # [[ ]] always understands extglob -### skip: TODO bashbox corpus gap, bashkit output differs from real bash x=abc; [[ $x == @(a|b)* ]] && echo y1; [[ $x == +([a-c]) ]] && echo y2; [[ $x == !(a*) ]] || echo y3; [[ "" == !(x) ]] && echo y4; [[ "a|b" == @(a\|b) ]] && echo y5; [[ abab == +(ab|a) ]] && echo y6; [[ b == *(a)b ]] && echo y7; [[ ab == ?(a)b ]] && echo y8 ### expect y1 @@ -19,7 +18,6 @@ y8 ### bashbox_shopt_pattern_case_needs_shopt_s_extglob # case needs shopt -s extglob -### skip: TODO bashbox corpus gap, bashkit output differs from real bash p='@(abc)'; case abc in $p) echo no;; *) echo plain;; esac shopt -s extglob case abc in $p) echo yes;; esac; case abc in @(abc)) echo yes;; esac; case a in !(a)*) echo m1;; esac @@ -60,7 +58,6 @@ q7 ### bashbox_shopt_pattern_nocasematch_covers_case_and_substitution_not_removal # nocasematch covers case, [[ ]] and substitution, not removal -### skip: TODO bashbox corpus gap, bashkit output differs from real bash x=ABC; shopt -s nocasematch case ABC in a*) echo m1;; esac; [[ ABC == ab? ]] && echo m2; [[ ABC =~ ^ab ]] && echo m3; [[ ABC = abc ]] && echo m4; echo ${x#a} ${x/b/z} ${x%c} ${x//b/z} ${x/#a/q}; [[ $x != abc ]]; echo $? shopt -u nocasematch; [[ ABC == abc ]] || echo m5 @@ -83,7 +80,6 @@ BC A ABC ABC ### bashbox_shopt_pattern_array_and_positional_operations_apply_per_element # array and positional operations apply per element -### skip: TODO bashbox corpus gap, bashkit output differs from real bash set -- ab ac; echo "${@#a}" "${@/c/x}"; a=(xa ya); echo "${a[@]%a}" "${a[*]^^}"; x=aab; echo ${x/q/z} ${x/#/-} ${x/%/-} "${x//b/\$}"; declare -A m=([k1]=a [k2]=b); for k in "${!m[@]}"; do echo "<$k>"; done ### expect b c ab ax @@ -95,7 +91,6 @@ aab -aab aab- aa$ ### bashbox_shopt_pattern_quotes_inside_a_group # quotes inside a group -### skip: TODO bashbox corpus gap, bashkit output differs from real bash [[ "b)" == @(a|'b)') ]] && echo q1; [[ 'a"' == @("a\"") ]] && echo q2; [[ 'a|' == @(a\|) ]] && echo q3 ### expect q1 @@ -112,7 +107,6 @@ lit ### bashbox_shopt_pattern_a_negation_nested_in_another_group # a negation nested in another group -### skip: TODO bashbox corpus gap, bashkit output differs from real bash [[ c == @(!(a)|b) ]] && echo y1; [[ a == @(!(a)) ]] || echo y2; [[ ab == !(x)? ]] && echo y3 ### expect y1 @@ -122,7 +116,6 @@ y3 ### bashbox_shopt_pattern_groups_next_to_a_negation # groups next to a negation -### skip: TODO bashbox corpus gap, bashkit output differs from real bash [[ abab == !(x)+(ab) ]] && echo y1; [[ ab == @(a)!(x) ]] && echo y2; [[ b == ?(a)!(z) ]] && echo y3; [[ aab == *(a)!(z) ]] && echo y4; [[ ab == +(a|ab)!(z) ]] && echo y5; [[ aaa == !(z)*(a|aa) ]] && echo y6; [[ x == !(z)+(a) ]] || echo y7; [[ ab == !(x)+(a|b) ]] && echo y8 ### expect y1 From cf515a8578206965f58b2526769efda57d851a33 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 03:29:17 +0000 Subject: [PATCH 02/11] fix(interpreter): substring/slice ranges and ${#@} count ${s:off:-n} counts back from the end, an offset before the start or past the end selects nothing, a negative array/positional length is bash's 'substring expression < 0' line abort, and ${#@}/${#*} count the positional parameters. --- crates/bashkit/src/interpreter/expansion.rs | 113 +++++++++++------- .../bash/bashbox-process-substitution.test.sh | 2 - .../bash/bashbox-shopt-glob.test.sh | 1 - .../bash/bashbox-word-expander.test.sh | 4 - 4 files changed, 71 insertions(+), 49 deletions(-) diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 88fed888d..7852030d3 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -426,6 +426,10 @@ impl Interpreter { let start = (bracket_pos + 1).min(index_end); let index_str = &name[start..index_end]; self.expand_array_access_part(arr_name, index_str) + } else if name == "@" || name == "*" { + // `${#@}` / `${#*}` count the positional parameters. + result.push_str(&self.expand_variable("#")); + continue; } else { self.expand_variable(name) }; @@ -517,7 +521,9 @@ impl Interpreter { offset, length, } if name == "@" || name == "*" => { - let items = self.positional_slice(offset, length.as_deref()); + let items = self + .positional_slice(offset, length.as_deref()) + .map_err(crate::error::Error::LineAbort)?; let sep = if name == "*" { self.get_ifs_separator() } else { @@ -531,19 +537,10 @@ impl Interpreter { length, } => { let value = self.expand_variable(name); - let char_count = value.chars().count(); - let offset_val: isize = self.evaluate_arithmetic(offset) as isize; - let start = if offset_val < 0 { - (char_count as isize + offset_val).max(0) as usize - } else { - (offset_val as usize).min(char_count) - }; - let substr: String = if let Some(len_expr) = length { - let len_val = self.evaluate_arithmetic(len_expr) as usize; - value.chars().skip(start).take(len_val).collect() - } else { - value.chars().skip(start).collect() - }; + let (start, end) = self + .slice_range(value.chars().count(), offset, length.as_deref(), true) + .map_err(crate::error::Error::LineAbort)?; + let substr: String = value.chars().skip(start).take(end - start).collect(); Self::append_expansion_for_word(&mut result, word, &substr); } WordPart::ArraySlice { @@ -553,22 +550,14 @@ impl Interpreter { } => { if let Some(items) = self.array_view(name) { let values: Vec = items.into_iter().map(|(_, v)| v).collect(); - - let offset_val: isize = self.evaluate_arithmetic(offset) as isize; - let start = if offset_val < 0 { - (values.len() as isize + offset_val).max(0) as usize - } else { - (offset_val as usize).min(values.len()) - }; - - let sliced = if let Some(len_expr) = length { - let len_val = self.evaluate_arithmetic(len_expr) as usize; - let end = start.saturating_add(len_val).min(values.len()); - &values[start..end] - } else { - &values[start..] - }; - Self::append_expansion_for_word(&mut result, word, &sliced.join(" ")); + let (start, end) = self + .slice_range(values.len(), offset, length.as_deref(), false) + .map_err(crate::error::Error::LineAbort)?; + Self::append_expansion_for_word( + &mut result, + word, + &values[start..end].join(" "), + ); } } WordPart::IndirectExpansion { @@ -1516,7 +1505,8 @@ impl Interpreter { offset, length, } if name == "@" || name == "*" => { - let items = self.positional_slice(offset, length.as_deref()); + // A range error is raised by the plain expansion path. + let items = self.positional_slice(offset, length.as_deref()).ok()?; (name, items) } _ => return None, @@ -1526,7 +1516,11 @@ impl Interpreter { /// `${@:offset:length}`: positional parameters counted from `$0` /// (offset 0 includes it); a negative offset counts back from the last. - fn positional_slice(&mut self, offset: &str, length: Option<&str>) -> Vec { + fn positional_slice( + &mut self, + offset: &str, + length: Option<&str>, + ) -> std::result::Result, String> { let positional = self .call_stack .last() @@ -1535,22 +1529,57 @@ impl Interpreter { let mut all = Vec::with_capacity(positional.len() + 1); all.push(self.expand_variable("0")); all.extend(positional); + let (start, end) = self.slice_range(all.len(), offset, length, false)?; + all.truncate(end); + Ok(all.split_off(start)) + } + + /// `[start, end)` of `${v:offset:length}` over `count` items. A negative + /// offset counts back from the end; one reaching before the start, or an + /// offset past the end, selects nothing. A negative length counts back + /// from the end for strings (`neg_len_from_end`); for arrays and `$@`, or + /// when it ends before `start`, it is bash's `substring expression < 0`. + fn slice_range( + &mut self, + count: usize, + offset: &str, + length: Option<&str>, + neg_len_from_end: bool, + ) -> std::result::Result<(usize, usize), String> { let off = self.evaluate_arithmetic(offset); let start = if off < 0 { let back = usize::try_from(off.unsigned_abs()).unwrap_or(usize::MAX); - match all.len().checked_sub(back) { - Some(s) => s, - None => return Vec::new(), - } + count.checked_sub(back) } else { - usize::try_from(off).unwrap_or(usize::MAX).min(all.len()) + usize::try_from(off).ok().filter(|&o| o <= count) }; - let mut items: Vec = all.into_iter().skip(start).collect(); - if let Some(len) = length { - let n = self.evaluate_arithmetic(len); - items.truncate(usize::try_from(n.max(0)).unwrap_or(usize::MAX)); + let len = length.map(|l| (l, self.evaluate_arithmetic(l))); + if let Some((expr, n)) = len + && n < 0 + { + let end = usize::try_from(n.unsigned_abs()) + .ok() + .and_then(|back| count.checked_sub(back)); + match (neg_len_from_end, start, end) { + (true, Some(s), Some(e)) if e >= s => return Ok((s, e)), + (true, None, _) => return Ok((0, 0)), + _ => { + return Err( + self.arith_diag("", &format!("{}: substring expression < 0", expr.trim())) + ); + } + } } - items + let Some(start) = start else { + return Ok((0, 0)); + }; + let end = match len { + Some((_, n)) => start + .saturating_add(usize::try_from(n).unwrap_or(usize::MAX)) + .min(count), + None => count, + }; + Ok((start, end)) } fn is_elementwise_op(operator: &ParameterOp) -> bool { diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh index b005819d7..479602b71 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh @@ -95,7 +95,6 @@ b ### bashbox_process_substitution_nested_substitutions # nested substitutions -### skip: TODO bashbox corpus gap, bashkit output differs from real bash cat <(cat <(echo inner)); cat <(echo $(echo cmd)) ### expect inner @@ -139,7 +138,6 @@ y ### bashbox_process_substitution_a_function_listing_keeps_it # a function listing keeps it -### skip: TODO bashbox corpus gap, bashkit output differs from real bash f(){ cat <(echo a) > >(cat); }; declare -f f ### expect f () diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh index b4bfc67ff..b4e30326d 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh @@ -4,7 +4,6 @@ ### bashbox_shopt_glob_nullglob_drops_a_pattern_that_matches_nothing # nullglob drops a pattern that matches nothing -### skip: TODO bashbox corpus gap, bashkit output differs from real bash mkdir -p d/e; touch a.txt b.txt .hid C.TXT d/x.txt d/e/y.txt shopt -s nullglob; echo a *.nope b; x=(*.nope); echo ${#x[@]}; echo "*.nope" ### expect diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-word-expander.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-word-expander.test.sh index b2735fea8..2c2c0d420 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-word-expander.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-word-expander.test.sh @@ -4,7 +4,6 @@ ### bashbox_word_expander_braced_positional_and_counts # braced positional and counts -### skip: TODO bashbox corpus gap, bashkit output differs from real bash set -- a b; echo ${1} ${#} ${#@} ${#*} ### expect a 2 2 2 @@ -33,7 +32,6 @@ x x ### bashbox_word_expander_substring_with_negative_offset_length # substring with negative offset/length -### skip: TODO bashbox corpus gap, bashkit output differs from real bash s=hello; echo ${s: -2} ${s:1:-1} ${s: -3:2} "[${s: -10}]" ${s:1+1:2} ### expect lo ell ll [] ll @@ -41,7 +39,6 @@ lo ell ll [] ll ### bashbox_word_expander_array_slices # array slices -### skip: TODO bashbox corpus gap, bashkit output differs from real bash a=(1 2 3 4); echo ${a[@]: -2} ${a[@]:1:2} "[${a[@]: -9}]" ### expect 3 4 2 3 [] @@ -289,7 +286,6 @@ f(){ for x in "$@"; do echo "[$x]"; done; }; f a "b c" ### bashbox_word_expander_escaped_comma_and_non_expanding_braces # escaped comma and non-expanding braces -### skip: TODO bashbox corpus gap, bashkit output differs from real bash echo {a\,b,c} {a}b {abc} {1..c} ### expect a,b c {a}b {abc} {1..c} From de067ee3c4d78557e3e56aa1f7999fdc9ab42f86 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 03:31:59 +0000 Subject: [PATCH 03/11] fix(shopt): report unknown names and still apply the rest Each invalid name is reported and skipped; the valid names in the same call still take effect. shopt -s/-u with -o keeps status 0 on a bad name, as bash does. Document execution-time alias expansion as a limitation. --- crates/bashkit/src/builtins/vars.rs | 90 +++++++++---------- .../spec_cases/bash/bashbox-shopt.test.sh | 4 +- knowledge/operations/limitations.md | 1 + 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/crates/bashkit/src/builtins/vars.rs b/crates/bashkit/src/builtins/vars.rs index b2df1efb1..ac869b36e 100644 --- a/crates/bashkit/src/builtins/vars.rs +++ b/crates/bashkit/src/builtins/vars.rs @@ -530,14 +530,17 @@ fn shopt_set_o( } else { opts.iter().map(String::as_str).collect() }; - // An unknown name fails the whole command before anything changes. - if let Some(bad) = names.iter().find(|n| set_option_by_name(n).is_none()) { - return ExecResult::err(format!("bash: shopt: {bad}: invalid option name\n"), 1); - } + // Each unknown name is reported and skipped; the rest still apply. + // bash keeps status 0 for `-s`/`-u` and fails the other verbs. let mut out = String::new(); + let mut err = String::new(); let mut status = 0; for name in names { let Some((_, _, var, default)) = set_option_by_name(name) else { + err.push_str(&format!("bash: shopt: {name}: invalid option name\n")); + if !matches!(mode, Some('s' | 'u')) { + status = 1; + } continue; }; match mode { @@ -563,6 +566,7 @@ fn shopt_set_o( } } let mut result = ExecResult::ok(out); + result.stderr = err.into(); result.exit_code = status; result } @@ -623,33 +627,48 @@ impl Builtin for Shopt { return Ok(shopt_set_o(ctx.variables, mode, &opts)); } + // Each unknown name is reported and skipped (status 1); the + // valid names around it still apply. + let mut err = String::new(); + if !opts.is_empty() { + opts.retain(|opt| { + let known = SHOPT_OPTIONS.contains(&opt.as_str()); + if !known { + err.push_str(&format!("bash: shopt: {opt}: invalid shell option name\n")); + } + known + }); + if !err.is_empty() { + let mut result = if opts.is_empty() { + ExecResult::ok(String::new()) + } else { + Self::run_valid(ctx, mode, opts) + }; + result.stderr = err.into(); + result.exit_code = 1; + return Ok(result); + } + } + Ok(Self::run_valid(ctx, mode, opts)) + } +} + +impl Shopt { + /// Run `shopt` on names already checked against `SHOPT_OPTIONS`. + fn run_valid(ctx: Context<'_>, mode: Option, opts: Vec) -> ExecResult { match mode { Some('s') => { - // Set options for opt in &opts { - if !SHOPT_OPTIONS.contains(&opt.as_str()) { - return Ok(ExecResult::err( - format!("bash: shopt: {}: invalid shell option name\n", opt), - 1, - )); - } ctx.variables .insert(format!("SHOPT_{}", opt), "1".to_string()); } - Ok(ExecResult::ok(String::new())) + ExecResult::ok(String::new()) } Some('u') => { - // Unset options for opt in &opts { - if !SHOPT_OPTIONS.contains(&opt.as_str()) { - return Ok(ExecResult::err( - format!("bash: shopt: {}: invalid shell option name\n", opt), - 1, - )); - } ctx.variables.remove(&format!("SHOPT_{}", opt)); } - Ok(ExecResult::ok(String::new())) + ExecResult::ok(String::new()) } Some('q') => { // Query: exit 0 if all named options are on, 1 otherwise @@ -657,13 +676,13 @@ impl Builtin for Shopt { let key = format!("SHOPT_{}", opt); ctx.variables.get(&key).map(|v| v == "1").unwrap_or(false) }); - Ok(ExecResult { + ExecResult { stdout: crate::StreamData::new(), stderr: crate::StreamData::new(), exit_code: if all_on { 0 } else { 1 }, control_flow: crate::interpreter::ControlFlow::None, ..Default::default() - }) + } } Some('p') => { // Print in reusable format @@ -681,7 +700,7 @@ impl Builtin for Shopt { let on = ctx.variables.get(&key).map(|v| v == "1").unwrap_or(false); output.push_str(&format!("shopt {} {}\n", if on { "-s" } else { "-u" }, opt)); } - Ok(ExecResult::ok(output)) + ExecResult::ok(output) } None => { // No flag: show status of named options @@ -697,36 +716,17 @@ impl Builtin for Shopt { if on { "on" } else { "off" } )); } - return Ok(ExecResult::ok(output)); + return ExecResult::ok(output); } let mut output = String::new(); - let mut any_invalid = false; for opt in &opts { - if !SHOPT_OPTIONS.contains(&opt.as_str()) { - output.push_str(&format!( - "bash: shopt: {}: invalid shell option name\n", - opt - )); - any_invalid = true; - continue; - } let key = format!("SHOPT_{}", opt); let on = ctx.variables.get(&key).map(|v| v == "1").unwrap_or(false); output.push_str(&format!("{:<15}\t{}\n", opt, if on { "on" } else { "off" })); } - if any_invalid { - Ok(ExecResult { - stdout: crate::StreamData::new(), - stderr: output.into(), - exit_code: 1, - control_flow: crate::interpreter::ControlFlow::None, - ..Default::default() - }) - } else { - Ok(ExecResult::ok(output)) - } + ExecResult::ok(output) } - _ => Ok(ExecResult::ok(String::new())), + _ => ExecResult::ok(String::new()), } } } diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt.test.sh index 1f632abae..eefdb25ae 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt.test.sh @@ -4,7 +4,6 @@ ### bashbox_shopt_unknown_names_are_reported_but_the_rest_still_apply # unknown names are reported but the rest still apply -### skip: TODO bashbox corpus gap, bashkit output differs from real bash shopt -s extglob nope nullglob; echo s=$?; shopt -p nullglob extglob; shopt -q nope; echo s=$? ### expect s=1 @@ -24,7 +23,6 @@ s=2 ### bashbox_shopt_o_with_a_bad_name # -o with a bad name -### skip: TODO bashbox corpus gap, bashkit output differs from real bash shopt -so nope; echo s=$?; shopt -uo nope; echo s=$?; shopt -o nope; echo s=$?; shopt -po nope errexit; echo s=$? ### expect s=0 @@ -43,7 +41,7 @@ shopt -s lastpipe; echo a | read v; echo "[$v]" ### bashbox_shopt_aliases_expand_only_with_expand_aliases_from_the_next_line_o # aliases expand only with expand_aliases, from the next line on -### skip: TODO bashbox corpus gap, bashkit output differs from real bash +### skip: aliases expand when the command runs, not when its line is read (limitations.md, Aliases row) alias e='echo hi' e there shopt -s expand_aliases diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 5df8bb994..5f6e9d0d0 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -108,6 +108,7 @@ Target: IEEE 1003.1-2024 Shell Command Language. | `bash`/`sh` | `-c`, `-n`, `-e`, `-x`, `-u`, `-f`, `-o option`, script files, stdin, `--version`, `--help` | Login shell | | Expansion errors | `set -u` unbound variables stop the shell | `$((1/0))` evaluates to 0 and `${x!}` expands empty; bash reports them and drops the rest of the line the failing command ends on (`bashbox_line_number_an_expansion_error_drops_the_rest_of_the_line_it_ends_on`, skipped) | | Process substitution | `<(cmd)` and `>(cmd)` as filenames, nesting, redirections | The path is `/dev/fd/proc_sub_`, not bash's `/dev/fd/63`. Several interpreters can share one filesystem, so a per-shell descending number would let one tenant's substitution overwrite another's. A script that parses the number sees a different one (`scripts/bash-oracle/cases/process-substitution-dev-fd.sh`) | +| Aliases | `alias`/`unalias`, `shopt -s expand_aliases`, trailing-space chaining, recursion guard | Expanded when the simple command runs, not when its line is read: bash parses a whole line (and a function body at definition) before an `alias`/`unalias` on it takes effect; Bashkit applies them to the very next command | | Named fds / `<>` | `{var}>file`, `{var}file`, `{var}>>file`, `{var}<<&N`/`<&N` allocate the lowest free fd >= 10 on any command (persisting, as in bash); `{var}>&-` closes; `>&N` with N closed is `Bad file descriptor`; `N<>file`, `N<<file` has no shared read/write offset (writes append); `3&1`, `( ... ) 3>&1`) is dropped; a `Bad file descriptor` message ignores redirect order (`2>&1 >&7`) | | Locale | UTF-8 text everywhere; `${#v}` counts bytes when `LC_ALL`/`LC_CTYPE`/`LANG` (first non-empty) is `C` or `POSIX` | With no locale variable set bashkit is UTF-8, where bash falls back to C. Under C, `${s:off:len}`, `${s^^}`, `${s#?}` and glob `?` still work on characters, not bytes: shell words are Rust strings and cannot hold a split UTF-8 sequence (L-STREAM-001) | | Brace expansion | Runs before other expansions on unquoted literal text; lists, nesting, ranges with step and zero padding, quoted/escaped braces and commas | Works on parsed words, not raw text: `$v{1,2}` expands `$v` then appends `1`/`2` (bash reads `$v1`, `$v2`); write `${v}` | From 7e66339917aad121d718da9f18190cb82288b15f Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 03:37:43 +0000 Subject: [PATCH 04/11] fix(glob): globstar ** as a path component, trailing-slash globs ** under globstar matches zero or more directories in any component position (files too when last), with relative output and bash's d/ vs d spelling for the zero-directory match. A trailing / keeps only directories and stays on each match (d/*/ -> d/e/). --- crates/bashkit/src/interpreter/glob.rs | 196 ++++++++---------- .../glob_intermediate_component_tests.rs | 3 +- .../bash/bashbox-shopt-glob.test.sh | 2 - 3 files changed, 91 insertions(+), 110 deletions(-) diff --git a/crates/bashkit/src/interpreter/glob.rs b/crates/bashkit/src/interpreter/glob.rs index 5e2c550b8..619b58ff3 100644 --- a/crates/bashkit/src/interpreter/glob.rs +++ b/crates/bashkit/src/interpreter/glob.rs @@ -905,18 +905,17 @@ impl Interpreter { /// `/skills/*/SKILL.md` by globbing each component in turn against the VFS. /// Non-final components only match directories (a plain file can't be /// descended into), which is what makes read-only tree mounts globbable. + /// A trailing `/` keeps only directories and stays on every match + /// (`d/*/` gives `d/e/`). Under `shopt -s globstar` a `**` component + /// matches zero or more directories, and files too when it is last. pub(crate) async fn expand_glob(&self, pattern: &str) -> Result> { - // Check for ** (recursive glob) — only when globstar is enabled - if pattern.contains("**") && self.is_globstar() { - return self.expand_glob_recursive(pattern).await; - } - let dotglob = self.is_dotglob(); + let globstar = self.is_globstar(); let opts = self.glob_opts(); let is_absolute = pattern.starts_with('/'); + let trailing_slash = pattern.len() > 1 && pattern.ends_with('/'); - // Empty components collapse `//` and drop a trailing `/`, matching the - // previous `Path::file_name()` behaviour for patterns like `/dir/*/`. + // Empty components collapse `//`; a trailing `/` is tracked above. let components: Vec<&str> = pattern.split('/').filter(|s| !s.is_empty()).collect(); if components.is_empty() { return Ok(Vec::new()); @@ -942,11 +941,43 @@ impl Interpreter { String::new(), )]; + let mut literal_prefix = true; for (idx, component) in components.iter().enumerate() { let is_last = idx + 1 == components.len(); + // Only directories can carry the rest of the pattern or a `/`. + let dirs_only = !is_last || trailing_slash; let mut next: Vec<(PathBuf, String)> = Vec::new(); - if self.contains_glob_chars(component) || self.contains_extglob(component) { + if globstar && *component == "**" { + for (dir, out) in &candidates { + // Zero directories: the candidate itself. A bare `**` + // never yields the empty cwd path; a literal `d/**` + // yields `d/`, a matched `*/**` yields plain `d`. + if !is_last { + next.push((dir.clone(), out.clone())); + } else if !out.is_empty() { + let own = if literal_prefix { + format!("{out}/") + } else { + out.clone() + }; + next.push((dir.clone(), own)); + } + self.globstar_walk( + dir, + out, + is_absolute, + dotglob, + dirs_only, + max_candidates, + &mut next, + ) + .await; + if next.len() > max_candidates { + break; + } + } + } else if self.contains_glob_chars(component) || self.contains_extglob(component) { // Dotfiles are hidden per component unless dotglob is set or this // component explicitly starts with '.'. let component_starts_with_dot = component.starts_with('.'); @@ -962,8 +993,7 @@ impl Interpreter { if entry.name.starts_with('.') && !dotglob && !component_starts_with_dot { continue; } - // Only a directory can carry the rest of the pattern. - if !is_last && !entry.metadata.file_type.is_dir() { + if dirs_only && !entry.metadata.file_type.is_dir() { continue; } if self.glob_match_impl(&entry.name, component, opts, 0) { @@ -1004,115 +1034,67 @@ impl Interpreter { return Ok(Vec::new()); } candidates = next; + literal_prefix &= + !(self.contains_glob_chars(component) || self.contains_extglob(component)); } // Sort matches alphabetically (bash behavior) - let mut matches: Vec = candidates.into_iter().map(|(_, out)| out).collect(); - matches.sort(); - Ok(matches) - } - - /// Expand a glob pattern containing ** (recursive directory matching). - async fn expand_glob_recursive(&self, pattern: &str) -> Result> { - let is_absolute = pattern.starts_with('/'); - let components: Vec<&str> = pattern.split('/').filter(|s| !s.is_empty()).collect(); - let dotglob = self.is_dotglob(); - let opts = self.glob_opts(); - - // Find the ** component - let star_star_idx = match components.iter().position(|&c| c == "**") { - Some(i) => i, - None => return Ok(Vec::new()), - }; - - // Build the base directory from components before ** - let base_dir = if is_absolute { - let mut p = PathBuf::from("/"); - for c in &components[..star_star_idx] { - p.push(c); - } - p - } else { - let mut p = self.cwd.clone(); - for c in &components[..star_star_idx] { - p.push(c); - } - p - }; - - // Pattern components after ** - let after_pattern: Vec<&str> = components[star_star_idx + 1..].to_vec(); - - // Collect all directories recursively (including the base) - let mut all_dirs = vec![base_dir.clone()]; - // THREAT[TM-DOS-049]: Cap recursion depth using filesystem path depth limit - let max_depth = self.fs.limits().max_path_depth; - self.collect_dirs_recursive(&base_dir, &mut all_dirs, max_depth, dotglob) - .await; - - let mut matches = Vec::new(); - - for dir in &all_dirs { - if after_pattern.is_empty() { - // ** alone matches all files recursively - if let Ok(entries) = self.fs.read_dir(dir).await { - for entry in entries { - if entry.name.starts_with('.') && !dotglob { - continue; - } - if !entry.metadata.file_type.is_dir() { - matches.push(vfs_join(dir, &entry.name).to_string_lossy().to_string()); - } - } + let mut matches: Vec = candidates + .into_iter() + .map(|(_, mut out)| { + if trailing_slash && !out.ends_with('/') { + out.push('/'); } - } else if after_pattern.len() == 1 { - // Single pattern after **: match files in this directory - let pat = after_pattern[0]; - let pattern_starts_with_dot = pat.starts_with('.'); - if let Ok(entries) = self.fs.read_dir(dir).await { - for entry in entries { - if entry.name.starts_with('.') && !dotglob && !pattern_starts_with_dot { - continue; - } - if self.glob_match_impl(&entry.name, pat, opts, 0) { - matches.push(vfs_join(dir, &entry.name).to_string_lossy().to_string()); - } - } - } - } - } - + out + }) + .collect(); matches.sort(); + matches.dedup(); Ok(matches) } - /// Recursively collect all subdirectories starting from dir. - /// THREAT[TM-DOS-049]: `max_depth` caps recursion to prevent stack exhaustion. - pub(crate) fn collect_dirs_recursive<'a>( - &'a self, - dir: &'a Path, - result: &'a mut Vec, - max_depth: usize, + /// Every directory below `dir` (and every file too unless `dirs_only`), + /// for a globstar `**` component. Hidden entries need `dotglob`. + /// THREAT[TM-DOS-049]/[TM-DOS-095]: iterative, depth-capped by the + /// filesystem path-depth limit, and stops at `max_candidates` results. + #[allow(clippy::too_many_arguments)] + async fn globstar_walk( + &self, + dir: &Path, + out: &str, + is_absolute: bool, dotglob: bool, - ) -> std::pin::Pin + Send + 'a>> { - Box::pin(async move { - if max_depth == 0 { - return; + dirs_only: bool, + max_candidates: usize, + next: &mut Vec<(PathBuf, String)>, + ) { + let max_depth = self.fs.limits().max_path_depth; + let mut stack: Vec<(PathBuf, String, usize)> = + vec![(dir.to_path_buf(), out.to_string(), 0)]; + while let Some((dir, out, depth)) = stack.pop() { + if depth >= max_depth || next.len() > max_candidates { + continue; } - if let Ok(entries) = self.fs.read_dir(dir).await { - for entry in entries { - if entry.metadata.file_type.is_dir() { - if entry.name.starts_with('.') && !dotglob { - continue; - } - let subdir = vfs_join(dir, &entry.name); - result.push(subdir.clone()); - self.collect_dirs_recursive(&subdir, result, max_depth - 1, dotglob) - .await; - } + let Ok(mut entries) = self.fs.read_dir(&dir).await else { + continue; + }; + entries.sort_by(|a, b| b.name.cmp(&a.name)); + for entry in entries { + if entry.name.starts_with('.') && !dotglob { + continue; + } + let is_dir = entry.metadata.file_type.is_dir(); + if !is_dir && dirs_only { + continue; + } + let path = vfs_join(&dir, &entry.name); + let output = Self::glob_join_output(&out, &entry.name, is_absolute); + next.push((path.clone(), output.clone())); + if is_dir { + stack.push((path, output, depth + 1)); } } - }) + } } } diff --git a/crates/bashkit/tests/integration/glob_intermediate_component_tests.rs b/crates/bashkit/tests/integration/glob_intermediate_component_tests.rs index 8d6a70bc8..28022a509 100644 --- a/crates/bashkit/tests/integration/glob_intermediate_component_tests.rs +++ b/crates/bashkit/tests/integration/glob_intermediate_component_tests.rs @@ -171,7 +171,8 @@ async fn trailing_slash_glob_still_lists_directories() { let result = bash.exec("echo /skills/*/").await.unwrap(); - assert_eq!(result.stdout, "/skills/pdf /skills/xlsx\n"); + // bash keeps the trailing slash on every match. + assert_eq!(result.stdout, "/skills/pdf/ /skills/xlsx/\n"); } /// Per-component expansion walks the VFS, so it must not become a traversal diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh index b4e30326d..df18e6802 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-shopt-glob.test.sh @@ -35,7 +35,6 @@ c.txt ### bashbox_shopt_glob_globstar_makes_recurse # globstar makes ** recurse -### skip: TODO bashbox corpus gap, bashkit output differs from real bash mkdir -p d/e; touch a.txt b.txt .hid C.TXT d/x.txt d/e/y.txt shopt -s globstar; echo **; echo **/*.txt; echo d/**; echo d/**/; shopt -u globstar; echo **/*.txt ### expect @@ -48,7 +47,6 @@ d/x.txt ### bashbox_shopt_glob_a_literal_last_part_must_exist # a literal last part must exist -### skip: TODO bashbox corpus gap, bashkit output differs from real bash mkdir -p d/e; touch a.txt b.txt .hid C.TXT d/x.txt d/e/y.txt echo */x.txt */nope.txt; echo d/*/; echo "*".txt \*.txt ### expect From 6223b3ae93698691ca0b6e69e12a0463486a40b3 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 03:49:32 +0000 Subject: [PATCH 05/11] fix(parser): $'...' is literal inside double quotes and here-docs A $ before ' or the closing " in double quotes starts no expansion ("$'q'" and "$"'q' print $'q' and $q); here-document bodies keep $'a' as written. Function listings print $'...' as the single-quoted text it decodes to and $"..." as "...", as bash does. --- crates/bashkit/src/parser/lexer.rs | 67 +++++++++++++++++-- crates/bashkit/src/parser/mod.rs | 7 +- crates/bashkit/src/parser/print_cmd.rs | 46 ++++++++++--- .../bash/bashbox-ansi-c-quoting.test.sh | 3 - 4 files changed, 104 insertions(+), 19 deletions(-) diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index d4f71b486..922ee77b3 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -625,8 +625,11 @@ impl<'a> Lexer<'a> { } // Track quoted expansions for IFS-split suppression if c == '$' && quote_char == '"' { - word.push(c); self.advance(); + if self.take_dquote_dollar(&mut word) { + has_quoted_expansion = true; + continue; + } if self.peek_char().is_some_and(|nc| { nc.is_ascii_alphanumeric() || nc == '_' @@ -634,6 +637,19 @@ impl<'a> Lexer<'a> { }) { has_quoted_expansion = true; } + // Read `$(...)`/`${...}` whole so quotes inside them + // keep their own meaning. + if self.peek_char() == Some('(') { + word.push('('); + self.advance(); + self.read_command_subst_into(&mut word); + } else if self.peek_char() == Some('{') { + word.push('{'); + self.advance(); + if let Err(msg) = self.read_param_expansion_into(&mut word) { + return Some(Token::Error(msg)); + } + } continue; } if quote_char == '\'' && c == '$' { @@ -800,8 +816,11 @@ impl<'a> Lexer<'a> { // Handle $(...) inside double-quoted word segments // to preserve single-quoted strings within command substitutions if c == '$' && quote_char == '"' { - word.push(c); self.advance(); + if self.take_dquote_dollar(&mut word) { + has_quoted_expansion = true; + continue; + } // Mark that this word contains a quoted expansion so IFS // splitting is suppressed (e.g. +"$fmt" stays one field). if self.peek_char().is_some_and(|nc| { @@ -817,6 +836,15 @@ impl<'a> Lexer<'a> { self.read_command_subst_into(&mut word); continue; } + if self.peek_char() == Some('{') { + // `${...}` body in double-quote context, so a + // `$'...'` operand inside it stays ANSI-C quoting. + word.push('{'); + self.advance(); + if let Err(msg) = self.read_param_expansion_into(&mut word) { + return Some(Token::Error(msg)); + } + } continue; } if quote_char == '\'' && c == '$' { @@ -904,8 +932,10 @@ impl<'a> Lexer<'a> { } } if c == '$' { - word.push(c); self.advance(); + if self.take_dquote_dollar(&mut word) { + continue; + } if let Some(nc) = self.peek_char() { if nc == '{' { word.push(nc); @@ -1671,6 +1701,33 @@ impl<'a> Lexer<'a> { } /// Characters `escape_glob_metas_in_quoted_ranges` backslash-escapes. + /// Decode the body of `$'...'` at the start of `s` (just after `$'`): + /// the resolved text and the bytes consumed through the closing `'`. + pub(crate) fn decode_ansi_c_body(s: &str) -> Option<(String, usize)> { + let mut lexer = Lexer::new(s); + let (text, closed) = lexer.read_dollar_single_quoted_content(); + closed.then_some((text, lexer.position.offset)) + } + + /// Push a `$` just consumed inside double quotes. Before `'` + /// (`"$'q'"`) or the closing `"` it starts no expansion, so it gets the + /// NUL sentinel and stays a literal `$` even when quoted text follows + /// (`"$"'q'`). `$$` is consumed whole (returns true) so its second `$` + /// is not mistaken for a lone one before `"`. + fn take_dquote_dollar(&mut self, word: &mut String) -> bool { + let next = self.peek_char(); + if matches!(next, Some('\'' | '"') | None) { + word.push('\x00'); + } + word.push('$'); + if next == Some('$') { + word.push('$'); + self.advance(); + return true; + } + false + } + /// Push one quoted character of an extglob group so it stays literal: /// pattern metacharacters get a backslash, `$`/`` ` `` the NUL sentinel. fn push_extglob_literal(word: &mut String, ch: char) { @@ -1750,8 +1807,10 @@ impl<'a> Lexer<'a> { } } '$' => { - content.push('$'); self.advance(); + if self.take_dquote_dollar(&mut content) { + continue; + } if self.peek_char() == Some('(') { // $(...) command substitution — track paren depth content.push('('); diff --git a/crates/bashkit/src/parser/mod.rs b/crates/bashkit/src/parser/mod.rs index 7c541cde6..37c6baad2 100644 --- a/crates/bashkit/src/parser/mod.rs +++ b/crates/bashkit/src/parser/mod.rs @@ -4620,7 +4620,7 @@ pub(crate) fn arith_exec_text(raw: &str) -> String { /// for its own parser (only `\` is removed); backticks become /// `$(...)`. fn heredoc_body_escapes(content: &str) -> String { - if !content.contains(['\\', '`']) { + if !content.contains(['\\', '`']) && !content.contains("$'") { return content.to_string(); } let mut out = String::with_capacity(content.len()); @@ -4637,6 +4637,11 @@ fn heredoc_body_escapes(content: &str) -> String { } _ => out.push('\\'), }, + // `$'...'` is not ANSI-C quoting in a here-document. + '$' if chars.peek() == Some(&'\'') => { + out.push('\x00'); + out.push('$'); + } '$' if matches!(chars.peek(), Some('(' | '{')) => { out.push('$'); let open = chars.next().unwrap_or_default(); diff --git a/crates/bashkit/src/parser/print_cmd.rs b/crates/bashkit/src/parser/print_cmd.rs index 693feac03..8cb098fd0 100644 --- a/crates/bashkit/src/parser/print_cmd.rs +++ b/crates/bashkit/src/parser/print_cmd.rs @@ -653,21 +653,25 @@ fn assignment_text(a: &Assignment) -> String { /// reconstruction when the parser did not record the source. fn word_text(w: &Word) -> String { match &w.raw { - Some(raw) => strip_line_continuations(raw), + Some(raw) => normalize_raw_word(raw), None => reconstruct_word(w), } } -/// Bash's lexer drops backslash-newline outside single quotes. -fn strip_line_continuations(raw: &str) -> String { - if !raw.contains("\\\n") { +/// Source text the way bash prints it back: backslash-newline is dropped +/// outside single quotes, `$'...'` becomes the single-quoted text it decodes +/// to (`$'it\'s'` prints `'it'\''s'`) and `$"..."` prints as `"..."`. +/// Inside double quotes both stay as written. +fn normalize_raw_word(raw: &str) -> String { + if !raw.contains("\\\n") && !raw.contains("$'") && !raw.contains("$\"") { return raw.to_string(); } let mut out = String::with_capacity(raw.len()); - let mut chars = raw.chars().peekable(); let mut in_single = false; let mut in_double = false; - while let Some(c) = chars.next() { + let mut i = 0; + while let Some(c) = raw[i..].chars().next() { + i += c.len_utf8(); match c { '\'' if !in_double => { in_single = !in_single; @@ -677,13 +681,24 @@ fn strip_line_continuations(raw: &str) -> String { in_double = !in_double; out.push(c); } + '$' if !in_single && !in_double && raw[i..].starts_with('\'') => { + match super::lexer::Lexer::decode_ansi_c_body(&raw[i + 1..]) { + Some((text, used)) => { + out.push_str(&super::raw::single_quote(&text)); + i += 1 + used; + } + None => out.push(c), + } + } + '$' if !in_single && !in_double && raw[i..].starts_with('"') => {} '\\' if !in_single => { - if chars.peek() == Some(&'\n') { - chars.next(); + if raw[i..].starts_with('\n') { + i += 1; } else { out.push(c); - if let Some(n) = chars.next() { + if let Some(n) = raw[i..].chars().next() { out.push(n); + i += n.len_utf8(); } } } @@ -920,8 +935,17 @@ mod tests { #[test] fn line_continuation_dropped_outside_single_quotes() { - assert_eq!(strip_line_continuations("a\\\nb"), "ab"); - assert_eq!(strip_line_continuations("'a\\\nb'"), "'a\\\nb'"); + assert_eq!(normalize_raw_word("a\\\nb"), "ab"); + assert_eq!(normalize_raw_word("'a\\\nb'"), "'a\\\nb'"); + } + + #[test] + fn ansi_c_and_locale_quotes_print_as_plain_quotes() { + assert_eq!(normalize_raw_word("$'a\\tb'"), "'a\tb'"); + assert_eq!(normalize_raw_word("$'it\\'s'"), "'it'\\''s'"); + assert_eq!(normalize_raw_word("$\"y\""), "\"y\""); + assert_eq!(normalize_raw_word("\"$'x'\""), "\"$'x'\""); + assert_eq!(normalize_raw_word("'$\"z\"'"), "'$\"z\"'"); } #[test] diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-ansi-c-quoting.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-ansi-c-quoting.test.sh index f3c466baa..e8694aa3b 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-ansi-c-quoting.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-ansi-c-quoting.test.sh @@ -4,7 +4,6 @@ ### bashbox_ansi_c_quoting_the_result_is_quoted_text # the result is quoted text -### skip: TODO bashbox corpus gap, bashkit output differs from real bash x=$'a\nb'; echo "$x"; echo "$'q'" '$'"'q'"; echo $"dq $x"; echo $'it\'s' $'a''b' x$'\t'y; touch zz; echo $'z'* $'*' ### expect a @@ -18,7 +17,6 @@ zz * ### bashbox_ansi_c_quoting_type_shows_it_single_quoted # type shows it single-quoted -### skip: TODO bashbox corpus gap, bashkit output differs from real bash f() { echo $'a\tb' $'it\'s' "$'x'" $"y"; } type f ### expect @@ -31,7 +29,6 @@ f () ### bashbox_ansi_c_quoting_here_documents_expand_as_in_double_quotes_quotes_stay # here-documents expand as in double quotes, quotes stay -### skip: TODO bashbox corpus gap, bashkit output differs from real bash x=1; cat < Date: Thu, 8 Oct 2026 03:54:50 +0000 Subject: [PATCH 06/11] fix(interpreter): \x01/\x02 in values stay data Mixed quoted/unquoted words now mark their quoted spans with the parser's quote-boundary markers, which parse_word turns into per-part quotedness, instead of in-band \x01/\x02 that expansion stripped from every value. Field splitting picks its protected-span markers from characters absent from the data, like the empty-field sentinel. --- crates/bashkit/src/interpreter/expansion.rs | 59 +++++++++++++------ crates/bashkit/src/interpreter/mod.rs | 5 -- crates/bashkit/src/parser/lexer.rs | 16 +++-- .../bash/bashbox-marker-bytes.test.sh | 2 - 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 7852030d3..b9b3138c2 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -216,7 +216,6 @@ impl Interpreter { ) -> std::pin::Pin> + Send + 'a>> { Box::pin(async move { let expanded = self.expand_word_inner(word).await?; - let expanded = Self::strip_quote_markers(&expanded); // A `QuotedGlobWord` carries glob escapes for its quoted text; // contexts that do not glob (command names, redirect targets, // operands) want the text itself. @@ -283,7 +282,6 @@ impl Interpreter { /// quoted expansions from `append_expansion_for_word`. pub(super) async fn expand_pattern_word(&mut self, word: &Word) -> Result { let expanded = Box::pin(self.expand_word_inner(word)).await?; - let expanded = Self::strip_quote_markers(&expanded); if word.quoted && !word.has_unquoted_glob { Ok(Self::quote_expansion_for_quoted_glob(&expanded)) } else { @@ -813,25 +811,40 @@ impl Interpreter { }) .flatten(); + // Protected-span markers, likewise chosen absent from the + // data so a value holding any byte (`\x01` included) stays data. + let mut free_marks = OPERAND_QUOTE_MARK_CANDIDATES.iter().copied().filter(|c| { + Some(*c) != empty_field_sentinel && !sentinel_haystack.contains(*c) + }); + let marks = free_marks.next().zip(free_marks.next()); + let mut expanded_for_split = String::new(); for (value, is_protected, preserves_empty_field) in segments { if is_protected { // Field splitting scans the whole expanded word. Mark literal and // quoted segments as protected so unquoted expansion delimiters can // still create boundaries between adjacent protected segments. - expanded_for_split.push(QUOTED_SEGMENT_START); + if let Some((start, _)) = marks { + expanded_for_split.push(start); + } if preserves_empty_field && let Some(empty_field_sentinel) = empty_field_sentinel { expanded_for_split.push(empty_field_sentinel); } expanded_for_split.push_str(&value); - expanded_for_split.push(QUOTED_SEGMENT_END); + if let Some((_, end)) = marks { + expanded_for_split.push(end); + } } else { expanded_for_split.push_str(&value); } } - let mut fields = self.ifs_split(&expanded_for_split)?; + let mut fields = self.ifs_split_marked( + &expanded_for_split, + self.limits.max_word_split_fields, + marks, + )?; if let Some(empty_field_sentinel) = empty_field_sentinel { for field in &mut fields { field.retain(|ch| ch != empty_field_sentinel); @@ -859,7 +872,7 @@ impl Interpreter { if has_expansion { self.ifs_split(&expanded) } else { - Ok(vec![Self::strip_quote_markers(&expanded)]) + Ok(vec![expanded]) } }) } @@ -967,19 +980,15 @@ impl Interpreter { None } - pub(super) fn strip_quote_markers(s: &str) -> String { - s.chars() - .filter(|&c| c != QUOTED_SEGMENT_START && c != QUOTED_SEGMENT_END) - .collect() - } - - pub(super) fn quote_marker_chars(s: &str) -> Vec<(char, bool)> { + /// `s` as (char, protected) pairs: text between the `marks` pair is + /// protected from splitting; the marks themselves are dropped. + fn quote_marker_chars(s: &str, marks: Option<(char, char)>) -> Vec<(char, bool)> { let mut quoted = false; let mut chars = Vec::new(); for c in s.chars() { - match c { - QUOTED_SEGMENT_START => quoted = true, - QUOTED_SEGMENT_END => quoted = false, + match marks { + Some((start, _)) if c == start => quoted = true, + Some((_, end)) if c == end => quoted = false, _ => chars.push((c, quoted)), } } @@ -999,6 +1008,17 @@ impl Interpreter { /// Split a string on IFS characters, returning an error if resource caps are exceeded. pub(super) fn ifs_split_limited(&self, s: &str, limit: usize) -> Result> { + self.ifs_split_marked(s, limit, None) + } + + /// `ifs_split_limited` where text between the `marks` pair is protected + /// from splitting (the marks are removed). + fn ifs_split_marked( + &self, + s: &str, + limit: usize, + marks: Option<(char, char)>, + ) -> Result> { // Clamp so callers passing a larger value (e.g. remaining array capacity) // cannot bypass the configured max_word_split_fields cap. let limit = limit.min(self.limits.max_word_split_fields); @@ -1014,7 +1034,10 @@ impl Interpreter { .unwrap_or_else(|| " \t\n".to_string()); if ifs.is_empty() { - let field = Self::strip_quote_markers(s); + let field: String = Self::quote_marker_chars(s, marks) + .into_iter() + .map(|(c, _)| c) + .collect(); let bytes = field.len(); return self.push_ifs_field(Vec::new(), field, limit, bytes); } @@ -1023,7 +1046,7 @@ impl Interpreter { let is_ifs_ws = |c: char, quoted: bool| !quoted && ifs.contains(c) && " \t\n".contains(c); let is_ifs_nws = |c: char, quoted: bool| !quoted && ifs.contains(c) && !" \t\n".contains(c); let all_whitespace_ifs = ifs.chars().all(|c| " \t\n".contains(c)); - let chars = Self::quote_marker_chars(s); + let chars = Self::quote_marker_chars(s, marks); if all_whitespace_ifs { // IFS is only whitespace: split on unquoted runs, elide empties. diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 4920d1d79..2230876a4 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -43,11 +43,6 @@ static TIME_REPORT_COUNTER: AtomicU64 = AtomicU64::new(0); // bashkit crate semver so scripts that gate on Bash features keep working. const COMPAT_BASH_VERSION: &str = "5.2.15(1)-release"; const COMPAT_BASH_VERSINFO: [&str; 6] = ["5", "2", "15", "1", "release", "virtual"]; -// Important decision: lexer emits these only for mixed words where an initial -// quoted segment is followed by an unquoted expansion. Keep them internal and -// strip before observable output. -const QUOTED_SEGMENT_START: char = '\x01'; -const QUOTED_SEGMENT_END: char = '\x02'; // Important decision: operand quote sentinels must be selected from a small, // parser-inert set. Exhaustive Unicode probing is attacker-amplifiable CPU work. diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index 922ee77b3..6ee2a62ff 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -21,11 +21,15 @@ pub struct SpannedToken { /// THREAT[TM-DOS-044]: Prevents stack overflow from deeply nested $() patterns. const DEFAULT_MAX_SUBST_DEPTH: usize = 50; -// Important decision: markers preserve quoted segments only until expansion. -// They let later unquoted continuations split without splitting the quoted prefix. -// Marker insertion is one-pass, never repeated String::insert, to avoid parser DoS. -const QUOTED_SEGMENT_START: char = '\x01'; -const QUOTED_SEGMENT_END: char = '\x02'; +// Important decision: a word that starts quoted and continues with an +// unquoted expansion marks its quoted spans with the parser's quote-boundary +// markers (`\u{1e}`/`\u{1f}`), which `parse_word` turns into per-part +// quotedness (`Word::part_quoted`), so later unquoted continuations split +// without splitting the quoted prefix. No in-band marker survives into +// expansion, so data bytes like `\x01` stay data. Marker insertion is +// one-pass, never repeated String::insert, to avoid parser DoS. +const QUOTED_SEGMENT_START: char = '\u{1e}'; +const QUOTED_SEGMENT_END: char = '\u{1f}'; #[derive(Default)] struct ContinuationFlags { @@ -2522,7 +2526,7 @@ mod tests { let mut lexer = Lexer::new(&script); assert_eq!( lexer.next_token(), - Some(Token::Word("\x01a\x02$x".to_string())) + Some(Token::Word("\u{1e}a\u{1f}$x".to_string())) ); assert_eq!(lexer.next_token(), None); } diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-marker-bytes.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-marker-bytes.test.sh index 9c1de46e0..805850e5d 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-marker-bytes.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-marker-bytes.test.sh @@ -4,7 +4,6 @@ ### bashbox_marker_bytes_values_holding_the_bytes_the_expander_uses_internally_surviv # values holding the bytes the expander uses internally survive expansion -### skip: TODO bashbox corpus gap, bashkit output differs from real bash v=$(printf 'a\001b\002c\003d\004e\005f\006g\177h'); show() { printf '%s' "$1" | tr '\001\002\003\004\005\006\177' '1234567'; echo; } show "$v"; echo ${#v} for w in $v; do show "[$w]"; done @@ -30,7 +29,6 @@ same ### bashbox_marker_bytes_a_byte_used_as_ifs_splits_and_an_escaped_one_in_a_pattern_ma # a byte used as IFS splits, and an escaped one in a pattern matches itself -### skip: TODO bashbox corpus gap, bashkit output differs from real bash v=$(printf 'a\001b\002c'); show() { printf '%s' "$1" | tr '\001\002' '12'; echo; } IFS=$(printf '\001'); for w in $v; do show "($w)"; done unset IFS; w=$(printf 'x\001 y'); for q in $w; do show "{$q}"; done; : ${u:=$v}; show "$u"; k=$(printf '\004'); echo "${k:+set}" "${#k}" From a0d6aca5820ad0c1cf665c8d75395f727f1318af Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 04:00:18 +0000 Subject: [PATCH 07/11] fix(interpreter): process substitution runs in a subshell <(list) and >(list) no longer leak variable, cwd, option or $? changes into the parent shell, and exit inside one ends only the substitution. Document the remaining proc-sub gaps (/dev/fd/63 numbering, joining to adjacent word text) as limitations. --- crates/bashkit/src/interpreter/mod.rs | 38 +++++++++++++++++-- .../bash/bashbox-process-substitution.test.sh | 9 ++--- knowledge/operations/limitations.md | 2 +- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 2230876a4..4110526b9 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -6919,16 +6919,37 @@ impl Interpreter { } else { None }; + // `>(cmd)` runs in a subshell too (see expand_process_substitution). + let snapshot = self.snapshot_subshell_state(); + let last_exit_code = self.last_exit_code; + self.bash_subshell += 1; + let mut run = Ok(()); for cmd in &commands { let prev_stdin = self.pipeline_stdin.take(); self.pipeline_stdin = stdin_data.clone(); - let cmd_result = self.execute_command(cmd).await?; + let cmd_result = self.execute_command(cmd).await; self.pipeline_stdin = prev_stdin; + let cmd_result = match cmd_result { + Ok(r) => r, + Err(e) => { + run = Err(e); + break; + } + }; if let Ok(r) = result { r.stdout.append(&cmd_result.stdout); r.stderr.append(&cmd_result.stderr); } + if matches!( + cmd_result.control_flow, + ControlFlow::Exit(_) | ControlFlow::Abort + ) { + break; + } } + self.restore_subshell_state(snapshot); + self.last_exit_code = last_exit_code; + run?; } Ok(()) } @@ -10332,12 +10353,22 @@ impl Interpreter { if is_input { let mut stdout = String::new(); + // The substituted list runs in a subshell: nothing it changes + // (variables, cwd, options, `$?`) reaches the parent. + let snapshot = self.snapshot_subshell_state(); + let last_exit_code = self.last_exit_code; self.bash_subshell += 1; let mut failed = None; for cmd in commands { match self.execute_command(cmd).await { Ok(cmd_result) => { - stdout.push_str(&cmd_result.stdout.command_substitution_text()) + stdout.push_str(&cmd_result.stdout.command_substitution_text()); + if matches!( + cmd_result.control_flow, + ControlFlow::Exit(_) | ControlFlow::Abort + ) { + break; + } } Err(e) => { failed = Some(e); @@ -10345,7 +10376,8 @@ impl Interpreter { } } } - self.bash_subshell -= 1; + self.restore_subshell_state(snapshot); + self.last_exit_code = last_exit_code; if let Some(e) = failed { return Err(e); } diff --git a/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh b/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh index 479602b71..5009d343c 100644 --- a/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/bashbox-process-substitution.test.sh @@ -11,7 +11,7 @@ hi ### bashbox_process_substitution_two_substitutions_are_numbered_down_from_63 # two substitutions are numbered down from 63 -### skip: TODO bashbox corpus gap, bashkit output differs from real bash +### skip: process substitution paths are /dev/fd/proc_sub_, not /dev/fd/63 (limitations.md, Process substitution row: tenant-safe naming on a shared VFS) echo <(true) <(true) >(true); cat <(echo a) <(echo b) ### expect /dev/fd/63 /dev/fd/62 /dev/fd/61 @@ -21,7 +21,7 @@ b ### bashbox_process_substitution_the_number_is_reused_once_the_command_is_done # the number is reused once the command is done -### skip: TODO bashbox corpus gap, bashkit output differs from real bash +### skip: process substitution paths are /dev/fd/proc_sub_, not /dev/fd/63 (limitations.md, Process substitution row: tenant-safe naming on a shared VFS) echo <(true); f(){ echo <(true); }; f ### expect /dev/fd/63 @@ -30,7 +30,7 @@ echo <(true); f(){ echo <(true); }; f ### bashbox_process_substitution_part_of_a_word # part of a word -### skip: TODO bashbox corpus gap, bashkit output differs from real bash +### skip: process substitution is its own word, not joined to adjacent text (x<(true) is two words), and paths are /dev/fd/proc_sub_ (limitations.md, Process substitution row) echo x<(true) ### expect x/dev/fd/63 @@ -69,7 +69,6 @@ a ### bashbox_process_substitution_it_runs_in_a_subshell # it runs in a subshell -### skip: TODO bashbox corpus gap, bashkit output differs from real bash x=1; cat <(x=2; echo $x); echo $x ### expect 2 @@ -78,7 +77,7 @@ x=1; cat <(x=2; echo $x); echo $x ### bashbox_process_substitution_assigned_to_a_variable # assigned to a variable -### skip: TODO bashbox corpus gap, bashkit output differs from real bash +### skip: x=<(true) parses as an assignment then a separate process-substitution word, and paths are /dev/fd/proc_sub_ (limitations.md, Process substitution row) x=<(true); echo $x ### expect /dev/fd/63 diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 5f6e9d0d0..ad5151868 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -107,7 +107,7 @@ Target: IEEE 1003.1-2024 Shell Command Language. | `timeout` | Basic usage | `-k` kill timeout | | `bash`/`sh` | `-c`, `-n`, `-e`, `-x`, `-u`, `-f`, `-o option`, script files, stdin, `--version`, `--help` | Login shell | | Expansion errors | `set -u` unbound variables stop the shell | `$((1/0))` evaluates to 0 and `${x!}` expands empty; bash reports them and drops the rest of the line the failing command ends on (`bashbox_line_number_an_expansion_error_drops_the_rest_of_the_line_it_ends_on`, skipped) | -| Process substitution | `<(cmd)` and `>(cmd)` as filenames, nesting, redirections | The path is `/dev/fd/proc_sub_`, not bash's `/dev/fd/63`. Several interpreters can share one filesystem, so a per-shell descending number would let one tenant's substitution overwrite another's. A script that parses the number sees a different one (`scripts/bash-oracle/cases/process-substitution-dev-fd.sh`) | +| Process substitution | `<(cmd)` and `>(cmd)` as filenames, nesting, redirections | The path is `/dev/fd/proc_sub_`, not bash's `/dev/fd/63`. Several interpreters can share one filesystem, so a per-shell descending number would let one tenant's substitution overwrite another's. A script that parses the number sees a different one (`scripts/bash-oracle/cases/process-substitution-dev-fd.sh`). A substitution is always its own word: bash joins it to adjacent text (`x<(true)` is `x/dev/fd/63`, `x=<(true)` assigns the path), Bashkit splits it off (`bashbox-process-substitution.test.sh`, skipped). The substituted list runs in a subshell, as in bash | | Aliases | `alias`/`unalias`, `shopt -s expand_aliases`, trailing-space chaining, recursion guard | Expanded when the simple command runs, not when its line is read: bash parses a whole line (and a function body at definition) before an `alias`/`unalias` on it takes effect; Bashkit applies them to the very next command | | Named fds / `<>` | `{var}>file`, `{var}file`, `{var}>>file`, `{var}<<&N`/`<&N` allocate the lowest free fd >= 10 on any command (persisting, as in bash); `{var}>&-` closes; `>&N` with N closed is `Bad file descriptor`; `N<>file`, `N<<file` has no shared read/write offset (writes append); `3&1`, `( ... ) 3>&1`) is dropped; a `Bad file descriptor` message ignores redirect order (`2>&1 >&7`) | | Locale | UTF-8 text everywhere; `${#v}` counts bytes when `LC_ALL`/`LC_CTYPE`/`LANG` (first non-empty) is `C` or `POSIX` | With no locale variable set bashkit is UTF-8, where bash falls back to C. Under C, `${s:off:len}`, `${s^^}`, `${s#?}` and glob `?` still work on characters, not bytes: shell words are Rust strings and cannot hold a split UTF-8 sequence (L-STREAM-001) | From 5da29461a10505f5d085b03143cccc1ce3a579c4 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 04:01:08 +0000 Subject: [PATCH 08/11] chore(knowledge): record pattern, glob, quoting and marker-byte decisions --- crates/bashkit/docs/threat-model.md | 2 +- knowledge/foundations/parser.md | 17 +++++++++++++++++ knowledge/foundations/vfs.md | 9 ++++++++- knowledge/log.md | 1 + knowledge/security/threat-model.md | 4 ++-- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index d614b9126..468c65301 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -100,7 +100,7 @@ through configurable limits. | Diff DoS (TM-DOS-028) | `diff` on large unrelated files | LCS matrix cap (10M cells) | MITIGATED | | Parser limit bypass (TM-DOS-030) | eval/source ignore limits | `Parser::with_limits()` | **FIXED** | | Arithmetic overflow (TM-DOS-029) | `$(( 2 ** -1 ))` | Use wrapping arithmetic | **MITIGATED** | -| ExtGlob blowup (TM-DOS-031) | `+(a\|aa)` exponential | Add depth limit | **MITIGATED** | +| ExtGlob blowup (TM-DOS-031) | `+(a\|aa)` exponential | Depth limit plus a shared per-match step budget | **MITIGATED** | | Tokio runtime exhaustion (TM-DOS-032) | Rapid `execute_sync()` calls | Shared runtime | **MITIGATED** | | Brace range OOM (TM-DOS-041) | `{1..999999999}` | Cap range size | **MITIGATED** | | Brace combinatorial (TM-DOS-042) | `{1..100}{1..100}{1..100}` | Cap total expansion | **MITIGATED** | diff --git a/knowledge/foundations/parser.md b/knowledge/foundations/parser.md index 6048a206f..6a9f67cbe 100644 --- a/knowledge/foundations/parser.md +++ b/knowledge/foundations/parser.md @@ -63,6 +63,23 @@ globbing uses the escaped text and drops the escapes when nothing matches; script analysis unescape literal parts; `case` and `[[ == ]]` build patterns with `expand_pattern_word`, which escapes fully quoted words. +**Quoted segments of mixed words.** A word that starts quoted and continues +with an unquoted expansion (`'a b'$x`, `"$y"$x`) also wraps its quoted spans +in `\x1e`/`\x1f`; `parse_word` turns them into `Word::part_quoted`, and field +splitting protects quoted parts using markers it picks from characters absent +from the data. No in-band marker reaches expansion, so a value holding any +byte (`\x01`, `\x02` included) stays data. + +**`$` inside double quotes.** A `$` before `'` or before the closing `"` starts +no expansion and is NUL-escaped, so `"$'q'"` is the text `$'q'` and `"$"'q'` +is `$q`. Here-document bodies keep `$'...'` as written. Function listings +(`type`, `declare -f`) print `$'...'` as the single-quoted text it decodes to +and `$"..."` as `"..."`, as bash does (`normalize_raw_word`). + +**Extglob groups.** A group (`@(...)`, `!(...)`, ...) is read whole; quotes +inside it make their text literal (`@(a|'b)')` has the alternative `b)`), and +`\|` stays a literal bar. An unbalanced group is plain text. + **Metacharacters vs reserved words.** Only space, tab, newline, `|`, `&`, `;`, `(`, `)`, `<` and `>` delimit a word. `{` and `}` do not: they are reserved words, recognized as such only when they stand alone. So `echo a}b` prints diff --git a/knowledge/foundations/vfs.md b/knowledge/foundations/vfs.md index bbb4aa22e..fa10491e0 100644 --- a/knowledge/foundations/vfs.md +++ b/knowledge/foundations/vfs.md @@ -329,7 +329,14 @@ Rules that fall out of the per-component walk: from the caller's spelling, so `./` and `../` prefixes survive expansion. - No match anywhere in the walk falls back to the literal pattern, or to nothing under `nullglob`, same as a trailing-component miss. -- `**` with `globstar` is handled separately by `expand_glob_recursive`. +- A trailing `/` keeps only directories and stays on every match (`d/*/` gives + `d/e/`). +- `**` with `globstar` is a component of the same walk: it matches zero or more + directories (files too when last), visible ones unless `dotglob`. As the last + component its zero-directory match is `d/` after a literal prefix (`d/**`), + plain `d` after a glob (`*/**`), and nothing for a bare `**`; duplicates from + several `**` collapse. The recursive walk is iterative, depth-capped by + `max_path_depth` and stops at the candidate cap. - THREAT[TM-DOS-095]: the candidate set multiplies per component, so patterns deeper than `FsLimits::max_path_depth` are rejected and the live candidate set is capped at `FsLimits::max_file_count`. diff --git a/knowledge/log.md b/knowledge/log.md index c46d9c905..0c37925c6 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -101,6 +101,7 @@ * **Fix**: Named file descriptors follow bash. `{var}>file` (and `<`, `>>`, `<>`, `<<<`, `<<`, `>&N`, `<&N`) allocates the lowest free fd >= 10 into `var` (local, array element; readonly fails) on any command, simple or compound, and the fd stays open as bash never undoes `{var}` redirections; `{var}>&-` closes it and a closed fd is reused. `>&N`/`<&N` to a closed fd is `Bad file descriptor` (status 1), a dup to a word is `ambiguous redirect`. The lexer accepts multi-digit fds (`10>x`), `<>` and `N<<f` in a loop is capped by `max_file_descriptors` (TM-DOS-063 tests updated: a prior `$v` value is ignored). Gaps in [Limitations](operations/limitations.md) ("Named fds / `<>`"). * **Fix**: xargs runs its command with stdin at EOF (`printf '' | xargs wc -l` prints 0); an empty pipe now reaches compound commands and functions as stdin at EOF; `printf %q` leaves printable non-ASCII unescaped under UTF-8. * **Fix**: Interpreter-core parity from the BashBox corpus (bash-verified cases unskipped). `break`/`continue` never leave a caller's loop and warn outside one; `return` outside a function fails with 2, `return` with no argument returns `$?`, and `return` ends a sourced file; `${x!}`/`${}` report "bad substitution" and `${1:=x}` "cannot assign in this way", abandoning the line; ERR/DEBUG traps keep `$?`; `trap` canonicalizes specs and lists in bash order with re-readable quoting (as do `alias` and a bare `set`); `getopts` always writes `OPTIND` (initialized to 1); `unset` parses options only before operands and refuses readonly elements; `a[-5]=x` past the start is a bad subscript; `[[ -eq ]]` operands are arithmetic; an invalid `=~` regex returns 2; `set -k`; `read -d ''`; `dirs`/`pushd` abbreviate `$HOME` as `~`; `hash`, `builtin`, `source` error statuses. `<>` and ENOTDIR under a regular file stay open. See [Builtins](foundations/builtins.md#control-flow-and-shell-state-builtins). +* **Fix**: Pattern, expansion and quoting parity from the BashBox corpus (24 bash-verified cases unskipped). `[[ == ]]` always matches with extglob and `shopt -s nocasematch` folds case for `case`, `[[ ==/=~ ]]` and `${v/pat/rep}` (not removal); `!(...)` may match an empty prefix, quotes and `\|` inside an extglob group stay literal, an unbalanced group is plain text, and extglob backtracking spends a shared step budget (TM-DOS-031). `${s:o:-n}` counts back from the end, out-of-range offsets select nothing, negative array/positional lengths are `substring expression < 0`, `${#@}` counts parameters. `shopt` reports a bad name and still applies the rest. `globstar` `**` is a per-component match with bash's output spelling, and `d/*/` keeps only directories with the slash. `"$'q'"` and here-documents keep `$'...'` literal; function listings print `$'...'` single-quoted. Values holding `\x01`/`\x02` survive expansion (mixed-word quote markers moved to `part_quoted`). `<(...)`/`>(...)` run in a subshell. Remaining gaps recorded in [Limitations](operations/limitations.md): execution-time aliases, `/dev/fd/proc_sub_` naming and proc-sub word joining. ## 2026-10-06 diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 08c9ed399..5caf38fc6 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -261,7 +261,7 @@ runaway scripts without permanently breaking the session. | TM-DOS-029 | Arithmetic overflow/panic | `$(( 2 ** -1 ))`, `$(( 1 << 64 ))`, `i64::MIN / -1` | `wrapping_*` / saturating ops; `wrapping_neg` for `i64::MIN / -1` and unary negate; `<<`/`>>` clamp shift amount | **MITIGATED** | | TM-DOS-043 | Arithmetic side-effect overflow/panic | `((x+=1))` or `$((x++))` at the `i64` boundaries | Compound assignment and prefix/postfix increment/decrement use wrapping arithmetic in both evaluator paths | **MITIGATED** | | TM-DOS-030 | Parser limit bypass via eval/source/trap | `eval`, `source`, trap handlers now use `Parser::with_limits()` |, | **FIXED** (2026-03 audit verified) | -| TM-DOS-031 | Glob/ExtGlob exponential blowup | `+(a\|aa)` against long string causes O(n!) recursion in `glob_match_impl`; a run of plain `*` (consecutive `****…` or separated `*a*b*…`) previously recursed per value position and blew up (a 33-`*` pattern timed out `glob_fuzz`) | Plain `*` now matches via a single backtracking restore point (the classic linear wildcard algorithm), worst case O(value·pattern), no per-position recursion; `glob_match_impl` still carries a recursion-depth cap for extglob nesting and bails on excessive depth; aliases that expand to huge brace ranges go through the same parser-budget check (`interpreter/mod.rs:4216`) | **MITIGATED** | +| TM-DOS-031 | Glob/ExtGlob exponential blowup | `+(a\|aa)` against long string causes O(n!) recursion in `glob_match_impl`; a run of plain `*` (consecutive `****…` or separated `*a*b*…`) previously recursed per value position and blew up (a 33-`*` pattern timed out `glob_fuzz`) | Plain `*` now matches via a single backtracking restore point (the classic linear wildcard algorithm), worst case O(value·pattern), no per-position recursion; `glob_match_impl` still carries a recursion-depth cap for extglob nesting and bails on excessive depth, and every recursive glob/extglob step spends from a `MAX_GLOB_STEPS` (100,000) budget shared by one match or one removal/substitution scan, failing the match once spent. `[[ == ]]` understands extglob without `shopt` (bash 5.2), so this budget is what bounds it (`test_cond_extglob_backtracking_is_bounded`); aliases that expand to huge brace ranges go through the same parser-budget check (`interpreter/mod.rs:4216`) | **MITIGATED** | | TM-DOS-035 | DEBUG trap recursive amplification | `trap 'a=1;b=2;...' DEBUG` amplifies N commands to N*M | Suppress DEBUG trap inside trap handlers (`in_trap` guard) | **FIXED** | | TM-DOS-032 | Tokio runtime exhaustion (Python) | Rapid `execute_sync()` calls each create new tokio runtime, exhausting OS threads | `PyBash` and `BashTool` create one `Arc` per instance in `__new__` via `make_runtime()` and reuse it for every `execute_sync` call; no per-call runtime construction | **MITIGATED** | | TM-DOS-033 | AWK unbounded loops | `BEGIN { while(1){} }` or deep user-function recursion | Every awk loop form (`while`, `do`, `for`, `for-in`) honours `ExecutionLimits::max_loop_iterations`; user-function recursion is capped at `AWK_MAX_CALL_DEPTH`; both abort the program with a fatal error (TM-DOS-109); 30s timeout backstop | **MITIGATED** | @@ -318,7 +318,7 @@ panicked. Resolved with `wrapping_*` ops, masked shift amounts, clamped exponent `execute_script_body(.., run_exit_trap=false)` (like `source`) so they do not fire the EXIT trap , previously `trap 'eval :' EXIT` could recurse one command per level until the budget aborted. -**TM-DOS-031** (mitigated): see table, linear backtracking for plain `*`, plus a recursion-depth cap for extglob nesting in `glob_match_impl` +**TM-DOS-031** (mitigated): see table, linear backtracking for plain `*`, plus a recursion-depth cap for extglob nesting in `glob_match_impl` and a shared step budget for extglob backtracking (`interpreter/glob.rs`). **Implementation**: `timeout` 30s + `parser_timeout` 5s + `max_parser_operations` 100K in `limits.rs` (TM-DOS-023/024); `AWK_MAX_PARSER_DEPTH` 100 (`builtins/awk/parser.rs`) and `MAX_JQ_JSON_DEPTH` 100 (`builtins/jq/`) for TM-DOS-027; `MAX_LCS_CELLS` 10M (`builtins/diff.rs`) for TM-DOS-028. From 8d10918c4e0e2aab3aa61cea51d4e18b4f83a713 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 04:03:52 +0000 Subject: [PATCH 09/11] chore(glob): allow match_extglob's argument count --- crates/bashkit/src/interpreter/glob.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bashkit/src/interpreter/glob.rs b/crates/bashkit/src/interpreter/glob.rs index 619b58ff3..01c0a19ab 100644 --- a/crates/bashkit/src/interpreter/glob.rs +++ b/crates/bashkit/src/interpreter/glob.rs @@ -484,6 +484,7 @@ impl Interpreter { /// op: b'@', b'?', b'*', b'+', b'!' /// alts: the | separated alternatives /// rest: pattern after the closing ) + #[allow(clippy::too_many_arguments)] fn match_extglob( &self, op: u8, From f247110aeb8edd5d682cf1ba5b31d11a53841503 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 04:09:30 +0000 Subject: [PATCH 10/11] test: negative array-slice length is bash's substring error --- crates/bashkit/tests/integration/byte_range_panic_tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/bashkit/tests/integration/byte_range_panic_tests.rs b/crates/bashkit/tests/integration/byte_range_panic_tests.rs index b87dd62eb..e6e86c4c2 100644 --- a/crates/bashkit/tests/integration/byte_range_panic_tests.rs +++ b/crates/bashkit/tests/integration/byte_range_panic_tests.rs @@ -65,7 +65,9 @@ async fn array_slice_negative_length_no_panic() { .exec("arr=(a b c d e); echo \"${arr[@]:1:-1}\"") .await .expect("negative slice length must not panic"); - assert_eq!(result.exit_code, 0); + // bash rejects a negative array-slice length and aborts the line. + assert_eq!(result.exit_code, 1); + assert!(result.stderr.contains("-1: substring expression < 0")); } /// Regression for #1414: ensure `start + len_val` near `usize::MAX` does From e40759f42e604dcd80f74c54a645e13ca5872155 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 04:22:58 +0000 Subject: [PATCH 11/11] fix(interpreter): box subshell snapshots in process substitution paths --- crates/bashkit/src/interpreter/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 4110526b9..48b591e8c 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -6920,7 +6920,9 @@ impl Interpreter { None }; // `>(cmd)` runs in a subshell too (see expand_process_substitution). - let snapshot = self.snapshot_subshell_state(); + // Boxed: keeps this future (in every simple command's await chain) + // small so deep `$(...)` nesting stays within the stack. + let snapshot = Box::new(self.snapshot_subshell_state()); let last_exit_code = self.last_exit_code; self.bash_subshell += 1; let mut run = Ok(()); @@ -6947,7 +6949,7 @@ impl Interpreter { break; } } - self.restore_subshell_state(snapshot); + self.restore_subshell_state(*snapshot); self.last_exit_code = last_exit_code; run?; } @@ -10355,7 +10357,7 @@ impl Interpreter { let mut stdout = String::new(); // The substituted list runs in a subshell: nothing it changes // (variables, cwd, options, `$?`) reaches the parent. - let snapshot = self.snapshot_subshell_state(); + let snapshot = Box::new(self.snapshot_subshell_state()); let last_exit_code = self.last_exit_code; self.bash_subshell += 1; let mut failed = None; @@ -10376,7 +10378,7 @@ impl Interpreter { } } } - self.restore_subshell_state(snapshot); + self.restore_subshell_state(*snapshot); self.last_exit_code = last_exit_code; if let Some(e) = failed { return Err(e);