From d517854544f06239ff64c52a5c5a66017d7dd728 Mon Sep 17 00:00:00 2001 From: xmakro Date: Thu, 8 Oct 2026 23:07:56 -0700 Subject: [PATCH 1/4] fix(parser): read a backquoted substitution after a quoted segment `'b='`cmd`` and `"b="`cmd`` lexed the backquote as a literal word character, so the substitution swallowed the script up to the next backquote: `echo 'b='`echo X`` failed with "unterminated backtick substitution". A quoted segment followed by `$(cmd)` already worked; the backquoted form is now read the same way. --- crates/bashkit/src/parser/lexer.rs | 8 +++++++ .../spec_cases/bash/command-subst.test.sh | 21 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index d8779f2fe..8869bb567 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -1696,6 +1696,14 @@ impl<'a> Lexer<'a> { } } } + Some('`') => { + // `"a"`cmd``: the backquoted form of `"a"$(cmd)`. + flags.has_unquoted_expansion = true; + if let Err(e) = self.read_backtick_into(content) { + flags.error = Some(e); + break; + } + } Some('\\') => { // Backslash escape after a quoted segment: `'a'\''b'`. self.advance(); diff --git a/crates/bashkit/tests/spec_cases/bash/command-subst.test.sh b/crates/bashkit/tests/spec_cases/bash/command-subst.test.sh index 6dfa927db..10c5aabc0 100644 --- a/crates/bashkit/tests/spec_cases/bash/command-subst.test.sh +++ b/crates/bashkit/tests/spec_cases/bash/command-subst.test.sh @@ -317,3 +317,24 @@ echo "$#" ### expect 3 ### end + +### subst_backtick_after_quoted_prefix +# Backquotes glued to a quoted segment, like "a"$(cmd) +echo 'b='`echo X` +echo "b="`echo X` +for u in 'a' 'b='`echo 42`; do echo $u; done +echo 'a'`echo X`'c' +### expect +b=X +b=X +a +b=42 +aXc +### end + +### subst_backtick_after_quoted_prefix_splits +# The unquoted result still word-splits +printf '[%s]' 'x'`echo "1 2"`; echo +### expect +[x1][2] +### end From 9a484a319187a0126508df0164ca8bb92686174a Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sat, 10 Oct 2026 18:06:46 -0500 Subject: [PATCH 2/4] fix(parser): keep quoted glob sentinel pairs intact --- crates/bashkit/src/parser/lexer.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index f634628d8..21887a462 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -2522,10 +2522,10 @@ impl<'a> Lexer<'a> { while i < n { let ch = char_vec[i]; - // Sentinel-escaped marker bytes are literal data, not boundaries. + // Sentinel pairs are indivisible; escaped NUL/marker bytes stay data. if ch == '\x00' && let Some(&next) = char_vec.get(i + 1) - && matches!(next, QUOTED_SEGMENT_START | QUOTED_SEGMENT_END) + && matches!(next, '\x00' | QUOTED_SEGMENT_START | QUOTED_SEGMENT_END) { result.push(ch); result.push(next); @@ -3162,6 +3162,18 @@ impl<'a> Lexer<'a> { mod tests { use super::*; + #[test] + fn quoted_glob_markers_do_not_split_sentinel_pairs() { + assert_eq!( + Lexer::escape_glob_metas_in_quoted_ranges("*\x00\x00*", &[(0, 3)]), + "\x1e\\*\x00\x00\x1f*" + ); + assert_eq!( + Lexer::escape_glob_metas_in_quoted_ranges("*\x00\x1fx", &[(0, 3)]), + "\x1e\\*\x00\x1f\x1fx" + ); + } + #[test] fn test_simple_words() { let mut lexer = Lexer::new("echo hello world"); From 15e308bdaa87810cb16786c0e57bcc2f09115c46 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sat, 10 Oct 2026 18:09:35 -0500 Subject: [PATCH 3/4] fix(parser): distinguish literal dollars from escaped NUL pairs --- crates/bashkit/src/parser/lexer.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index 21887a462..6a29bd1bc 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -2525,7 +2525,10 @@ impl<'a> Lexer<'a> { // Sentinel pairs are indivisible; escaped NUL/marker bytes stay data. if ch == '\x00' && let Some(&next) = char_vec.get(i + 1) - && matches!(next, '\x00' | QUOTED_SEGMENT_START | QUOTED_SEGMENT_END) + && matches!( + next, + '\x00' | '$' | QUOTED_SEGMENT_START | QUOTED_SEGMENT_END + ) { result.push(ch); result.push(next); @@ -2541,8 +2544,7 @@ impl<'a> Lexer<'a> { } let in_quoted = marked_quoted; if in_quoted { - if expansion_stack.is_empty() && ch == '$' && (i == 0 || char_vec[i - 1] != '\x00') - { + if expansion_stack.is_empty() && ch == '$' { // Peek at next char to detect ${ or $( if let Some(&next) = char_vec.get(i + 1) && (next == '{' || next == '(') @@ -2557,7 +2559,6 @@ impl<'a> Lexer<'a> { // name is not a glob character. if let Some(&next) = char_vec.get(i + 1) && matches!(next, '*' | '@' | '?' | '!' | '-') - && (i == 0 || char_vec[i - 1] != '\x00') { result.push(ch); result.push(next); @@ -3172,6 +3173,11 @@ mod tests { Lexer::escape_glob_metas_in_quoted_ranges("*\x00\x1fx", &[(0, 3)]), "\x1e\\*\x00\x1f\x1fx" ); + let quoted = "\x00\x00$(printf '*')"; + assert_eq!( + Lexer::escape_glob_metas_in_quoted_ranges(&format!("{quoted}*"), &[(0, quoted.len())]), + format!("\x1e{quoted}\x1f*") + ); } #[test] From bb690fc970780661c3bf84ab7f09b86d7c7bb929 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Sat, 10 Oct 2026 18:12:05 -0500 Subject: [PATCH 4/4] fix(interpreter): avoid repeated quote metadata scans --- crates/bashkit/src/interpreter/expansion.rs | 97 +++++++++++++++------ knowledge/foundations/parser.md | 2 + 2 files changed, 74 insertions(+), 25 deletions(-) diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 77c24c3cf..3e1a24c29 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -294,8 +294,8 @@ impl Interpreter { /// How many bytes `append_expansion_for_word` will append for `value`, /// counted before anything is allocated so the budget is charged first. - fn expansion_appended_len(word: &Word, idx: usize, value: &str) -> usize { - if word.has_unquoted_glob && Self::part_is_quoted(word, idx) { + fn expansion_appended_len(word: &Word, part_quoted: bool, value: &str) -> usize { + if word.has_unquoted_glob && part_quoted { value .chars() .map(|ch| ch.len_utf8() + usize::from(Self::needs_glob_escape(ch))) @@ -308,10 +308,10 @@ impl Interpreter { pub(super) fn append_expansion_for_word( result: &mut String, word: &Word, - idx: usize, + part_quoted: bool, value: &str, ) { - if word.has_unquoted_glob && Self::part_is_quoted(word, idx) { + if word.has_unquoted_glob && part_quoted { result.push_str(&Self::quote_expansion_for_quoted_glob(value)); } else { result.push_str(value); @@ -351,7 +351,15 @@ impl Interpreter { // been consumed; sibling substitutions otherwise evade live-byte caps. let mut substitution_leases = Vec::new(); + // Check the metadata once: many unquoted expansions before a quoted + // suffix must not rescan the whole word for every output append. + let has_part_quotes = word.part_quoted.iter().any(|q| *q); for (idx, part) in word.parts.iter().enumerate() { + let part_quoted = if has_part_quotes { + word.part_quoted.get(idx).copied().unwrap_or(false) + } else { + word.quoted + }; match part { WordPart::BadSubstitution(text) => { return Err(crate::error::Error::LineAbort( @@ -374,7 +382,7 @@ impl Interpreter { WordPart::Literal(s) => { // Tilde expansion (`~`, `~/x`, `x=~:~`); quoted literals // keep their tilde. - match (!Self::part_is_quoted(word, idx) && s.contains('~')) + match (!part_quoted && s.contains('~')) .then(|| self.tilde_expand_word_literal(word, idx, s)) .flatten() { @@ -386,7 +394,7 @@ impl Interpreter { if self.is_nounset() && !self.is_variable_set(name) { self.nounset_error = Some(self.unbound_variable_diag(name)); } - if name == "*" && Self::part_is_quoted(word, idx) { + if name == "*" && part_quoted { let positional = self .call_stack .last() @@ -403,14 +411,14 @@ impl Interpreter { Self::append_expansion_for_word( &mut result, word, - idx, + part_quoted, &positional.join(&sep), ); } else { Self::append_expansion_for_word( &mut result, word, - idx, + part_quoted, &self.expand_variable(name), ); } @@ -425,9 +433,9 @@ impl Interpreter { // THREAT[TM-DOS-089]: Delegate to Box::pin-ed helper to // prevent stack growth proportional to nesting depth. let trimmed = self.execute_cmd_subst(commands).await?; - let appended_bytes = Self::expansion_appended_len(word, idx, &trimmed); + let appended_bytes = Self::expansion_appended_len(word, part_quoted, &trimmed); substitution_leases.push(self.execution_budget.lease_bytes(appended_bytes)?); - Self::append_expansion_for_word(&mut result, word, idx, &trimmed); + Self::append_expansion_for_word(&mut result, word, part_quoted, &trimmed); } WordPart::ArithmeticExpansion(expr) => { let expanded_expr = if expr.contains("$(") || expr.contains('`') { @@ -439,7 +447,12 @@ impl Interpreter { let value = self .try_evaluate_arithmetic_with_assign(&expanded_expr) .map_err(|msg| crate::error::Error::LineAbort(self.arith_diag("", &msg)))?; - Self::append_expansion_for_word(&mut result, word, idx, &value.to_string()); + Self::append_expansion_for_word( + &mut result, + word, + part_quoted, + &value.to_string(), + ); } WordPart::Length(name) => { let value = if let Some(bracket_pos) = name.find('[') { @@ -523,7 +536,7 @@ impl Interpreter { // Delegate to sync helper to avoid bloating the async state // machine with Vec locals (causes stack overflow at // depth 32 in debug builds — see stack_overflow_regression_tests). - self.operand_outer_unquoted = !Self::part_is_quoted(word, idx); + self.operand_outer_unquoted = !part_quoted; let colon_variant = self.star_colon_variant( name, &value, @@ -539,7 +552,7 @@ impl Interpreter { is_set, ); self.operand_outer_unquoted = false; - Self::append_expansion_for_word(&mut result, word, idx, &expanded); + Self::append_expansion_for_word(&mut result, word, part_quoted, &expanded); } WordPart::ArrayAccess { name, index } => { // `${a[$(cmd)]}` in the source: the substitution runs @@ -558,7 +571,7 @@ impl Interpreter { } else { self.expand_array_access_part(name, index) }; - Self::append_expansion_for_word(&mut result, word, idx, &value); + Self::append_expansion_for_word(&mut result, word, part_quoted, &value); } WordPart::ArrayIndices { name, star } => { let keys = self.array_keys(name); @@ -567,7 +580,12 @@ impl Interpreter { } else { " ".to_string() }; - Self::append_expansion_for_word(&mut result, word, idx, &keys.join(&sep)); + Self::append_expansion_for_word( + &mut result, + word, + part_quoted, + &keys.join(&sep), + ); } WordPart::Substring { name, @@ -578,7 +596,7 @@ impl Interpreter { let value = self .substring_part(name, offset, length.as_deref()) .map_err(crate::error::Error::LineAbort)?; - Self::append_expansion_for_word(&mut result, word, idx, &value); + Self::append_expansion_for_word(&mut result, word, part_quoted, &value); } WordPart::IndirectExpansion { name, @@ -593,7 +611,7 @@ impl Interpreter { // Nameref without operator: ${!ref} is the name // at the end of the nameref chain. let target = self.resolve_nameref(name).to_string(); - Self::append_expansion_for_word(&mut result, word, idx, &target); + Self::append_expansion_for_word(&mut result, word, part_quoted, &target); } else { // Resolve the indirect target variable name let resolved_name = if let Some(target) = nameref_target { @@ -614,16 +632,31 @@ impl Interpreter { *colon_variant, is_set, ); - Self::append_expansion_for_word(&mut result, word, idx, &expanded); + Self::append_expansion_for_word( + &mut result, + word, + part_quoted, + &expanded, + ); } else { // Plain indirect expansion (no operator) if let Some(arr) = self.scoped.arrays.get(&resolved_name) { if let Some(first) = arr.get(&0) { - Self::append_expansion_for_word(&mut result, word, idx, first); + Self::append_expansion_for_word( + &mut result, + word, + part_quoted, + first, + ); } } else { let value = self.expand_variable(&resolved_name); - Self::append_expansion_for_word(&mut result, word, idx, &value); + Self::append_expansion_for_word( + &mut result, + word, + part_quoted, + &value, + ); } } } @@ -641,7 +674,12 @@ impl Interpreter { } else { " ".to_string() }; - Self::append_expansion_for_word(&mut result, word, idx, &names.join(&sep)); + Self::append_expansion_for_word( + &mut result, + word, + part_quoted, + &names.join(&sep), + ); } WordPart::ArrayLength(name) => { let resolved = self.resolve_nameref(name); @@ -658,14 +696,14 @@ impl Interpreter { let expanded = self .expand_process_substitution(commands, *is_input) .await?; - Self::append_expansion_for_word(&mut result, word, idx, &expanded); + Self::append_expansion_for_word(&mut result, word, part_quoted, &expanded); } WordPart::Transformation { name, operator } => { let value = match prompts.as_mut().and_then(Vec::pop) { Some(decoded) if *operator == 'P' => decoded, _ => self.transformation_part(name, *operator), }; - Self::append_expansion_for_word(&mut result, word, idx, &value); + Self::append_expansion_for_word(&mut result, word, part_quoted, &value); } } } @@ -3679,9 +3717,18 @@ mod expansion_charge_tests { for idx in 0..2 { for value in values { let mut appended = String::new(); - Interpreter::append_expansion_for_word(&mut appended, &w, idx, value); + Interpreter::append_expansion_for_word( + &mut appended, + &w, + Interpreter::part_is_quoted(&w, idx), + value, + ); assert_eq!( - Interpreter::expansion_appended_len(&w, idx, value), + Interpreter::expansion_appended_len( + &w, + Interpreter::part_is_quoted(&w, idx), + value + ), appended.len(), "charge != appended for {value:?} \ (quoted={quoted}, has_unquoted_glob={has_unquoted_glob})" diff --git a/knowledge/foundations/parser.md b/knowledge/foundations/parser.md index 298ac89d7..8ea0b67ad 100644 --- a/knowledge/foundations/parser.md +++ b/knowledge/foundations/parser.md @@ -67,6 +67,8 @@ when quoted text contains glob syntax or a quoted expansion could produce it: `"$prefix"$empty` and `'literal*'$(printf '')` keep the quoted glob literal. Expansion escaping and its byte-budget charge use each part's quote flag; unquoted expansion output still participates in glob and pattern matching. +The quote-metadata presence check is cached once per expanded word, avoiding +quadratic scans when many unquoted expansions precede a quoted suffix. The shared marker builder preserves empty quoted spans and ends adjacent variable names; sentinel-escaped marker bytes remain literal data.