From 925f088dc036c4b11cb7643e24214978e5bc7201 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 13:05:58 +0000 Subject: [PATCH] fix(lexer): keep $'\x1e' and $'\x1f' bytes as data The decoded bytes collided with the lexer's quote-boundary markers and were dropped, breaking SEP=$'\x1f' composite keys. Claude-Session: https://claude.ai/code/session_019aFikmptPc91Fj4N2iDXQA --- crates/bashkit/src/parser/lexer.rs | 37 +++++++++---------- .../bash/ansi-c-control-bytes.test.sh | 34 +++++++++++++++++ knowledge/log.md | 1 + 3 files changed, 53 insertions(+), 19 deletions(-) create mode 100644 crates/bashkit/tests/spec_cases/bash/ansi-c-control-bytes.test.sh diff --git a/crates/bashkit/src/parser/lexer.rs b/crates/bashkit/src/parser/lexer.rs index 24508f1ea..5053f7cdd 100644 --- a/crates/bashkit/src/parser/lexer.rs +++ b/crates/bashkit/src/parser/lexer.rs @@ -1074,12 +1074,9 @@ impl<'a> Lexer<'a> { // Single-quoted strings are literal - no variable expansion. Quote-boundary // markers are only for parsed mixed words; LiteralWord keeps contents raw. // Also decode any NUL escape sentinels from continued double-quoted segments. - content.retain(|ch| ch != '\u{1e}' && ch != '\u{1f}'); - Some(Token::LiteralWord(if content.contains('\x00') { - Self::decode_nul_escape_sentinel(&content) - } else { - content - })) + Some(Token::LiteralWord(Self::strip_markers_decode_sentinels( + &content, + ))) } /// After a closing quote, read any adjacent quoted or unquoted word chars @@ -1496,30 +1493,32 @@ impl<'a> Lexer<'a> { /// Append a literal segment while protecting sentinel-sensitive bytes from parse_word expansion. fn push_literal_with_escaped_dollar(dst: &mut String, segment: &str) { for ch in segment.chars() { - if matches!(ch, '\x00' | '$') { + // `\x1e`/`\x1f` are this lexer's quote-boundary markers; a decoded + // `$'\x1f'` byte must stay data (`SEP=$'\x1f'` is a common idiom). + if matches!(ch, '\x00' | '$' | '\u{1e}' | '\u{1f}') { dst.push('\x00'); } dst.push(ch); } } - /// Decode NUL-based escape sentinels: each `\x00` followed by a char is - /// collapsed to that char. Used for literal-token paths that bypass `parse_word()`. - fn decode_nul_escape_sentinel(segment: &str) -> String { - let mut decoded = String::with_capacity(segment.len()); + /// Drop quote-boundary markers and decode NUL escape sentinels in one + /// pass, so a NUL-escaped marker byte survives as data. + fn strip_markers_decode_sentinels(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); let mut chars = segment.chars(); - while let Some(ch) = chars.next() { - if ch == '\x00' { - if let Some(literal_ch) = chars.next() { - decoded.push(literal_ch); + match ch { + '\x00' => { + if let Some(next) = chars.next() { + out.push(next); + } } - } else { - decoded.push(ch); + '\u{1e}' | '\u{1f}' => {} + c => out.push(c), } } - - decoded + out } fn read_double_quoted_string(&mut self) -> Option { diff --git a/crates/bashkit/tests/spec_cases/bash/ansi-c-control-bytes.test.sh b/crates/bashkit/tests/spec_cases/bash/ansi-c-control-bytes.test.sh new file mode 100644 index 000000000..a44707bab --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/ansi-c-control-bytes.test.sh @@ -0,0 +1,34 @@ +# $'\x1e' and $'\x1f' (ASCII record/unit separators) are data. Scripts use +# SEP=$'\x1f' to build composite associative-array keys. + +### ansi_c_unit_separator_kept +printf '%s' $'a\x1fb\x1ec' | od -An -c | tr -s ' ' +### expect + a 037 b 036 c +### end + +### ansi_c_separator_in_concatenation +k="a"$'\x1f'"b"; printf '%s' "$k" | od -An -c | tr -s ' ' +SEP=$'\x1f'; m="x${SEP}y"; echo "${#m}" +### expect + a 037 b +3 +### end + +### ansi_c_separator_composite_assoc_key +declare -A v +section=server; key=host +v["${section}"$'\x1f'"${key}"]=localhost +mk="server"$'\x1f'"host" +echo "${v[$mk]}" "${#v[@]}" +for k in "${!v[@]}"; do printf '%s' "$k" | od -An -c | tr -s ' '; done +### expect +localhost 1 + s e r v e r 037 h o s t +### end + +### ansi_c_separator_after_single_quote +x='a'$'\x1f''b'; echo "${#x}" +### expect +3 +### end diff --git a/knowledge/log.md b/knowledge/log.md index ad2951726..56693d240 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Fix**: `$'\x1e'` and `$'\x1f'` bytes (record/unit separators) are kept as data. They collided with the lexer's quote-boundary markers and were silently dropped, which broke the common `SEP=$'\x1f'` composite-key idiom for associative arrays (`v["$section"$'\x1f'"$key"]`). Decoded ANSI-C content now NUL-escapes those bytes like `$`. Found by gap telemetry (`config_ini_merge` replay printed nothing). * **Fix**: A usage error in a bundled builtin no longer aborts the script. `grep -E '('`, `grep -P` with a bad pattern, `rg` with a bad regex, an awk syntax error, `grep` with no pattern and `grep -f missing` now fail that one command with exit 2 and the message on stderr, and the script continues (bash semantics; `set -e` still stops). `Err` from custom builtins (`BashBuilder::builtin`, registry, resolver) keeps aborting, per the public contract. See foundations/builtins "Errors: Failed Command vs Aborted Execution". * **Feature**: `gawk`, `mawk` and `nawk` run the awk builtin (same engine, including gawk extensions it already had such as `gensub`, `toupper`). Gap telemetry showed agents probing `gawk` by name. `--version` still identifies as bashkit awk. * **Fix**: `tar -P`/`--absolute-names` keeps the leading `/` (no notice), `-T FILE`/`--files-from` reads member names (blank lines skipped, `-` is stdin), and `--` ends options. `ls` implements the quoting styles: `-b`/`--escape`, `-N`, `-Q`, `--quoting-style=literal|escape|c|shell|shell-always|shell-escape|shell-escape-always` (GNU rules, including `"it's"` and headers left unescaped under `-b`); `locale`/`clocale` still report not implemented. Found by gap telemetry.