fix(lexer): keep $'\x1e' and $'\x1f' bytes as data

crates/bashkit/src/parser/lexer.rs

@@ -1074,12 +1074,9 @@ impl<'a> Lexer<'a> {
         // Single-quoted strings are literal - no variable expansion. Quote-boundary
         // markers are only for parsed mixed words; LiteralWord keeps contents raw.
         // Also decode any NUL escape sentinels from continued double-quoted segments.
-        content.retain(|ch| ch != '\u{1e}' && ch != '\u{1f}');
-        Some(Token::LiteralWord(if content.contains('\x00') {
-            Self::decode_nul_escape_sentinel(&content)
-        } else {
-            content
-        }))
+        Some(Token::LiteralWord(Self::strip_markers_decode_sentinels(
+            &content,
+        )))
     }
 
     /// After a closing quote, read any adjacent quoted or unquoted word chars
@@ -1496,30 +1493,32 @@ impl<'a> Lexer<'a> {
     /// Append a literal segment while protecting sentinel-sensitive bytes from parse_word expansion.
     fn push_literal_with_escaped_dollar(dst: &mut String, segment: &str) {
         for ch in segment.chars() {
-            if matches!(ch, '\x00' | '$') {
+            // `\x1e`/`\x1f` are this lexer's quote-boundary markers; a decoded
+            // `$'\x1f'` byte must stay data (`SEP=$'\x1f'` is a common idiom).
+            if matches!(ch, '\x00' | '$' | '\u{1e}' | '\u{1f}') {
                 dst.push('\x00');
             }
             dst.push(ch);
         }
     }
 
-    /// Decode NUL-based escape sentinels: each `\x00` followed by a char is
-    /// collapsed to that char. Used for literal-token paths that bypass `parse_word()`.
-    fn decode_nul_escape_sentinel(segment: &str) -> String {
-        let mut decoded = String::with_capacity(segment.len());
+    /// Drop quote-boundary markers and decode NUL escape sentinels in one
+    /// pass, so a NUL-escaped marker byte survives as data.
+    fn strip_markers_decode_sentinels(segment: &str) -> String {
+        let mut out = String::with_capacity(segment.len());
         let mut chars = segment.chars();
-
         while let Some(ch) = chars.next() {
-            if ch == '\x00' {
-                if let Some(literal_ch) = chars.next() {
-                    decoded.push(literal_ch);
+            match ch {
+                '\x00' => {
+                    if let Some(next) = chars.next() {
+                        out.push(next);
+                    }
                 }
-            } else {
-                decoded.push(ch);
+                '\u{1e}' | '\u{1f}' => {}
+                c => out.push(c),
             }
         }
-
-        decoded
+        out
     }
 
     fn read_double_quoted_string(&mut self) -> Option<Token> {

crates/bashkit/tests/spec_cases/bash/ansi-c-control-bytes.test.sh

@@ -0,0 +1,34 @@
+# $'\x1e' and $'\x1f' (ASCII record/unit separators) are data. Scripts use
+# SEP=$'\x1f' to build composite associative-array keys.
+
+### ansi_c_unit_separator_kept
+printf '%s' $'a\x1fb\x1ec' | od -An -c | tr -s ' '
+### expect
+ a 037 b 036 c
+### end
+
+### ansi_c_separator_in_concatenation
+k="a"$'\x1f'"b"; printf '%s' "$k" | od -An -c | tr -s ' '
+SEP=$'\x1f'; m="x${SEP}y"; echo "${#m}"
+### expect
+ a 037 b
+3
+### end
+
+### ansi_c_separator_composite_assoc_key
+declare -A v
+section=server; key=host
+v["${section}"$'\x1f'"${key}"]=localhost
+mk="server"$'\x1f'"host"
+echo "${v[$mk]}" "${#v[@]}"
+for k in "${!v[@]}"; do printf '%s' "$k" | od -An -c | tr -s ' '; done
+### expect
+localhost 1
+ s e r v e r 037 h o s t
+### end
+
+### ansi_c_separator_after_single_quote
+x='a'$'\x1f''b'; echo "${#x}"
+### expect
+3
+### end

knowledge/log.md

@@ -2,6 +2,7 @@
 
 ## 2026-10-07
 
+* **Fix**: `$'\x1e'` and `$'\x1f'` bytes (record/unit separators) are kept as data. They collided with the lexer's quote-boundary markers and were silently dropped, which broke the common `SEP=$'\x1f'` composite-key idiom for associative arrays (`v["$section"$'\x1f'"$key"]`). Decoded ANSI-C content now NUL-escapes those bytes like `$`. Found by gap telemetry (`config_ini_merge` replay printed nothing).
 * **Fix**: A usage error in a bundled builtin no longer aborts the script. `grep -E '('`, `grep -P` with a bad pattern, `rg` with a bad regex, an awk syntax error, `grep` with no pattern and `grep -f missing` now fail that one command with exit 2 and the message on stderr, and the script continues (bash semantics; `set -e` still stops). `Err` from custom builtins (`BashBuilder::builtin`, registry, resolver) keeps aborting, per the public contract. See foundations/builtins "Errors: Failed Command vs Aborted Execution".
 * **Feature**: `gawk`, `mawk` and `nawk` run the awk builtin (same engine, including gawk extensions it already had such as `gensub`, `toupper`). Gap telemetry showed agents probing `gawk` by name. `--version` still identifies as bashkit awk.
 * **Fix**: `tar -P`/`--absolute-names` keeps the leading `/` (no notice), `-T FILE`/`--files-from` reads member names (blank lines skipped, `-` is stdin), and `--` ends options. `ls` implements the quoting styles: `-b`/`--escape`, `-N`, `-Q`, `--quoting-style=literal|escape|c|shell|shell-always|shell-escape|shell-escape-always` (GNU rules, including `"it's"` and headers left unescaped under `-b`); `locale`/`clocale` still report not implemented. Found by gap telemetry.