From e68f9f4347d688058e7865005fc95b8de6c69bd8 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Thu, 8 Oct 2026 02:08:56 +0000 Subject: [PATCH] fix(interpreter): ${#v} counts bytes under the C locale LC_ALL, LC_CTYPE, LANG (first non-empty) set to C or POSIX make ${#v} count bytes like bash. ${#v} inside a ${x:-...} operand counted bytes in every locale and now counts characters. --- crates/bashkit/src/interpreter/arithmetic.rs | 9 +-- crates/bashkit/src/interpreter/expansion.rs | 19 ++++++- .../spec_cases/bash/c-locale-length.test.sh | 56 +++++++++++++++++++ knowledge/log.md | 1 + knowledge/operations/limitations.md | 1 + 5 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 crates/bashkit/tests/spec_cases/bash/c-locale-length.test.sh diff --git a/crates/bashkit/src/interpreter/arithmetic.rs b/crates/bashkit/src/interpreter/arithmetic.rs index f2e390942..43b6d4b94 100644 --- a/crates/bashkit/src/interpreter/arithmetic.rs +++ b/crates/bashkit/src/interpreter/arithmetic.rs @@ -918,14 +918,11 @@ impl Interpreter { } return "0".to_string(); } - return self - .expand_name_or_array_element(&rest[..=end]) - .chars() - .count() - .to_string(); + let val = self.expand_name_or_array_element(&rest[..=end]); + return self.shell_length(&val).to_string(); } let val = self.expand_variable(rest); - return val.chars().count().to_string(); + return self.shell_length(&val).to_string(); } if let Some(bracket) = inner.find('[') diff --git a/crates/bashkit/src/interpreter/expansion.rs b/crates/bashkit/src/interpreter/expansion.rs index 6f756d879..d3d640faa 100644 --- a/crates/bashkit/src/interpreter/expansion.rs +++ b/crates/bashkit/src/interpreter/expansion.rs @@ -120,6 +120,21 @@ impl Interpreter { result } + /// `${#v}`: characters, or bytes when the shell's locale is C/POSIX. + /// The locale comes from `LC_ALL`, then `LC_CTYPE`, then `LANG`, the first + /// one set and non-empty. With none set bashkit stays UTF-8 (bash would + /// fall back to C), so only an explicit `C`/`POSIX` switches to bytes. + pub(super) fn shell_length(&self, value: &str) -> usize { + let locale = ["LC_ALL", "LC_CTYPE", "LANG"] + .iter() + .map(|name| self.expand_variable(name)) + .find(|v| !v.is_empty()); + match locale.as_deref() { + Some("C" | "POSIX") => value.len(), + _ => value.chars().count(), + } + } + /// Apply a `${var@operator}` transformation. pub(super) fn apply_transformation(&self, name: &str, operator: char) -> String { // `${v@a}`: the variable's attribute letters, the same ones @@ -409,7 +424,7 @@ impl Interpreter { } else { self.expand_variable(name) }; - result.push_str(&value.chars().count().to_string()); + result.push_str(&self.shell_length(&value).to_string()); } WordPart::ParameterExpansion { name, @@ -1225,7 +1240,7 @@ impl Interpreter { Self::push_operand_expansion(&mut result, &expanded, in_marked || force_quoted); } WordPart::Length(name) => { - let value = self.expand_variable(name).len().to_string(); + let value = self.shell_length(&self.expand_variable(name)).to_string(); Self::push_operand_expansion(&mut result, &value, in_marked || force_quoted); } // Run ahead by `prefetch_operand_substs` (default-family diff --git a/crates/bashkit/tests/spec_cases/bash/c-locale-length.test.sh b/crates/bashkit/tests/spec_cases/bash/c-locale-length.test.sh new file mode 100644 index 000000000..d38a8b0c3 --- /dev/null +++ b/crates/bashkit/tests/spec_cases/bash/c-locale-length.test.sh @@ -0,0 +1,56 @@ +### length_counts_bytes_under_c_locale +s="ação" +LC_ALL=C +echo "${#s}" +LC_ALL=C.UTF-8 +echo "${#s}" +### expect +6 +4 +### end + +### length_c_locale_from_lc_ctype_and_lang +s="日本" +LC_ALL= +LC_CTYPE=POSIX +echo "${#s}" +LC_CTYPE=C.UTF-8 +echo "${#s}" +LC_CTYPE= +LANG=C +echo "${#s}" +### expect +6 +2 +6 +### end + +### length_c_locale_array_element_and_arithmetic +LC_ALL=C +a=(ação) +s=ñ +echo "${#a[0]} $(( ${#s} + 0 ))" +### expect +6 2 +### end + +### length_c_locale_scoped_by_local +s="ação" +LC_ALL=C +f() { local LC_ALL=C.UTF-8; echo "in=${#s}"; } +f +echo "out=${#s}" +### expect +in=4 +out=6 +### end + +### length_inside_default_operand_counts_characters +s="ação" +echo "${unset_x:-${#s}}" +LC_ALL=C +echo "${unset_x:-${#s}}" +### expect +4 +6 +### end diff --git a/knowledge/log.md b/knowledge/log.md index 00212057b..76491a082 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -3,6 +3,7 @@ ## 2026-10-07 * **Fix**: `${v@a}` prints every attribute letter `declare -p` shows, in bash's order (`a`/`A`, `i`, `n`, `r`, `x`, `l`, `u`), follows namerefs, takes the array from a subscript, and repeats per element for `${arr[@]@a}`. It only knew `r` and `x`, so `declare -i n; ${n@a}` was empty. +* **Fix**: `${#v}` counts bytes when the shell's locale is C/POSIX (`LC_ALL`, then `LC_CTYPE`, then `LANG`, first non-empty), like bash; `LC_ALL=C; s=ação; ${#s}` is 6. With no locale variable set bashkit stays UTF-8, where bash would fall back to C. `${#s}` inside a `${x:-...}` operand counted bytes in every locale and now counts characters. Other operators (`${s:0:2}`, case changes, globs) stay character-based under C (limitations, "Locale"). - Added the bash-oracle scoreboard (`scripts/bash-oracle`, `just bash-oracle`): 106 whole-script cases replayed against output recorded from real bash 5.2, scored as `match` (status + stdout, 90/106) and `strict` (also stderr, diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 9d0713ddf..c521fd31d 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -109,6 +109,7 @@ Target: IEEE 1003.1-2024 Shell Command Language. | Expansion errors | `set -u` unbound variables stop the shell | `$((1/0))` evaluates to 0 and `${x!}` expands empty; bash reports them and drops the rest of the line the failing command ends on (`bashbox_line_number_an_expansion_error_drops_the_rest_of_the_line_it_ends_on`, skipped) | | Process substitution | `<(cmd)` and `>(cmd)` as filenames, nesting, redirections | The path is `/dev/fd/proc_sub_`, not bash's `/dev/fd/63`. Several interpreters can share one filesystem, so a per-shell descending number would let one tenant's substitution overwrite another's. A script that parses the number sees a different one (`scripts/bash-oracle/cases/process-substitution-dev-fd.sh`) | | Named fds / `<>` | `{var}>file`, `{var}file`, `{var}>>file`, `{var}<<&N`/`<&N` allocate the lowest free fd >= 10 on any command (persisting, as in bash); `{var}>&-` closes; `>&N` with N closed is `Bad file descriptor`; `N<>file`, `N<<file` has no shared read/write offset (writes append); `3&1`, `( ... ) 3>&1`) is dropped; a `Bad file descriptor` message ignores redirect order (`2>&1 >&7`) | +| Locale | UTF-8 text everywhere; `${#v}` counts bytes when `LC_ALL`/`LC_CTYPE`/`LANG` (first non-empty) is `C` or `POSIX` | With no locale variable set bashkit is UTF-8, where bash falls back to C. Under C, `${s:off:len}`, `${s^^}`, `${s#?}` and glob `?` still work on characters, not bytes: shell words are Rust strings and cannot hold a split UTF-8 sequence (L-STREAM-001) | | Brace expansion | Runs before other expansions on unquoted literal text; lists, nesting, ranges with step and zero padding, quoted/escaped braces and commas | Works on parsed words, not raw text: `$v{1,2}` expands `$v` then appends `1`/`2` (bash reads `$v1`, `$v2`); write `${v}` | ## Builtins