From 233cc0588d48e6aebedc1b709c93e7e2dab77837 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 19:06:48 +0000 Subject: [PATCH] fix(awk): visit for-in keys in gawk's storage order Unsorted for (k in a) now models gawk's cint/int/str arrays, so loops print like Debian awk. All gawk cases in the Debian-oracle awk set pass. Claude-Session: https://claude.ai/code/session_01VscpEzxtqpCKx5eRMpywNj --- crates/bashkit/src/builtins/awk/interp.rs | 28 +- crates/bashkit/src/builtins/awk/mod.rs | 1 + crates/bashkit/src/builtins/awk/order.rs | 305 ++++++++++++++++++ .../bashkit/tests/spec_cases/awk/awk.test.sh | 2 +- knowledge/foundations/builtins.md | 8 +- knowledge/log.md | 1 + knowledge/operations/limitations.md | 2 +- 7 files changed, 320 insertions(+), 27 deletions(-) create mode 100644 crates/bashkit/src/builtins/awk/order.rs diff --git a/crates/bashkit/src/builtins/awk/interp.rs b/crates/bashkit/src/builtins/awk/interp.rs index d89a8694a..64f582c73 100644 --- a/crates/bashkit/src/builtins/awk/interp.rs +++ b/crates/bashkit/src/builtins/awk/interp.rs @@ -7,10 +7,9 @@ //! - Arrays live in an arena and are referenced by id, which gives awk's //! by-reference array arguments for free. A function's untyped local that //! becomes an array is owned by its call frame and freed on return. -//! - `for (k in a)` visits numeric keys ascending, then the others by -//! bytes (gawk's order is a hash order). Arrays keep insertion order, -//! which `PROCINFO["sorted_in"] = "@unsorted"` exposes; the other gawk -//! `sorted_in` orders are supported. +//! - `for (k in a)` (and `sorted_in` `@unsorted`) visits keys in gawk's +//! storage order, modelled in `order`; arrays themselves keep insertion +//! order. The other gawk `sorted_in` orders are supported. //! - Runtime errors that gawk reports (`division by zero attempted`, ...) //! carry gawk's `awk: cmd. line:N: fatal:` prefix. Bashkit resource caps //! keep the plain `awk: fatal: ...` form (TM-DOS-109). @@ -1511,21 +1510,8 @@ impl Interp { Cell::Val(_) => String::new(), }; let mut keys = arr.keys(); - if order == "@unsorted" { - return keys; - } - if order.is_empty() { - // Default: numeric keys ascending, then the rest by bytes. - // Deterministic; gawk's own order follows its hash tables. - // TODO: reproduce gawk's str/int/cint array iteration order - // so unsorted `for (k in a)` output matches Debian awk. - keys.sort_by(|a, b| match (key_num(a), key_num(b)) { - (Some(x), Some(y)) => x.total_cmp(&y).then_with(|| a.cmp(b)), - (Some(_), None) => std::cmp::Ordering::Less, - (None, Some(_)) => std::cmp::Ordering::Greater, - (None, None) => a.cmp(b), - }); - return keys; + if order.is_empty() || order == "@unsorted" { + return super::order::gawk_order(keys); } let val = |k: &String| match arr.get(k) { Some(Elem::Val(v)) => v.clone(), @@ -1704,10 +1690,6 @@ fn elem_cost(key: &str, e: &Elem) -> usize { } } -fn key_num(k: &str) -> Option { - k.parse::().ok().filter(|n| n.is_finite()) -} - fn num_cmp(a: &str, b: &str) -> std::cmp::Ordering { let (x, y) = (super::value::str_to_num(a), super::value::str_to_num(b)); x.total_cmp(&y).then_with(|| a.cmp(b)) diff --git a/crates/bashkit/src/builtins/awk/mod.rs b/crates/bashkit/src/builtins/awk/mod.rs index e0e72939a..f57fc1d77 100644 --- a/crates/bashkit/src/builtins/awk/mod.rs +++ b/crates/bashkit/src/builtins/awk/mod.rs @@ -30,6 +30,7 @@ mod funcs; mod interp; mod io; mod lexer; +mod order; mod parser; mod regex; mod value; diff --git a/crates/bashkit/src/builtins/awk/order.rs b/crates/bashkit/src/builtins/awk/order.rs new file mode 100644 index 000000000..dab7e2113 --- /dev/null +++ b/crates/bashkit/src/builtins/awk/order.rs @@ -0,0 +1,305 @@ +//! gawk's unsorted `for (k in a)` order. +//! +//! Decisions: +//! - Debian's awk is gawk, and scripts (and their expected output) depend on +//! the order `for (k in a)` visits keys without `PROCINFO["sorted_in"]`. +//! That order falls out of gawk's array storage, so this module models the +//! storage, not just a sort: the first key stored in an empty array picks +//! the kind, as in gawk 5.2: +//! - **cint** (first key a non-negative integer): integers are kept by +//! value and listed ascending; other keys spill into an extra array +//! that is listed first. Sparse integers spill too once the integer +//! storage would waste more than 2048 slots. +//! - **int** (first key a negative integer): a chained hash of integers, +//! two per bucket, plus a string array for other keys, listed first. +//! - **str**: a chained hash with sdbm hashing (32-bit), 13/127/1021/... +//! buckets, growth when keys per bucket exceed 2, new entries at the +//! bucket head. Listing walks buckets in ascending order. +//! - The model replays the surviving keys in insertion order. gawk's order +//! also depends on deleted keys (tables never shrink until empty), so after +//! deletes the order can differ (L-AWK-001). Behavior was derived from +//! observed output and gawk's documented design; no gawk code is copied. +//! - Cost is O(n) per loop start, like the list gawk builds itself. + +/// Keys of an array (insertion order) in gawk's unsorted order. +pub(super) fn gawk_order(keys: Vec) -> Vec { + let Some(first) = keys.first() else { + return keys; + }; + match int_key(first) { + Some(n) if n >= 0 => cint_order(keys), + Some(_) => { + let mut a = IntArray::default(); + for k in keys { + a.insert(k); + } + a.list() + } + None => { + let mut a = StrArray::default(); + for k in keys { + a.insert(k); + } + a.list() + } + } +} + +/// gawk treats a subscript as an integer when it is the canonical decimal +/// form of a 32-bit integer (`"3"`, `"-3"`; not `"03"`, `"-0"`, `"+3"`). +fn int_key(k: &str) -> Option { + let digits = k.strip_prefix('-').unwrap_or(k); + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + if digits.len() > 1 && digits.starts_with('0') || k == "-0" { + return None; + } + let n: i64 = k.parse().ok()?; + (i64::from(i32::MIN)..=i64::from(i32::MAX)) + .contains(&n) + .then_some(n) +} + +const SIZES: [usize; 21] = [ + 13, 127, 1021, 8191, 16381, 32749, 65497, 131101, 262147, 524309, 1048583, 2097169, 4194319, + 8388617, 16777259, 33554467, 67108879, 134217757, 268435459, 536870923, 1073741827, +]; +const CHAIN_MAX: usize = 2; + +fn next_size(cur: usize) -> Option { + SIZES.iter().copied().find(|&s| s > cur) +} + +fn str_hash(k: &str) -> u64 { + let mut h: u64 = 0; + for &b in k.as_bytes() { + // C `char` is signed on gawk's Debian targets. + let c = b as i8 as i64 as u64; + let t = h << 6; + h = c.wrapping_add(t).wrapping_add(t << 10).wrapping_sub(h) & 0xFFFF_FFFF; + } + h +} + +/// Hash table of strings; each bucket lists its newest entry first. +#[derive(Default)] +struct StrArray { + buckets: Vec>, + len: usize, +} + +impl StrArray { + fn insert(&mut self, k: String) { + if self.buckets.is_empty() { + self.buckets = vec![Vec::new(); SIZES[0]]; + } + let code = str_hash(&k); + self.len += 1; + if self.len / self.buckets.len() > CHAIN_MAX + && let Some(n) = next_size(self.buckets.len()) + { + let old = std::mem::replace(&mut self.buckets, vec![Vec::new(); n]); + for (c, key) in old.into_iter().flatten() { + self.buckets[(c % n as u64) as usize].insert(0, (c, key)); + } + } + let n = self.buckets.len() as u64; + self.buckets[(code % n) as usize].insert(0, (code, k)); + } + + fn list(self) -> Vec { + self.buckets.into_iter().flatten().map(|(_, k)| k).collect() + } +} + +fn int_hash(k: i64, size: usize) -> usize { + let mut k = k as u32; + k ^= k << 3; + k = k.wrapping_add(k >> 5); + k ^= k << 4; + k = k.wrapping_add(k >> 17); + k ^= k << 25; + k = k.wrapping_add(k >> 6); + (k as usize) % size +} + +/// Hash table of integers (two per bucket, head bucket filled first) plus +/// a string array for other keys. +#[derive(Default)] +struct IntArray { + /// Bucket chains, head first; each chain link holds up to two keys. + buckets: Vec>>, + ints: usize, + rest: Option, +} + +impl IntArray { + fn insert(&mut self, k: String) { + let Some(n) = int_key(&k) else { + self.rest.get_or_insert_with(StrArray::default).insert(k); + return; + }; + if self.buckets.is_empty() { + self.buckets = vec![Vec::new(); SIZES[0]]; + } + self.ints += 1; + if self.ints / self.buckets.len() > CHAIN_MAX + && let Some(size) = next_size(self.buckets.len()) + { + let old = std::mem::replace(&mut self.buckets, vec![Vec::new(); size]); + for (num, key) in old.into_iter().flatten().flatten() { + self.put(num, key); + } + } + self.put(n, k); + } + + fn put(&mut self, n: i64, k: String) { + let size = self.buckets.len(); + let chain = &mut self.buckets[int_hash(n, size)]; + match chain.first_mut() { + Some(head) if head.len() < 2 => head.push((n, k)), + _ => chain.insert(0, vec![(n, k)]), + } + } + + fn list(self) -> Vec { + let mut out = self.rest.map(StrArray::list).unwrap_or_default(); + out.extend(self.buckets.into_iter().flatten().flatten().map(|(_, k)| k)); + out + } +} + +const NHAT: u32 = 10; +const THRESHOLD: i64 = 1 << (NHAT + 1); + +/// Storage group of a non-negative integer: everything below 2^NHAT shares +/// one, then one per power of two. +fn cint_group(k: i64) -> u32 { + let r = 63 - (k.max(1) as u64).leading_zeros(); + if r < NHAT { NHAT } else { r + 1 } +} + +/// The fixed-size slot block (base, size) that holds `k`. +fn cint_leaf(k: i64) -> (i64, i64) { + let m = cint_group(k) - 1; + let (mut m, mut base) = if m < NHAT { (NHAT, 0) } else { (m, 1i64 << m) }; + loop { + let n = m.div_ceil(2); + let size = 1i64 << n; + base += size * ((k - base) / size); + if n > NHAT { + m = n; + } else { + return (base, size); + } + } +} + +fn cint_order(keys: Vec) -> Vec { + let mut leaves = std::collections::HashSet::new(); + let mut capacity: i64 = 0; + let mut ints: Vec<(i64, String)> = Vec::new(); + let mut rest: Option = None; + for k in keys { + if let Some(n) = int_key(&k).filter(|&n| n >= 0) { + let m = cint_group(n) - 1; + let mut li = m.max(NHAT); + while li >= NHAT { + li = li.div_ceil(2); + } + if capacity + (1i64 << li) - ints.len() as i64 <= THRESHOLD { + let (base, size) = cint_leaf(n); + if leaves.insert(base) { + capacity += size; + } + ints.push((n, k)); + continue; + } + } + match &mut rest { + Some(Rest::Int(a)) => a.insert(k), + Some(Rest::Str(a)) => a.insert(k), + None if int_key(&k).is_some() => { + let mut a = IntArray::default(); + a.insert(k); + rest = Some(Rest::Int(a)); + } + None => { + let mut a = StrArray::default(); + a.insert(k); + rest = Some(Rest::Str(a)); + } + } + } + ints.sort_by_key(|(n, _)| *n); + let mut out = match rest { + Some(Rest::Int(a)) => a.list(), + Some(Rest::Str(a)) => a.list(), + None => Vec::new(), + }; + out.extend(ints.into_iter().map(|(_, k)| k)); + out +} + +enum Rest { + Int(IntArray), + Str(StrArray), +} + +#[cfg(test)] +mod tests { + use super::*; + + fn order(keys: &[&str]) -> Vec { + gawk_order(keys.iter().map(|k| k.to_string()).collect()) + } + + #[test] + fn string_keys_follow_hash_buckets() { + // gawk 5.2.1: marketing, support, sales, engineering. + assert_eq!( + order(&["engineering", "marketing", "sales", "support"]), + ["marketing", "support", "sales", "engineering"] + ); + } + + #[test] + fn integer_keys_ascend() { + assert_eq!(order(&["3", "1", "10", "2"]), ["1", "2", "3", "10"]); + } + + #[test] + fn non_integer_keys_list_before_integers() { + let got = order(&["2", "x", "1"]); + assert_eq!(got, ["x", "1", "2"]); + } + + #[test] + fn int_key_is_canonical_i32() { + assert_eq!(int_key("0"), Some(0)); + assert_eq!(int_key("-3"), Some(-3)); + assert_eq!(int_key("03"), None); + assert_eq!(int_key("-0"), None); + assert_eq!(int_key("+3"), None); + assert_eq!(int_key("1.5"), None); + assert_eq!(int_key("2147483648"), None); + assert_eq!(int_key(""), None); + } + + #[test] + fn every_key_listed_once() { + let keys: Vec = (0..5000).map(|i| format!("k{i}")).collect(); + let mut got = gawk_order(keys.clone()); + assert_eq!(got.len(), keys.len()); + got.sort(); + let mut want = keys; + want.sort(); + assert_eq!(got, want); + let ints: Vec = (-3000..3000).map(|i| i.to_string()).collect(); + assert_eq!(gawk_order(ints.clone()).len(), ints.len()); + let sparse: Vec = (0..200).map(|i| (i * 100_000).to_string()).collect(); + assert_eq!(gawk_order(sparse).len(), 200); + } +} diff --git a/crates/bashkit/tests/spec_cases/awk/awk.test.sh b/crates/bashkit/tests/spec_cases/awk/awk.test.sh index 7e92bb296..75b73012b 100644 --- a/crates/bashkit/tests/spec_cases/awk/awk.test.sh +++ b/crates/bashkit/tests/spec_cases/awk/awk.test.sh @@ -528,7 +528,7 @@ printf 'a\n' | awk 'BEGIN {a[1]="x"; a[2]="y"} {for (k in a) print k, a[k]}' ### end ### awk_for_in_string_keys -# for-in with string keys sorts lexically +# for-in visits keys in gawk hash order (single bytes land in byte order) printf 'a\n' | awk 'BEGIN {a["b"]="2"; a["a"]="1"; a["c"]="3"} {for (k in a) print k, a[k]}' ### expect a 1 diff --git a/knowledge/foundations/builtins.md b/knowledge/foundations/builtins.md index 3ef7bb775..837e694c7 100644 --- a/knowledge/foundations/builtins.md +++ b/knowledge/foundations/builtins.md @@ -394,8 +394,12 @@ and the extensions agents use (`gensub`, `asort`/`asorti`, `patsplit`, `strftime`/`mktime`/`systime`, `BEGINFILE`/`ENDFILE`, `switch`, arrays of arrays, `PROCINFO["sorted_in"]`, `FIELDWIDTHS`/`FPAT`, `IGNORECASE`, `RS` as a regex, `RT`). `rand()` reproduces gawk's random() sequence. Debian-oracle -differential score: 94% of 260 awk cases (51% before the rewrite); the -remaining gawk gap is unsorted `for (k in a)` order. +differential score: 94% of 260 awk cases (51% before the rewrite); every +gawk case passes, the misses expect `mawk`'s own dialect. Unsorted +`for (k in a)` follows gawk's storage (`order`): a non-negative integer +first key makes an integer array (other keys listed first, then integers +ascending), otherwise a chained hash (sdbm for strings) listed bucket by +bucket, newest first. Pipeline: `lexer` -> `parser` (names resolved to slots, AST in `ast`) -> `interp` (async evaluator) with `io` (records, `getline`, redirections, diff --git a/knowledge/log.md b/knowledge/log.md index dc8424604..c48747837 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Fix**: awk `for (k in a)` without `PROCINFO["sorted_in"]` now visits keys in gawk's order (integer arrays ascending with other keys first; string keys in gawk's hash-bucket order), so unsorted loops print like Debian awk. All gawk cases in the Debian-oracle awk set now pass (L-AWK-001 narrowed to post-`delete` order). * **Feature**: awk rewritten to target gawk 5.2 (Debian's `awk`). New lexer, parser, async evaluator, ERE translation with POSIX leftmost-longest matching, gawk number formatting and `printf`, gawk's `rand()` sequence, `substr`/`getline`/`typeof` edge cases, `-f`/`-e`/`--` options and gawk's error messages. `system()`, `print | cmd` and `cmd | getline` now run through the sandbox shell (execution-plan driver). Debian-oracle score for awk went from 51% to 94% of 260 cases; the remaining gawk gap is unsorted `for (k in a)` order (L-AWK-001). All resource caps kept; new TM-DOS-128 for awk commands. See foundations/builtins "awk". * **Decision**: CPython's workload priority is agents running `python3` for scripted logic (processing a file, transforming data). Low-use stdlib modules are no longer shipped: `unittest`, `doctest`, profilers, `compileall`/`zipapp`, `dbm`/`shelve`, `plistlib`, `wave`, `netrc`, `cmd`, `tty`/`pty`, and `bz2`/`lzma`/`compression.zstd` (no C codec in the guest anyway). Stdlib zip 3.58 to 3.16 MB. L-CPY-009 updated. See [CPython WebAssembly Runtime](runtimes/cpython-wasm.md#workload-priority). * **Feature**: `yes` and `seq` stream into pipelines. As a producer stage they write chunks into the pipe as they go (`Context::stdout_stream`), so `yes | head -1` and `seq 1000000 | head -1` stop after the first chunk with `PIPESTATUS` `141 0` instead of running to the output cap and printing a truncation notice. Step 2 of the roadmap (streaming builtin I/O), starting with the generators. diff --git a/knowledge/operations/limitations.md b/knowledge/operations/limitations.md index 2ad901bed..f3a7b09cf 100644 --- a/knowledge/operations/limitations.md +++ b/knowledge/operations/limitations.md @@ -145,7 +145,7 @@ pass in CI); only divergences and boundaries are recorded here. | ID | Tool | Limitation | Evidence | |----|------|------------|----------| -| L-AWK-001 | awk | `for (k in a)` without `PROCINFO["sorted_in"]` visits numeric keys ascending, then the rest by bytes; gawk uses its hash-table order, so unsorted loops can print in a different order than Debian awk | `awk_for_in_string_keys` spec | +| L-AWK-001 | awk | `for (k in a)` without `PROCINFO["sorted_in"]` follows a model of gawk's array storage replayed from the keys still present; after `delete`, gawk's order also depends on the deleted keys, so it can differ from Debian awk | `order::tests`, `awk_for_in_string_keys` spec | | L-AWK-002 | awk | `print \| cmd` collects the command's input and runs it at `close()` or exit (as if the pipe were read at once); `\|&` coprocesses, `@include`/`@load`/`@namespace`, MPFR (`-M`), `--profile` and the debugger are not supported | stance | | L-AWK-003 | awk | Commands (`system()`, pipes) need the shell: through `Bash::exec` they run as `sh -c` in the sandbox; a direct `Builtin::execute` call (embedders) has no shell, so they print a notice and report status 127 | `test_awk_print_redirect_pipe_needs_plan_driver` | | L-AWK-004 | awk | `mawk` and `nawk` run the gawk dialect: mawk's `-W` options, messages and its `substr`/division-by-zero behavior are not emulated | stance |