From d84379833c7e010ef6f2780b2443398bdd5f35ef Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Wed, 7 Oct 2026 18:06:10 +0000 Subject: [PATCH] feat(awk): rewrite awk to match gawk 5.2 New lexer, parser, async evaluator and I/O layer targeting gawk, Debian's awk. Commands (system, print | cmd, cmd | getline) run in the sandbox shell through an execution-plan driver. ERE translation adds POSIX leftmost-longest matching. Number output, printf, rand, substr, getline and diagnostics follow gawk. All resource caps are kept; split and record splitting now stop at the memory cap before building their vectors. Debian-oracle awk score: 51.5% -> 93.8% of 260 cases. Claude-Session: https://claude.ai/code/session_01VscpEzxtqpCKx5eRMpywNj --- Cargo.lock | 1 + Cargo.toml | 1 + crates/bashkit/Cargo.toml | 1 + crates/bashkit/docs/threat-model.md | 1 + crates/bashkit/src/builtins/awk/ast.rs | 343 ++ crates/bashkit/src/builtins/awk/format.rs | 594 ++++ crates/bashkit/src/builtins/awk/funcs.rs | 883 +++++ crates/bashkit/src/builtins/awk/interp.rs | 1900 +++++++++++ .../bashkit/src/builtins/awk/interpreter.rs | 1871 ----------- crates/bashkit/src/builtins/awk/io.rs | 792 +++++ crates/bashkit/src/builtins/awk/lexer.rs | 667 ++++ crates/bashkit/src/builtins/awk/mod.rs | 1312 +++----- crates/bashkit/src/builtins/awk/parser.rs | 2941 +++++++++-------- crates/bashkit/src/builtins/awk/regex.rs | 584 ++++ crates/bashkit/src/builtins/awk/tests.rs | 91 +- crates/bashkit/src/builtins/awk/value.rs | 223 ++ crates/bashkit/src/builtins/date.rs | 18 + crates/bashkit/src/interpreter/mod.rs | 7 +- .../integration/builtin_fs_error_tests.rs | 2 +- .../bashkit/tests/spec_cases/awk/awk.test.sh | 50 + knowledge/foundations/builtins.md | 32 + knowledge/log.md | 1 + knowledge/operations/limitations.md | 5 +- knowledge/security/threat-model.md | 9 +- 24 files changed, 8163 insertions(+), 4166 deletions(-) create mode 100644 crates/bashkit/src/builtins/awk/ast.rs create mode 100644 crates/bashkit/src/builtins/awk/format.rs create mode 100644 crates/bashkit/src/builtins/awk/funcs.rs create mode 100644 crates/bashkit/src/builtins/awk/interp.rs delete mode 100644 crates/bashkit/src/builtins/awk/interpreter.rs create mode 100644 crates/bashkit/src/builtins/awk/io.rs create mode 100644 crates/bashkit/src/builtins/awk/lexer.rs create mode 100644 crates/bashkit/src/builtins/awk/regex.rs create mode 100644 crates/bashkit/src/builtins/awk/value.rs diff --git a/Cargo.lock b/Cargo.lock index 999bf17b6..2ae16ade3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -400,6 +400,7 @@ dependencies = [ "proptest", "rand 0.10.3", "regex", + "regex-automata", "reqwest", "russh", "rustls", diff --git a/Cargo.toml b/Cargo.toml index fcca46cb7..554f99397 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ rustls = { version = "0.23", default-features = false, features = ["std", "tls12 # Regex regex = "1" +regex-automata = { version = "0.4", default-features = false, features = ["std", "syntax", "hybrid", "unicode", "perf"] } fancy-regex = "0.19" # Date/Time diff --git a/crates/bashkit/Cargo.toml b/crates/bashkit/Cargo.toml index 44f8f605b..ba28efdf6 100644 --- a/crates/bashkit/Cargo.toml +++ b/crates/bashkit/Cargo.toml @@ -34,6 +34,7 @@ serde_yaml_ng = { workspace = true, optional = true } # Regex regex = { workspace = true } +regex-automata = { workspace = true } fancy-regex = { workspace = true } # Date/Time diff --git a/crates/bashkit/docs/threat-model.md b/crates/bashkit/docs/threat-model.md index 99439bec9..99909ec48 100644 --- a/crates/bashkit/docs/threat-model.md +++ b/crates/bashkit/docs/threat-model.md @@ -75,6 +75,7 @@ through configurable limits. | Long computation (TM-DOS-023) | Complex awk/sed regex, including repeated awk and `[[ =~ ]]` operands | Linear-time engine; bounded runtime regex caches; timeout (30s) | MITIGATED | | Regex backtrack (TM-DOS-025) | `grep "a](*b)*c" file` | Regex crate limits | PARTIAL | | AWK unbounded loops (TM-DOS-033) | `BEGIN { while(1){} }`, nested loops, deep recursion | Per-loop and whole-program loop caps from `ExecutionLimits`, call-depth cap; all fatal (exit 2) | MITIGATED | +| awk commands (TM-DOS-128) | `system()`, `print \| cmd` or `cmd \| getline` in a loop | Each command runs as `sh -c` in the sandbox shell under the session's command budget and timeout; awk's loop, output and getline caps still apply | MITIGATED | **Stack Overflow / Recursion:** diff --git a/crates/bashkit/src/builtins/awk/ast.rs b/crates/bashkit/src/builtins/awk/ast.rs new file mode 100644 index 000000000..97699f38a --- /dev/null +++ b/crates/bashkit/src/builtins/awk/ast.rs @@ -0,0 +1,343 @@ +//! awk syntax tree. +//! +//! Decisions: +//! - Variables are resolved at parse time to slots: `Global(i)` indexes the +//! interpreter's global table (special variables first, see `special`), +//! `Local(i)` a function parameter in the current call frame. +//! - Lvalues are the expression forms `Var`, `Elem` and `Field`; assignment +//! targets are checked by the parser. +//! - `Elem` keeps one subscript group per `[...]` so gawk arrays of arrays +//! (`a[i][j]`) work; commas inside one group join with `SUBSEP`. +//! - User function calls hold a function index; definitions may follow use, +//! undefined functions are reported after parsing (gawk does the same). + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum VarRef { + Global(u32), + Local(u32), +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum BinOp { + Add, + Sub, + Mul, + Div, + Mod, + Pow, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum CmpOp { + Lt, + Le, + Gt, + Ge, + Eq, + Ne, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum Bi { + Length, + Substr, + Index, + Split, + Sub, + Gsub, + Gensub, + Match, + Sprintf, + Sin, + Cos, + Atan2, + Exp, + Log, + Sqrt, + Int, + Rand, + Srand, + Tolower, + Toupper, + System, + Close, + Fflush, + Strtonum, + Systime, + Strftime, + Mktime, + Asort, + Asorti, + Typeof, + Isarray, + And, + Or, + Xor, + Lshift, + Rshift, + Compl, + Patsplit, +} + +impl Bi { + pub(super) fn from_name(name: &str) -> Option { + Some(match name { + "length" => Bi::Length, + "substr" => Bi::Substr, + "index" => Bi::Index, + "split" => Bi::Split, + "sub" => Bi::Sub, + "gsub" => Bi::Gsub, + "gensub" => Bi::Gensub, + "match" => Bi::Match, + "sprintf" => Bi::Sprintf, + "sin" => Bi::Sin, + "cos" => Bi::Cos, + "atan2" => Bi::Atan2, + "exp" => Bi::Exp, + "log" => Bi::Log, + "sqrt" => Bi::Sqrt, + "int" => Bi::Int, + "rand" => Bi::Rand, + "srand" => Bi::Srand, + "tolower" => Bi::Tolower, + "toupper" => Bi::Toupper, + "system" => Bi::System, + "close" => Bi::Close, + "fflush" => Bi::Fflush, + "strtonum" => Bi::Strtonum, + "systime" => Bi::Systime, + "strftime" => Bi::Strftime, + "mktime" => Bi::Mktime, + "asort" => Bi::Asort, + "asorti" => Bi::Asorti, + "typeof" => Bi::Typeof, + "isarray" => Bi::Isarray, + "and" => Bi::And, + "or" => Bi::Or, + "xor" => Bi::Xor, + "lshift" => Bi::Lshift, + "rshift" => Bi::Rshift, + "compl" => Bi::Compl, + "patsplit" => Bi::Patsplit, + _ => return None, + }) + } + + /// Argument positions that name an array (not evaluated as values). + pub(super) fn array_args(self) -> &'static [usize] { + match self { + Bi::Split | Bi::Patsplit => &[1, 3], + Bi::Match => &[2], + Bi::Asort | Bi::Asorti => &[0, 1], + _ => &[], + } + } +} + +#[derive(Debug, Clone)] +pub(super) enum GetlineSrc { + /// Plain `getline`: the main input. + Main, + /// `getline < file` + File(Box), + /// `cmd | getline` + Cmd(Box), +} + +#[derive(Debug, Clone)] +pub(super) enum Expr { + Num(f64), + Str(String), + /// A regex literal in value position: `$0 ~ /re/`. + Regex(u32), + Var(VarRef), + Elem(VarRef, Vec>), + Field(Box), + /// `lv = e` or `lv op= e`. + Assign(Box, Option, Box), + Neg(Box), + Plus(Box), + Not(Box), + Binary(BinOp, Box, Box), + Cmp(CmpOp, Box, Box), + /// `a ~ b` (or `!~` when the flag is set). + Match(bool, Box, Box), + And(Box, Box), + Or(Box, Box), + Cond(Box, Box, Box), + Concat(Box, Box), + /// `(subs) in array` + In(Vec, Box), + /// `++lv`, `lv--`, ...: (lvalue, delta, is_prefix) + IncDec(Box, f64, bool), + Call(u32, Vec), + /// `@name(args)`: function chosen by the value of a variable. + Indirect(Box, Vec), + Builtin(Bi, Vec), + Getline(GetlineSrc, Option>), + /// Parenthesized expression (kept to tell `(a, b) in x` and + /// `print (a) > f` from other forms; evaluated transparently). + Group(Box), + /// `(a, b)` grouping, only valid before `in` or as print arguments. + List(Vec), +} + +impl Expr { + pub(super) fn is_lvalue(&self) -> bool { + matches!(self, Expr::Var(_) | Expr::Elem(..) | Expr::Field(_)) + } +} + +#[derive(Debug, Clone)] +pub(super) enum Redirect { + File(Expr), + Append(Expr), + Pipe(Expr), +} + +#[derive(Debug, Clone)] +pub(super) struct Stmt { + pub(super) kind: StmtKind, + /// Index into `Program::locs`. + pub(super) loc: u32, +} + +pub(super) type Block = Vec; + +#[derive(Debug, Clone)] +pub(super) enum StmtKind { + Expr(Expr), + Print(Vec, Option), + Printf(Vec, Option), + If(Expr, Block, Option), + While(Expr, Block), + Do(Block, Expr), + For(Option>, Option, Option>, Block), + /// `for (lv in array)` + ForIn(Expr, Expr, Block), + Block(Block), + Next, + NextFile, + Exit(Option), + Return(Option), + Break, + Continue, + /// `delete array` or `delete array[subs]` (the expression is the array + /// or element form). + Delete(Expr), + Switch(Expr, Vec<(CaseLabel, Block)>), +} + +#[derive(Debug, Clone)] +pub(super) enum CaseLabel { + Value(Expr), + Regex(u32), + Default, +} + +#[derive(Debug, Clone)] +pub(super) enum Pattern { + All, + Expr(Expr), + /// Range pattern; the index is its slot in the range-state table. + Range(Expr, Expr, usize), +} + +#[derive(Debug, Clone)] +pub(super) struct Rule { + pub(super) pattern: Pattern, + /// `None`: the default action, `print $0`. + pub(super) body: Option, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct Func { + pub(super) name: String, + pub(super) params: Vec, + pub(super) body: Block, + /// Per parameter: used as an array inside the body (directly or by + /// passing it on). An untyped argument for such a parameter becomes an + /// array in the caller. + pub(super) array_params: Vec, + pub(super) defined: bool, +} + +/// Source text of one program piece (`-f file`, `-e text`, the command line). +#[derive(Debug, Clone)] +pub(super) struct Source { + /// Label in messages: `cmd. line` or the file name. + pub(super) name: String, + /// Byte offset of this source in the joined program text. + pub(super) start: usize, +} + +#[derive(Debug, Clone, Default)] +pub(super) struct Program { + pub(super) begin: Vec, + pub(super) end: Vec, + pub(super) beginfile: Vec, + pub(super) endfile: Vec, + pub(super) rules: Vec, + pub(super) funcs: Vec, + /// Global variable names; index = `VarRef::Global` slot. + pub(super) globals: Vec, + /// Regex literal sources; index = `Expr::Regex` id. + pub(super) regexes: Vec, + /// Statement locations: (source label, line number). + pub(super) locs: Vec<(String, u32)>, + pub(super) ranges: usize, +} + +/// Special variables, in global slot order. +pub(super) mod special { + pub(crate) const NAMES: &[&str] = &[ + "NR", + "NF", + "FNR", + "FS", + "OFS", + "ORS", + "RS", + "FILENAME", + "SUBSEP", + "RSTART", + "RLENGTH", + "CONVFMT", + "OFMT", + "ENVIRON", + "ARGC", + "ARGV", + "RT", + "IGNORECASE", + "FIELDWIDTHS", + "FPAT", + "PROCINFO", + "ERRNO", + "SYMTAB", + ]; + pub(crate) const NR: u32 = 0; + pub(crate) const NF: u32 = 1; + pub(crate) const FNR: u32 = 2; + pub(crate) const FS: u32 = 3; + pub(crate) const OFS: u32 = 4; + pub(crate) const ORS: u32 = 5; + pub(crate) const RS: u32 = 6; + pub(crate) const FILENAME: u32 = 7; + pub(crate) const SUBSEP: u32 = 8; + pub(crate) const RSTART: u32 = 9; + pub(crate) const RLENGTH: u32 = 10; + pub(crate) const CONVFMT: u32 = 11; + pub(crate) const OFMT: u32 = 12; + pub(crate) const ENVIRON: u32 = 13; + pub(crate) const ARGC: u32 = 14; + pub(crate) const ARGV: u32 = 15; + pub(crate) const RT: u32 = 16; + pub(crate) const IGNORECASE: u32 = 17; + pub(crate) const FIELDWIDTHS: u32 = 18; + pub(crate) const FPAT: u32 = 19; + pub(crate) const PROCINFO: u32 = 20; + pub(crate) const ERRNO: u32 = 21; + pub(crate) const SYMTAB: u32 = 22; + pub(crate) const COUNT: u32 = 23; +} diff --git a/crates/bashkit/src/builtins/awk/format.rs b/crates/bashkit/src/builtins/awk/format.rs new file mode 100644 index 000000000..d01ec5eae --- /dev/null +++ b/crates/bashkit/src/builtins/awk/format.rs @@ -0,0 +1,594 @@ +//! awk number formatting and `printf`/`sprintf`. +//! +//! Decisions (gawk 5 behavior, C `printf` semantics): +//! - `print`/string conversion of an integral number prints all its digits +//! (`2^70` is `1180591620717411303424`), never through `OFMT`/`CONVFMT`. +//! - Infinities and NaN print as `+inf`, `-inf`, `+nan`, `-nan` in +//! `print`, and as C does (`inf`, `-nan`, ...) through `printf` formats. +//! - `%d` of a value outside the 64-bit range prints its integral digits. +//! - `%c` of a number is the Unicode character with that code (UTF-8 locale). +//! - Width or precision above `MAX_FORMAT_WIDTH` is a fatal error (guard +//! against `%999999999d` allocating gigabytes). + +use super::value::Value; +use crate::builtins::MAX_FORMAT_WIDTH; + +/// Convert a number to its awk string form, using `fmt` (`CONVFMT` or +/// `OFMT`) for non-integral values. +pub(super) fn format_number(n: f64, fmt: &str) -> String { + if n.is_nan() { + return if n.is_sign_negative() { "-nan" } else { "+nan" }.to_string(); + } + if n.is_infinite() { + return if n < 0.0 { "-inf" } else { "+inf" }.to_string(); + } + // Integral values print as integers, so -0 prints as "0" (gawk). + if n == n.trunc() { + if n.abs() < 1e15 { + return (n as i64).to_string(); + } + return format!("{n:.0}"); + } + sprintf(fmt, &[Value::Num(n)], "%.6g").unwrap_or_else(|_| format!("{n}")) +} + +/// Error from `sprintf`: a fatal message (without the `fatal: ` prefix). +/// `located` messages are gawk's own and carry the source location. +#[derive(Debug)] +pub(super) struct FormatError { + pub(super) msg: String, + pub(super) located: bool, +} + +#[derive(Default, Clone, Copy)] +struct Spec { + left: bool, + plus: bool, + space: bool, + alt: bool, + zero: bool, + width: usize, + precision: Option, +} + +/// C-style `sprintf` over awk values. +pub(super) fn sprintf( + fmt: &str, + args: &[Value], + convfmt: &str, +) -> std::result::Result { + let mut out = String::with_capacity(fmt.len()); + let bytes = fmt.as_bytes(); + let mut i = 0; + let mut next_arg = 0; + while i < bytes.len() { + let Some(rel) = fmt[i..].find('%') else { + out.push_str(&fmt[i..]); + break; + }; + out.push_str(&fmt[i..i + rel]); + let start = i + rel; + i = start + 1; + if i >= bytes.len() { + out.push('%'); + break; + } + if bytes[i] == b'%' { + out.push('%'); + i += 1; + continue; + } + // Positional argument: %N$ + let mut positional = None; + { + let mut j = i; + while j < bytes.len() && bytes[j].is_ascii_digit() { + j += 1; + } + if j > i && j < bytes.len() && bytes[j] == b'$' { + let n: usize = fmt[i..j].parse().unwrap_or(0); + if n == 0 { + return Err(FormatError { + msg: "positional argument zero is not allowed".to_string(), + located: true, + }); + } + positional = Some(n - 1); + i = j + 1; + } + } + let mut spec = Spec::default(); + while i < bytes.len() { + match bytes[i] { + b'-' => spec.left = true, + b'+' => spec.plus = true, + b' ' => spec.space = true, + b'#' => spec.alt = true, + b'0' => spec.zero = true, + b'\'' => {} + _ => break, + } + i += 1; + } + let mut take_arg = |positional: Option| -> Option<&Value> { + match positional { + Some(p) => args.get(p), + None => { + let v = args.get(next_arg); + next_arg += 1; + v + } + } + }; + // Width. + if i < bytes.len() && bytes[i] == b'*' { + i += 1; + let Some(v) = take_arg(None) else { + return Err(not_enough(fmt, start)); + }; + let w = v.to_num(); + if w < 0.0 { + spec.left = true; + } + spec.width = clamp(w.abs()); + check_cap("width", spec.width)?; + } else { + let s = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + if i > s { + spec.width = fmt[s..i].parse::().unwrap_or(usize::MAX); + check_cap("width", spec.width)?; + } + } + // Precision. + if i < bytes.len() && bytes[i] == b'.' { + i += 1; + if i < bytes.len() && bytes[i] == b'*' { + i += 1; + let Some(v) = take_arg(None) else { + return Err(not_enough(fmt, start)); + }; + let p = v.to_num(); + spec.precision = if p < 0.0 { None } else { Some(clamp(p)) }; + if let Some(p) = spec.precision { + check_cap("precision", p)?; + } + } else { + let s = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + let p = fmt[s..i].parse::().unwrap_or(usize::MAX); + check_cap("precision", p)?; + spec.precision = Some(p); + } + } + // Length modifiers are accepted and ignored. + while i < bytes.len() && matches!(bytes[i], b'h' | b'l' | b'L' | b'q' | b'j' | b'z' | b't') + { + i += 1; + } + let Some(conv) = fmt[i..].chars().next() else { + // Incomplete spec at end: print it literally. + out.push_str(&fmt[start..]); + break; + }; + i += conv.len_utf8(); + match conv { + 'd' | 'i' | 'o' | 'x' | 'X' | 'u' | 'c' | 's' | 'e' | 'E' | 'f' | 'F' | 'g' | 'G' + | 'a' | 'A' => {} + _ => { + // Unknown conversion: gawk copies it through as text. + out.push_str(&fmt[start..i]); + continue; + } + } + let Some(arg) = take_arg(positional) else { + return Err(not_enough(fmt, start)); + }; + match conv { + 'c' => { + let s = match arg { + Value::Num(n) | Value::StrNum(_, n) => { + let code = if n.is_finite() { *n as i64 } else { 0 }; + match u32::try_from(code).ok().and_then(char::from_u32) { + Some(c) => c.to_string(), + None => char::from((code & 0xff) as u8).to_string(), + } + } + Value::Uninit => "\0".to_string(), + Value::Str(s) => s.chars().next().map(String::from).unwrap_or_default(), + }; + pad(&mut out, &s, &spec, false); + } + 's' => { + let s = arg.to_str_fmt(convfmt); + let s = match spec.precision { + Some(p) => match s.char_indices().nth(p) { + Some((idx, _)) => &s[..idx], + None => &s[..], + }, + None => &s[..], + }; + pad(&mut out, s, &spec, false); + } + 'd' | 'i' => { + let n = arg.to_num(); + if !n.is_finite() { + let s = special(n, false, &spec); + pad(&mut out, &s, &spec, false); + continue; + } + let t = n.trunc(); + let digits = if t.abs() < 9.2e18 { + (t.abs() as u64).to_string() + } else { + format!("{:.0}", t.abs()) + }; + let sign = if t < 0.0 { + "-" + } else if spec.plus { + "+" + } else if spec.space { + " " + } else { + "" + }; + int_out(&mut out, sign, "", &digits, &spec); + } + 'o' | 'x' | 'X' | 'u' => { + let n = arg.to_num(); + if !n.is_finite() { + let s = special(n, false, &spec); + pad(&mut out, &s, &spec, false); + continue; + } + let t = n.trunc(); + let u: Option = if t < 0.0 { + if t >= -9.2e18 { + Some(t as i64 as u64) + } else { + None + } + } else if t < 1.8e19 { + Some(t as u64) + } else { + None + }; + let Some(u) = u else { + // Out of range: gawk falls back to a float format. + let s = fmt_g(n, 6, false, false); + pad(&mut out, &s, &spec, false); + continue; + }; + let (digits, prefix) = match conv { + 'o' => { + let d = format!("{u:o}"); + let p = if spec.alt && !d.starts_with('0') { + "0" + } else { + "" + }; + (d, p) + } + 'x' => (format!("{u:x}"), if spec.alt && u != 0 { "0x" } else { "" }), + 'X' => (format!("{u:X}"), if spec.alt && u != 0 { "0X" } else { "" }), + _ => (u.to_string(), ""), + }; + int_out(&mut out, "", prefix, &digits, &spec); + } + _ => { + let n = arg.to_num(); + let upper = conv.is_ascii_uppercase(); + if !n.is_finite() { + let s = special(n, upper, &spec); + pad(&mut out, &s, &spec, false); + continue; + } + let prec = spec.precision.unwrap_or(6); + let body = match conv.to_ascii_lowercase() { + 'f' => { + let mut s = format!("{:.*}", prec, n.abs()); + if spec.alt && prec == 0 { + s.push('.'); + } + s + } + 'e' => fmt_e(n.abs(), prec, spec.alt, upper), + 'a' => fmt_e(n.abs(), prec, spec.alt, upper), + _ => fmt_g(n.abs(), prec, spec.alt, upper), + }; + let sign = if n.is_sign_negative() && !(n == 0.0 && body_is_zero(&body)) { + "-" + } else if spec.plus { + "+" + } else if spec.space { + " " + } else { + "" + }; + // C prints -0.0 as "-0.000000"; keep that. + let sign = if n == 0.0 && n.is_sign_negative() { + "-" + } else { + sign + }; + float_out(&mut out, sign, &body, &spec); + } + } + } + Ok(out) +} + +fn body_is_zero(body: &str) -> bool { + body.bytes().all(|b| matches!(b, b'0' | b'.')) +} + +fn clamp(v: f64) -> usize { + if v.is_finite() && v < usize::MAX as f64 { + v as usize + } else { + usize::MAX + } +} + +/// THREAT[TM-DOS-110]: refuse widths/precisions above `MAX_FORMAT_WIDTH`. +fn check_cap(what: &str, v: usize) -> std::result::Result<(), FormatError> { + if v > MAX_FORMAT_WIDTH { + let shown = if v == usize::MAX { + "too large".to_string() + } else { + v.to_string() + }; + return Err(FormatError { + msg: format!("format {what} {shown} exceeds maximum ({MAX_FORMAT_WIDTH})"), + located: false, + }); + } + Ok(()) +} + +fn not_enough(fmt: &str, at: usize) -> FormatError { + FormatError { + msg: format!( + "not enough arguments to satisfy format string\n\t`{fmt}'\n\t{}^ ran out for this one", + " ".repeat(at + 1) + ), + located: true, + } +} + +/// `inf`/`nan` through a numeric conversion. +fn special(n: f64, upper: bool, spec: &Spec) -> String { + let word = if n.is_nan() { "nan" } else { "inf" }; + let word = if upper { + word.to_ascii_uppercase() + } else { + word.to_string() + }; + let sign = if n.is_sign_negative() { + "-" + } else if spec.plus { + "+" + } else if spec.space { + " " + } else { + "" + }; + format!("{sign}{word}") +} + +/// Pad `s` to the spec width (spaces; zero padding is numeric only). +fn pad(out: &mut String, s: &str, spec: &Spec, _numeric: bool) { + let len = s.chars().count(); + if len >= spec.width { + out.push_str(s); + return; + } + let fill = spec.width - len; + if spec.left { + out.push_str(s); + out.extend(std::iter::repeat_n(' ', fill)); + } else { + out.extend(std::iter::repeat_n(' ', fill)); + out.push_str(s); + } +} + +fn int_out(out: &mut String, sign: &str, prefix: &str, digits: &str, spec: &Spec) { + let mut digits = digits.to_string(); + if let Some(p) = spec.precision { + if p == 0 && digits == "0" { + digits.clear(); + } + if digits.len() < p { + digits = "0".repeat(p - digits.len()) + &digits; + } + } + let len = sign.len() + prefix.len() + digits.len(); + if spec.zero && !spec.left && spec.precision.is_none() && len < spec.width { + out.push_str(sign); + out.push_str(prefix); + out.extend(std::iter::repeat_n('0', spec.width - len)); + out.push_str(&digits); + return; + } + let s = format!("{sign}{prefix}{digits}"); + pad(out, &s, spec, true); +} + +fn float_out(out: &mut String, sign: &str, body: &str, spec: &Spec) { + let len = sign.len() + body.len(); + if spec.zero && !spec.left && len < spec.width { + out.push_str(sign); + out.extend(std::iter::repeat_n('0', spec.width - len)); + out.push_str(body); + return; + } + let s = format!("{sign}{body}"); + pad(out, &s, spec, true); +} + +/// `%e` body for a non-negative finite value. +fn fmt_e(n: f64, prec: usize, alt: bool, upper: bool) -> String { + let s = format!("{:.*e}", prec, n); + let (mant, exp) = s.split_once('e').unwrap_or((&s, "0")); + let exp: i32 = exp.parse().unwrap_or(0); + let mut mant = mant.to_string(); + if alt && prec == 0 { + mant.push('.'); + } + let e = if upper { 'E' } else { 'e' }; + let sign = if exp < 0 { '-' } else { '+' }; + format!("{mant}{e}{sign}{:02}", exp.abs()) +} + +/// `%g` body for a non-negative finite value. +fn fmt_g(n: f64, prec: usize, alt: bool, upper: bool) -> String { + let p = if prec == 0 { 1 } else { prec }; + if n == 0.0 { + let mut s = "0".to_string(); + if alt { + s.push('.'); + s.push_str(&"0".repeat(p - 1)); + } + return s; + } + // Exponent after rounding to p significant digits. + let e_repr = format!("{:.*e}", p - 1, n); + let x: i32 = e_repr + .split_once('e') + .and_then(|(_, e)| e.parse().ok()) + .unwrap_or(0); + let mut s = if x < -4 || x >= p as i32 { + fmt_e(n, p - 1, alt, upper) + } else { + let decimals = (p as i32 - 1 - x).max(0) as usize; + let mut s = format!("{:.*}", decimals, n); + if alt && !s.contains('.') { + s.push('.'); + } + s + }; + if !alt { + // Strip trailing zeros of the fraction (before any exponent). + let (mant, exp) = match s.find(['e', 'E']) { + Some(pos) => (s[..pos].to_string(), s[pos..].to_string()), + None => (s.clone(), String::new()), + }; + if mant.contains('.') { + let m = mant.trim_end_matches('0').trim_end_matches('.'); + s = format!("{m}{exp}"); + } + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + fn f(fmt: &str, args: &[Value]) -> String { + sprintf(fmt, args, "%.6g").unwrap() + } + fn n(x: f64) -> Value { + Value::Num(x) + } + fn s(x: &str) -> Value { + Value::Str(x.to_string()) + } + + #[test] + fn number_to_string() { + assert_eq!(format_number(3.0, "%.6g"), "3"); + assert_eq!(format_number(1.23456789, "%.6g"), "1.23457"); + assert_eq!(format_number(1e16, "%.6g"), "10000000000000000"); + assert_eq!( + format_number(2f64.powi(70), "%.6g"), + "1180591620717411303424" + ); + assert_eq!(format_number(0.1, "%.2f"), "0.10"); + assert_eq!(format_number(1e-5, "%.6g"), "1e-05"); + assert_eq!(format_number(f64::INFINITY, "%.6g"), "+inf"); + assert_eq!(format_number(-f64::NAN, "%.6g"), "-nan"); + assert_eq!(format_number(-0.0, "%.6g"), "0"); + } + + #[test] + fn conversions() { + assert_eq!(f("%c %c", &[n(65.0), s("hello")]), "A h"); + assert_eq!(f("%c%c", &[n(225.0), n(8364.0)]), "á€"); + assert_eq!(f("%d %x %X", &[n(10.0), n(255.0), n(255.0)]), "10 ff FF"); + assert_eq!(f("%e", &[n(1234.5)]), "1.234500e+03"); + assert_eq!(f("%E", &[n(0.000123)]), "1.230000E-04"); + assert_eq!(f("%g %g", &[n(1234567.0), n(0.0001)]), "1.23457e+06 0.0001"); + assert_eq!( + f( + "%g %g %g %g %.3g", + &[n(1e5), n(1e6), n(1e-4), n(1e-5), n(1.23456)] + ), + "100000 1e+06 0.0001 1e-05 1.23" + ); + assert_eq!(f("%5.1f|%-5s|", &[n(1.23456), s("ab")]), " 1.2|ab |"); + } + + #[test] + fn flags_and_star() { + assert_eq!( + f( + "%05d|%+d|% d|%05.1f|%-5d|%#o|%#x", + &[n(42.0), n(42.0), n(42.0), n(1.23), n(7.0), n(8.0), n(255.0)] + ), + "00042|+42| 42|001.2|7 |010|0xff" + ); + assert_eq!( + f( + "%*d|%-*s|%.*f", + &[n(5.0), n(42.0), n(4.0), s("ab"), n(2.0), n(1.23456)] + ), + " 42|ab |1.23" + ); + assert_eq!(f("%2$s %1$s", &[s("world"), s("hello")]), "hello world"); + } + + #[test] + fn integers() { + assert_eq!(f("%d", &[n(2f64.powi(64))]), "18446744073709551616"); + assert_eq!( + f("%d", &[n(-9223372036854775808.0)]), + "-9223372036854775808" + ); + assert_eq!(f("%d %d", &[s("12abc"), s("abc")]), "12 0"); + assert_eq!(f("%x", &[n(-1.0)]), "ffffffffffffffff"); + assert_eq!(f("%.3d", &[n(7.0)]), "007"); + assert_eq!(f("%d", &[n(-3.9)]), "-3"); + } + + #[test] + fn not_enough_args() { + let e = sprintf("%s %s\n", &[s("a")], "%.6g").unwrap_err(); + assert!(e.msg.contains("not enough arguments"), "{}", e.msg); + assert!( + e.msg.ends_with("\t ^ ran out for this one"), + "{:?}", // debug-ok: assert-failure message + e.msg + ); + } + + #[test] + fn special_values() { + assert_eq!(f("%f %d", &[n(f64::INFINITY), n(-f64::NAN)]), "inf -nan"); + assert_eq!(f("%5.1f%%", &[n(99.25)]), " 99.2%"); + } + + #[test] + fn huge_width_is_an_error() { + let e = sprintf("%10001s", &[s("x")], "%.6g").unwrap_err(); + assert_eq!(e.msg, "format width 10001 exceeds maximum (10000)"); + let e = sprintf("%.10001s", &[s("x")], "%.6g").unwrap_err(); + assert_eq!(e.msg, "format precision 10001 exceeds maximum (10000)"); + assert_eq!(f("%10000s", &[s("x")]).len(), 10000); + } +} diff --git a/crates/bashkit/src/builtins/awk/funcs.rs b/crates/bashkit/src/builtins/awk/funcs.rs new file mode 100644 index 000000000..b732e17aa --- /dev/null +++ b/crates/bashkit/src/builtins/awk/funcs.rs @@ -0,0 +1,883 @@ +//! awk builtin functions. +//! +//! Decisions: +//! - String positions and lengths count characters (gawk in a UTF-8 +//! locale), not bytes. +//! - `substr` rounds start and length like POSIX/gawk. +//! - `sub`/`gsub` replacement follows POSIX 2008 (gawk): `&` is the match, +//! `\&` a literal `&`, `\\` a backslash. `gensub` adds `\0`..`\9`. +//! - Empty matches in `gsub` follow gawk: an empty match right after a +//! previous match is skipped (`gsub(/b*/, "-", "abc")` is `-a-c-`). +//! - `rand()` reproduces gawk's generator (BSD `random()` with a 256-byte +//! state seeded with 1, two draws per number), so seeded and unseeded +//! sequences match gawk. +//! - Time functions use the sandbox clock and `TZ` (TM-INF-018). + +use std::sync::Arc; + +use super::ast::{Bi, Expr, VarRef, special}; +use super::format::sprintf; +use super::interp::{ + Cell, Elem, Interp, Loc, R, Splitter, bool_val, fix_nan, next_char, split_text, +}; +use super::regex::AwkRegex; +use super::value::{Value, str_to_num}; + +const RAND_DEG: usize = 63; +const RAND_SEP: usize = 1; + +/// gawk's `random()` (BSD, TYPE_4 state). +pub(super) struct Rand { + state: [u32; RAND_DEG], + f: usize, + r: usize, + pub(super) seed: f64, +} + +impl Rand { + pub(super) fn new() -> Self { + let mut r = Rand { + state: [0; RAND_DEG], + f: RAND_SEP, + r: 0, + seed: 0.0, + }; + r.srandom(1); + r + } + + fn good_rand(x: i32) -> i32 { + let mut x = x as i64; + if x == 0 { + x = 123_459_876; + } + let hi = x / 127_773; + let lo = x % 127_773; + x = 16_807 * lo - 2_836 * hi; + if x < 0 { + x += 0x7fff_ffff; + } + x as i32 + } + + pub(super) fn srandom(&mut self, x: u32) { + self.state[0] = x; + for i in 1..RAND_DEG { + self.state[i] = Self::good_rand(self.state[i - 1] as i32) as u32; + } + self.f = RAND_SEP; + self.r = 0; + for _ in 0..10 * RAND_DEG { + self.random(); + } + } + + fn random(&mut self) -> u32 { + self.state[self.f] = self.state[self.f].wrapping_add(self.state[self.r]); + let v = (self.state[self.f] >> 1) & 0x7fff_ffff; + self.f += 1; + if self.f >= RAND_DEG { + self.f = 0; + self.r += 1; + } else { + self.r += 1; + if self.r >= RAND_DEG { + self.r = 0; + } + } + v + } + + pub(super) fn rand(&mut self) -> f64 { + const DIV: f64 = 2_147_483_648.0; + loop { + let d1 = self.random() as f64; + let d2 = self.random() as f64; + let v = 0.5 + ((d1 / DIV + d2) / DIV) - 0.5; + if v < 1.0 { + return v; + } + } + } +} + +enum Which { + All, + Nth(usize), +} + +impl Interp { + async fn arg_val(&mut self, args: &[Expr], i: usize) -> R { + match args.get(i) { + Some(e) => self.eval(e).await, + None => Ok(Value::Uninit), + } + } + + async fn arg_str(&mut self, args: &[Expr], i: usize) -> R { + let v = self.arg_val(args, i).await?; + Ok(self.to_str(&v)) + } + + async fn arg_num(&mut self, args: &[Expr], i: usize) -> R { + Ok(self.arg_val(args, i).await?.to_num()) + } + + pub(super) async fn call_builtin(&mut self, bi: Bi, args: &[Expr]) -> R { + let num = |n: f64| Ok(Value::Num(n)); + match bi { + Bi::Length => self.bi_length(args).await, + Bi::Substr => { + let s = self.arg_str(args, 0).await?; + let m = self.arg_num(args, 1).await?; + let n = if args.len() > 2 { + Some(self.arg_num(args, 2).await?) + } else { + None + }; + Ok(Value::Str(substr(&s, m, n))) + } + Bi::Index => { + let s = self.arg_str(args, 0).await?; + let t = self.arg_str(args, 1).await?; + let (s, t) = if self.ignorecase { + (s.to_lowercase(), t.to_lowercase()) + } else { + (s, t) + }; + num(match s.find(&t) { + Some(b) => (s[..b].chars().count() + 1) as f64, + None => 0.0, + }) + } + Bi::Split | Bi::Patsplit => self.bi_split(bi, args).await, + Bi::Sub | Bi::Gsub => { + let re = self.regex_operand(&args[0]).await?; + let repl = self.arg_str(args, 1).await?; + let loc = match args.get(2) { + Some(t) if is_lvalue(t) => Some(self.resolve(t).await?), + Some(_) => None, + None => Some(Loc::Field(0)), + }; + let target = match (&loc, args.get(2)) { + (Some(l), _) => self.load(l)?, + (None, Some(e)) => self.eval(e).await?, + (None, None) => Value::Uninit, + }; + let target = self.to_str(&target); + let which = if bi == Bi::Sub { + Which::Nth(1) + } else { + Which::All + }; + let (out, n) = self.substitute(&re, &target, which, false, |s, caps, out| { + let (a, b) = caps[0].unwrap_or((0, 0)); + expand_sub(&repl, &s[a..b], out); + })?; + if n > 0 + && let Some(l) = loc + { + self.store(l, Value::Str(out))?; + } + num(n as f64) + } + Bi::Gensub => { + let re = self.regex_operand(&args[0]).await?; + let repl = self.arg_str(args, 1).await?; + let how = self.arg_val(args, 2).await?; + let target = if args.len() > 3 { + self.arg_str(args, 3).await? + } else { + let v = self.get_field(0)?; + self.to_str(&v) + }; + let hs = self.to_str(&how); + let which = if hs.starts_with('g') || hs.starts_with('G') { + Which::All + } else { + let n = how.to_num(); + Which::Nth(if n >= 1.0 { n as usize } else { 1 }) + }; + let (out, _) = self.substitute(&re, &target, which, true, |s, caps, out| { + expand_gensub(&repl, s, caps, out); + })?; + Ok(Value::Str(out)) + } + Bi::Match => self.bi_match(args).await, + Bi::Sprintf => { + let mut vals = Vec::with_capacity(args.len()); + for a in args { + vals.push(self.eval(a).await?); + } + let fmt = self.to_str(&vals[0]); + match sprintf(&fmt, &vals[1..], &self.convfmt) { + Ok(s) => { + self.string_fits(s.len())?; + Ok(Value::Str(s)) + } + Err(e) if e.located => Err(self.fatal_at(&e.msg)), + Err(e) => Err(self.fatal(&e.msg)), + } + } + Bi::Sin => { + let x = self.arg_num(args, 0).await?; + num(fix_nan(x.sin(), &[x])) + } + Bi::Cos => { + let x = self.arg_num(args, 0).await?; + num(fix_nan(x.cos(), &[x])) + } + Bi::Atan2 => { + let y = self.arg_num(args, 0).await?; + let x = self.arg_num(args, 1).await?; + num(y.atan2(x)) + } + Bi::Exp => { + let x = self.arg_num(args, 0).await?; + num(x.exp()) + } + Bi::Log => { + let x = self.arg_num(args, 0).await?; + num(fix_nan(x.ln(), &[x])) + } + Bi::Sqrt => { + let x = self.arg_num(args, 0).await?; + num(fix_nan(x.sqrt(), &[x])) + } + Bi::Int => { + let x = self.arg_num(args, 0).await?; + num(x.trunc()) + } + Bi::Rand => num(self.rand.rand()), + Bi::Srand => { + let prev = self.rand.seed; + let seed = if args.is_empty() { + self.io.host.clock.now_epoch().0 as f64 + } else { + self.arg_num(args, 0).await? + }; + self.rand.seed = seed; + self.rand.srandom(seed as i64 as u32); + num(prev) + } + Bi::Tolower | Bi::Toupper => { + let s = self.arg_str(args, 0).await?; + let r = if bi == Bi::Tolower { + s.to_lowercase() + } else { + s.to_uppercase() + }; + self.string_fits(r.len())?; + Ok(Value::Str(r)) + } + Bi::System => { + let cmd = self.arg_str(args, 0).await?; + let r = self.run_command(false, &cmd, None).await?; + num(r.exit_code as f64) + } + Bi::Close => { + let name = self.arg_str(args, 0).await?; + let r = self.close_stream(&name).await?; + num(r) + } + Bi::Fflush => { + self.flush_all().await?; + num(0.0) + } + Bi::Strtonum => { + let v = self.arg_val(args, 0).await?; + num(match v { + Value::Num(n) => n, + other => strtonum(&self.to_str(&other)), + }) + } + Bi::Systime => num(self.io.host.clock.now_epoch().0 as f64), + Bi::Strftime => self.bi_strftime(args).await, + Bi::Mktime => { + let spec = self.arg_str(args, 0).await?; + let utc = if args.len() > 1 { + self.arg_val(args, 1).await?.to_bool() + } else { + false + }; + num(self.mktime(&spec, utc)) + } + Bi::Asort | Bi::Asorti => self.bi_asort(bi, args).await, + Bi::Typeof => { + let t = self.type_of(&args[0]).await?; + Ok(Value::Str(t.to_string())) + } + Bi::Isarray => { + let t = self.type_of(&args[0]).await?; + Ok(bool_val(t == "array")) + } + Bi::And | Bi::Or | Bi::Xor => { + let mut acc = to_bits(self.arg_num(args, 0).await?); + for i in 1..args.len() { + let v = to_bits(self.arg_num(args, i).await?); + acc = match bi { + Bi::And => acc & v, + Bi::Or => acc | v, + _ => acc ^ v, + }; + } + num(acc as f64) + } + Bi::Lshift | Bi::Rshift => { + let v = to_bits(self.arg_num(args, 0).await?); + let n = to_bits(self.arg_num(args, 1).await?).min(63) as u32; + num(if bi == Bi::Lshift { + v.wrapping_shl(n) + } else { + v >> n + } as f64) + } + Bi::Compl => { + let v = to_bits(self.arg_num(args, 0).await?); + num((!v & ((1u64 << 53) - 1)) as f64) + } + } + } + + async fn bi_length(&mut self, args: &[Expr]) -> R { + let n = match args.first() { + None => { + let v = self.get_field(0)?; + self.to_str(&v).chars().count() + } + Some(Expr::Var(v)) if *v != VarRef::Global(special::NF) => { + match &self.globals_or_local(*v) { + Some(Cell::Arr(id)) => self.array(*id).map_or(0, |a| a.len()), + _ => { + let v = self.get_var(*v)?; + self.to_str(&v).chars().count() + } + } + } + Some(e @ Expr::Elem(..)) => { + let loc = self.resolve(e).await?; + if let Loc::Elem(id, key) = &loc + && let Some(Elem::Arr(sub)) = self.array(*id).and_then(|a| a.get(key)) + { + self.array(*sub).map_or(0, |a| a.len()) + } else { + let v = self.load(&loc)?; + self.to_str(&v).chars().count() + } + } + Some(e) => { + let v = self.eval(e).await?; + self.to_str(&v).chars().count() + } + }; + Ok(Value::Num(n as f64)) + } + + fn globals_or_local(&self, v: VarRef) -> Option { + // Small helper so `length(arr)` can look without converting. + Some(self.cell(v).clone()) + } + + async fn bi_split(&mut self, bi: Bi, args: &[Expr]) -> R { + let s = self.arg_str(args, 0).await?; + let splitter = match args.get(2) { + None if bi == Bi::Patsplit => match &self.splitter { + Splitter::Fpat(re) => Splitter::Fpat(re.clone()), + _ => { + let fpat = match &self.globals[special::FPAT as usize] { + Cell::Val(v) => self.to_str(v), + Cell::Arr(_) => String::new(), + }; + Splitter::Fpat(self.dyn_regex(&fpat)?) + } + }, + None => self.splitter.clone(), + Some(e) => { + let re_lit = matches!(e, Expr::Regex(_)); + if bi == Bi::Patsplit { + Splitter::Fpat(self.regex_operand(e).await?) + } else if re_lit { + Splitter::Regex(self.regex_operand(e).await?) + } else { + let sep = self.arg_str(args, 2).await?; + self.classify_sep(&sep)? + } + } + }; + let mut seps = Vec::new(); + let max = self.split_budget(); + let parts = + split_text(&splitter, &s, Some(&mut seps), max).map_err(|e| self.too_many(e))?; + let n = parts.len(); + let arr = self.array_of_expr(&args[1]).await?; + self.fill_array(arr, parts.into_iter().map(Value::from_input).collect())?; + if let Some(sa) = args.get(3) { + let sid = self.array_of_expr(sa).await?; + // Default splitting: seps[0] is the leading blank run, and the + // leading and trailing entries exist only when not empty. + let (first, mut seps) = if matches!(splitter, Splitter::Space) { + (0, seps) + } else { + (1, seps) + }; + let lead_empty = first == 0 && seps.first().is_some_and(String::is_empty); + if first == 0 && seps.len() > n && seps.last().is_some_and(String::is_empty) { + seps.pop(); + } + self.fill_array_from(sid, first, seps.into_iter().map(Value::Str).collect())?; + if lead_empty { + self.elem_delete(sid, "0"); + } + } + Ok(Value::Num(n as f64)) + } + + async fn bi_match(&mut self, args: &[Expr]) -> R { + let s = self.arg_str(args, 0).await?; + let re = self.regex_operand(&args[1]).await?; + let caps = if args.len() > 2 { + re.captures_at(&s, 0) + } else { + re.find_at(&s, 0).map(|m| vec![Some(m)]) + }; + let (rstart, rlength) = match &caps { + Some(c) => { + let (a, b) = c[0].unwrap_or((0, 0)); + ( + (s[..a].chars().count() + 1) as f64, + s[a..b].chars().count() as f64, + ) + } + None => (0.0, -1.0), + }; + self.set_var(VarRef::Global(special::RSTART), Value::Num(rstart))?; + self.set_var(VarRef::Global(special::RLENGTH), Value::Num(rlength))?; + if let Some(arr_e) = args.get(2) { + let arr = self.array_of_expr(arr_e).await?; + self.clear_array(arr); + if let Some(c) = caps { + let subsep = self.subsep.clone(); + for (i, g) in c.iter().enumerate() { + if let Some((a, b)) = g { + let text = s[*a..*b].to_string(); + self.elem_store(arr, i.to_string(), Elem::Val(Value::from_input(text))); + let start = (s[..*a].chars().count() + 1) as f64; + let len = s[*a..*b].chars().count() as f64; + self.elem_store( + arr, + format!("{i}{subsep}start"), + Elem::Val(Value::Num(start)), + ); + self.elem_store( + arr, + format!("{i}{subsep}length"), + Elem::Val(Value::Num(len)), + ); + } + } + } + } + Ok(Value::Num(rstart)) + } + + async fn bi_strftime(&mut self, args: &[Expr]) -> R { + let fmt = if args.is_empty() { + self.procinfo_str("strftime") + .unwrap_or_else(|| "%a %b %e %H:%M:%S %Z %Y".to_string()) + } else { + self.arg_str(args, 0).await? + }; + let ts = if args.len() > 1 { + Some(self.arg_num(args, 1).await? as i64) + } else { + None + }; + let utc = if args.len() > 2 { + self.arg_val(args, 2).await?.to_bool() + } else { + false + }; + let (tz, fmt) = if utc { + // gmtime() names its zone "GMT". + (Some("UTC".to_string()), utc_zone_name(&fmt)) + } else { + (self.env_var("TZ"), fmt) + }; + let clock = self.io.host.clock; + let out = clock.strftime(tz.as_ref(), ts, &fmt).unwrap_or_default(); + self.string_fits(out.len())?; + Ok(Value::Str(out)) + } + + fn mktime(&self, spec: &str, utc: bool) -> f64 { + let parts: Vec = spec + .split_whitespace() + .map_while(|p| p.parse::().ok()) + .collect(); + if parts.len() < 6 || spec.split_whitespace().count() > 7 { + return -1.0; + } + let tz = if utc { + Some("UTC".to_string()) + } else { + self.env_var("TZ") + }; + let clock = self.io.host.clock; + clock + .mktime( + tz.as_ref(), + [parts[0], parts[1], parts[2], parts[3], parts[4], parts[5]], + ) + .map_or(-1.0, |t| t as f64) + } + + fn env_var(&self, name: &str) -> Option { + self.io + .host + .env + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.clone()) + } + + fn procinfo_str(&self, key: &str) -> Option { + let Cell::Arr(id) = self.globals[special::PROCINFO as usize] else { + return None; + }; + match self.array(id)?.get(key)? { + Elem::Val(v) => Some(self.to_str(v)), + Elem::Arr(_) => None, + } + } + + async fn bi_asort(&mut self, bi: Bi, args: &[Expr]) -> R { + let src = self.array_of_expr(&args[0]).await?; + let how = if args.len() > 2 { + self.arg_str(args, 2).await? + } else { + String::new() + }; + let mut items: Vec = match self.array(src) { + Some(a) => a + .iter() + .map(|(k, e)| { + if bi == Bi::Asorti { + Value::Str(k.clone()) + } else { + match e { + Elem::Val(v) => v.clone(), + Elem::Arr(_) => Value::Uninit, + } + } + }) + .collect(), + None => Vec::new(), + }; + let convfmt = self.convfmt.clone(); + let desc = how.ends_with("_desc"); + let numeric_keys = how.contains("_num_"); + items.sort_by(|a, b| { + let o = if bi == Bi::Asorti && !numeric_keys { + a.to_str_fmt(&convfmt).cmp(&b.to_str_fmt(&convfmt)) + } else if bi == Bi::Asorti { + str_to_num(&a.to_str_fmt(&convfmt)).total_cmp(&str_to_num(&b.to_str_fmt(&convfmt))) + } else if how.contains("_str_") { + a.to_str_fmt(&convfmt).cmp(&b.to_str_fmt(&convfmt)) + } else { + // Default: numbers before strings, each in its own order. + match (a.is_numeric(), b.is_numeric()) { + (true, true) => a.to_num().total_cmp(&b.to_num()), + (true, false) => std::cmp::Ordering::Less, + (false, true) => std::cmp::Ordering::Greater, + (false, false) => a.to_str_fmt(&convfmt).cmp(&b.to_str_fmt(&convfmt)), + } + }; + if desc { o.reverse() } else { o } + }); + let n = items.len(); + let dest = match args.get(1) { + Some(d) => self.array_of_expr(d).await?, + None => src, + }; + self.fill_array(dest, items)?; + Ok(Value::Num(n as f64)) + } + + async fn type_of(&mut self, e: &Expr) -> R<&'static str> { + Ok(match e { + Expr::Var(v) if *v == VarRef::Global(special::NF) => "number", + Expr::Var(v) => match self.globals_or_local(*v) { + Some(Cell::Arr(_)) => "array", + Some(Cell::Val(x)) => x.type_name(), + None => "untyped", + }, + Expr::Elem(v, groups) => { + let id = match self.globals_or_local(*v) { + Some(Cell::Arr(id)) => id, + _ => return Ok("untyped"), + }; + let mut id = id; + let (last, path) = groups.split_last().ok_or(super::interp::Unwind::Fatal)?; + for g in path { + let k = self.subscript(g).await?; + match self.array(id).and_then(|a| a.get(&k)) { + Some(Elem::Arr(sub)) => id = *sub, + _ => return Ok("untyped"), + } + } + let key = self.subscript(last).await?; + match self.array(id).and_then(|a| a.get(&key)) { + Some(Elem::Arr(_)) => "array", + Some(Elem::Val(Value::Uninit)) => "untyped", + Some(Elem::Val(x)) => x.type_name(), + None => "untyped", + } + } + Expr::Regex(_) => "regexp", + other => { + let v = self.eval(other).await?; + v.type_name() + } + }) + } + + /// Core of sub/gsub/gensub with gawk's empty-match rules and the + /// string cap checked as the result grows. + fn substitute( + &mut self, + re: &Arc, + s: &str, + which: Which, + need_caps: bool, + mut rep: impl FnMut(&str, &[Option<(usize, usize)>], &mut String), + ) -> R<(String, usize)> { + let mut out = String::new(); + let mut pos = 0; + let mut copied = 0; + let mut count = 0; + let mut replaced = 0; + let mut last_end: Option = None; + while pos <= s.len() { + let caps = if need_caps { + re.captures_at(s, pos) + } else { + re.find_at(s, pos).map(|m| vec![Some(m)]) + }; + let Some(caps) = caps else { break }; + let Some((ms, me)) = caps[0] else { break }; + if ms == me && last_end == Some(ms) { + pos = next_char(s, ms); + continue; + } + count += 1; + let hit = match which { + Which::All => true, + Which::Nth(n) => count == n, + }; + if hit { + out.push_str(&s[copied..ms]); + rep(s, &caps, &mut out); + copied = me; + replaced += 1; + self.string_fits(out.len())?; + } + if ms == me { + pos = next_char(s, ms); + } else { + pos = me; + last_end = Some(me); + } + if let Which::Nth(n) = which + && count >= n + { + break; + } + } + if copied < s.len() { + self.string_fits(out.len() + s.len() - copied)?; + out.push_str(&s[copied..]); + } + Ok((out, replaced)) + } +} + +fn is_lvalue(e: &Expr) -> bool { + match e { + Expr::Group(inner) => is_lvalue(inner), + other => other.is_lvalue(), + } +} + +/// `substr(s, m[, n])` with POSIX rounding, counting characters. +/// Replace `%Z` with `GMT` in a strftime format (`%%` stays as is). +fn utc_zone_name(fmt: &str) -> String { + let mut out = String::with_capacity(fmt.len()); + let mut chars = fmt.chars(); + while let Some(c) = chars.next() { + if c != '%' { + out.push(c); + continue; + } + match chars.next() { + Some('Z') => out.push_str("GMT"), + Some(n) => { + out.push('%'); + out.push(n); + } + None => out.push('%'), + } + } + out +} + +/// gawk's `substr`: a start below 1 counts from 1 without shortening +/// the length; fractional start and length are truncated; a length below +/// 1 (or NaN) gives "". +fn substr(s: &str, m: f64, n: Option) -> String { + let length = match n { + Some(n) if n.is_nan() || n < 1.0 => return String::new(), + Some(n) => Some(if n < usize::MAX as f64 { + n as usize + } else { + usize::MAX + }), + None => None, + }; + let m = if m >= 1.0 { m } else { 1.0 }; + let indx = if m <= usize::MAX as f64 { + (m - 1.0) as usize + } else { + usize::MAX + }; + let len = s.chars().count(); + if indx >= len { + return String::new(); + } + let length = length.unwrap_or(len - indx).min(len - indx); + s.chars().skip(indx).take(length).collect() +} + +/// POSIX `sub`/`gsub` replacement text. +fn expand_sub(repl: &str, matched: &str, out: &mut String) { + let mut chars = repl.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.peek() { + Some('\\') => { + chars.next(); + out.push('\\'); + } + Some('&') => { + chars.next(); + out.push('&'); + } + _ => out.push('\\'), + }, + '&' => out.push_str(matched), + c => out.push(c), + } + } +} + +/// `gensub` replacement: `&`/`\0` whole match, `\1`..`\9` groups. +fn expand_gensub(repl: &str, s: &str, caps: &[Option<(usize, usize)>], out: &mut String) { + let group = |i: usize| -> &str { caps.get(i).copied().flatten().map_or("", |(a, b)| &s[a..b]) }; + let mut chars = repl.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.peek().copied() { + Some(d @ '0'..='9') => { + chars.next(); + out.push_str(group(d as usize - '0' as usize)); + } + Some('&') => { + chars.next(); + out.push('&'); + } + Some('\\') => { + chars.next(); + out.push('\\'); + } + _ => out.push('\\'), + }, + '&' => out.push_str(group(0)), + c => out.push(c), + } + } +} + +/// gawk `strtonum`: `0x` hex, leading-zero octal, else decimal. +fn strtonum(s: &str) -> f64 { + let t = s.trim_start(); + let (neg, body) = match t.strip_prefix('-') { + Some(r) => (true, r), + None => (false, t.strip_prefix('+').unwrap_or(t)), + }; + let v = if let Some(h) = body.strip_prefix("0x").or_else(|| body.strip_prefix("0X")) { + let digits: String = h.chars().take_while(|c| c.is_ascii_hexdigit()).collect(); + u64::from_str_radix(&digits, 16).map_or(0.0, |v| v as f64) + } else if body.starts_with('0') + && body.len() > 1 + && body[1..].starts_with(|c: char| c.is_ascii_digit()) + { + let digits: String = body.chars().take_while(|c| c.is_ascii_digit()).collect(); + if digits.bytes().all(|b| b < b'8') { + u64::from_str_radix(&digits, 8).map_or(0.0, |v| v as f64) + } else { + str_to_num(body) + } + } else { + return str_to_num(s); + }; + if neg { -v } else { v } +} + +fn to_bits(v: f64) -> u64 { + if v < 0.0 { (v as i64) as u64 } else { v as u64 } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn substr_rounding() { + assert_eq!(substr("hello", 2.0, Some(3.0)), "ell"); + // gawk 5.2: start clamps to 1, length kept; fractions truncate. + assert_eq!(substr("hello", 0.0, Some(2.0)), "he"); + assert_eq!(substr("hello", -1.0, None), "hello"); + assert_eq!(substr("hello", 1.5, None), "hello"); + assert_eq!(substr("hello world", 1.5, Some(2.3)), "he"); + assert_eq!(substr("hello", 2.0, Some(0.5)), ""); + assert_eq!(substr("olá mundo", 1.0, Some(3.0)), "olá"); + assert_eq!(substr("abc", 5.0, None), ""); + } + + #[test] + fn replacement_rules() { + let mut out = String::new(); + expand_sub("<&>", "x", &mut out); + assert_eq!(out, ""); + out.clear(); + expand_sub("\\&", "x", &mut out); + assert_eq!(out, "&"); + out.clear(); + expand_sub("\\\\&", "x", &mut out); + assert_eq!(out, "\\x"); + } + + #[test] + fn strtonum_forms() { + assert_eq!(strtonum("0x1A"), 26.0); + assert_eq!(strtonum("017"), 15.0); + assert_eq!(strtonum("12abc"), 12.0); + assert_eq!(strtonum("-0x10"), -16.0); + } + + #[test] + fn rand_matches_gawk_first_value() { + let mut r = Rand::new(); + let v = r.rand(); + assert!((0.0..1.0).contains(&v)); + } +} diff --git a/crates/bashkit/src/builtins/awk/interp.rs b/crates/bashkit/src/builtins/awk/interp.rs new file mode 100644 index 000000000..d89a8694a --- /dev/null +++ b/crates/bashkit/src/builtins/awk/interp.rs @@ -0,0 +1,1900 @@ +//! awk evaluator: variables, arrays, records, expressions and statements. +//! +//! Decisions: +//! - Async all the way down so file and command I/O await the VFS and the +//! shell directly (no helper threads). Recursion goes through boxed +//! futures; leaf expressions are evaluated without boxing. +//! - Arrays live in an arena and are referenced by id, which gives awk's +//! by-reference array arguments for free. A function's untyped local that +//! becomes an array is owned by its call frame and freed on return. +//! - `for (k in a)` visits numeric keys ascending, then the others by +//! bytes (gawk's order is a hash order). Arrays keep insertion order, +//! which `PROCINFO["sorted_in"] = "@unsorted"` exposes; the other gawk +//! `sorted_in` orders are supported. +//! - Runtime errors that gawk reports (`division by zero attempted`, ...) +//! carry gawk's `awk: cmd. line:N: fatal:` prefix. Bashkit resource caps +//! keep the plain `awk: fatal: ...` form (TM-DOS-109). +//! - THREAT[TM-DOS-110]: every string is checked against +//! `AWK_MAX_STRING_BYTES` before it is built; variable, element and +//! call-frame bytes are accounted against `max_live_intermediate_bytes` +//! (checked before each statement, so growth overshoots by at most one +//! statement's worth). +//! - THREAT[TM-DOS-033]: loops are counted per loop and per run; +//! user-function recursion is capped by `AWK_MAX_CALL_DEPTH`. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use super::ast::*; +use super::format::{format_number, sprintf}; +use super::io::{Io, RsMode}; +use super::regex::{AwkRegex, RegexCache}; +use super::value::{Value, compare}; +use crate::builtins::limits::{ + AWK_MAX_CALL_DEPTH, AWK_MAX_FIELD_INDEX, AWK_MAX_STRING_BYTES, AWK_VARIABLE_OVERHEAD_BYTES, +}; + +pub(super) type ArrId = usize; +pub(super) type R = Result; +pub(super) type BoxFut<'a, T> = Pin + Send + 'a>>; + +/// Non-local exits out of expression/statement evaluation. +#[derive(Debug)] +pub(super) enum Unwind { + Next, + NextFile, + Exit, + Fatal, +} + +#[derive(Debug)] +pub(super) enum Flow { + Normal, + Break, + Continue, + Return(Value), +} + +#[derive(Debug, Clone)] +pub(super) enum Cell { + Val(Value), + Arr(ArrId), +} + +#[derive(Debug, Clone)] +pub(super) enum Elem { + Val(Value), + Arr(ArrId), +} + +/// Insertion-ordered associative array with tombstones (O(1) delete). +#[derive(Debug, Default)] +pub(super) struct AwkArray { + map: HashMap, + entries: Vec>, +} + +impl AwkArray { + pub(super) fn len(&self) -> usize { + self.map.len() + } + + pub(super) fn get(&self, k: &str) -> Option<&Elem> { + let i = *self.map.get(k)?; + self.entries[i].as_ref().map(|(_, e)| e) + } + + pub(super) fn contains(&self, k: &str) -> bool { + self.map.contains_key(k) + } + + /// Insert or replace; returns the previous element. + fn insert(&mut self, k: String, e: Elem) -> Option { + if let Some(&i) = self.map.get(&k) { + let slot = self.entries[i].as_mut()?; + return Some(std::mem::replace(&mut slot.1, e)); + } + self.map.insert(k.clone(), self.entries.len()); + self.entries.push(Some((k, e))); + None + } + + fn remove(&mut self, k: &str) -> Option { + let i = self.map.remove(k)?; + let old = self.entries[i].take().map(|(_, e)| e); + if self.entries.len() > 32 && self.map.len() * 2 < self.entries.len() { + self.compact(); + } + old + } + + fn compact(&mut self) { + let entries: Vec<_> = std::mem::take(&mut self.entries) + .into_iter() + .flatten() + .collect(); + self.map.clear(); + for (i, (k, _)) in entries.iter().enumerate() { + self.map.insert(k.clone(), i); + } + self.entries = entries.into_iter().map(Some).collect(); + } + + pub(super) fn keys(&self) -> Vec { + self.entries + .iter() + .flatten() + .map(|(k, _)| k.clone()) + .collect() + } + + fn drain(&mut self) -> Vec<(String, Elem)> { + self.map.clear(); + std::mem::take(&mut self.entries) + .into_iter() + .flatten() + .collect() + } + + pub(super) fn iter(&self) -> impl Iterator { + self.entries.iter().flatten().map(|(k, e)| (k, e)) + } +} + +struct Frame { + func: u32, + base: usize, + owned: Vec, +} + +/// Where an lvalue lives once its subscripts / field index are evaluated. +pub(super) enum Loc { + Var(VarRef), + Elem(ArrId, String), + Field(usize), +} + +/// How records are split into fields. +#[derive(Clone)] +pub(super) enum Splitter { + /// `FS = " "`: runs of blanks and newlines; edges ignored. + Space, + Char(char), + /// `FS = ""`: one field per character. + Chars, + Regex(Arc), + Csv, + /// `FIELDWIDTHS`: (skip, width); width `None` takes the rest. + Widths(Vec<(usize, Option)>), + Fpat(Arc), +} + +pub(super) struct Limits { + pub(super) max_loop: usize, + pub(super) max_total_loop: usize, + pub(super) max_mem: usize, +} + +pub(super) struct Interp { + pub(super) prog: Arc, + lit_regex: Vec>, + lit_regex_icase: Vec>>, + pub(super) regex_cache: RegexCache, + pub(super) globals: Vec, + locals: Vec, + frames: Vec, + pub(super) arrays: Vec>, + free_arrays: Vec, + // Current record. + pub(super) record: String, + fields: Vec, + nf: usize, + split_done: bool, + record_stale: bool, + // Cached special variables. + pub(super) convfmt: String, + pub(super) ofmt: String, + ofs: String, + pub(super) ors: String, + pub(super) subsep: String, + pub(super) splitter: Splitter, + /// Splitter chosen by the last of FS/FIELDWIDTHS/FPAT assignments. + pub(super) rs: RsMode, + pub(super) csv: bool, + pub(super) ignorecase: bool, + pub(super) nr: f64, + pub(super) fnr: f64, + range_active: Vec, + pub(super) io: Io, + // Accounting and limits. + pub(super) mem_bytes: usize, + pub(super) limits: Limits, + total_loop: usize, + call_depth: usize, + pub(super) exit_code: i32, + pub(super) fatal: bool, + pub(super) cur_loc: Option, + pub(super) rand: super::funcs::Rand, + /// Set when the shared request budget ran out; the builtin returns it + /// as an error so the dispatcher reports it. + pub(super) budget_error: Option, +} + +/// Number of the current record's fields that a read can address. +impl Interp { + pub(super) fn new(prog: Arc, io: Io, limits: Limits) -> Result { + let mut lit_regex = Vec::with_capacity(prog.regexes.len()); + for re in &prog.regexes { + match AwkRegex::new(re, false) { + Ok(r) => lit_regex.push(Arc::new(r)), + Err(e) => return Err(format!("{}: /{}/", e.0, truncate(re, 200))), + } + } + let n_lit = lit_regex.len(); + let mut globals = vec![Cell::Val(Value::Uninit); prog.globals.len()]; + let s = |v: &str| Cell::Val(Value::Str(v.to_string())); + globals[special::FS as usize] = s(" "); + globals[special::OFS as usize] = s(" "); + globals[special::ORS as usize] = s("\n"); + globals[special::RS as usize] = s("\n"); + globals[special::SUBSEP as usize] = s("\x1c"); + globals[special::CONVFMT as usize] = s("%.6g"); + globals[special::OFMT as usize] = s("%.6g"); + globals[special::NR as usize] = Cell::Val(Value::Num(0.0)); + globals[special::NF as usize] = Cell::Val(Value::Num(0.0)); + globals[special::FNR as usize] = Cell::Val(Value::Num(0.0)); + globals[special::RSTART as usize] = Cell::Val(Value::Num(0.0)); + globals[special::RLENGTH as usize] = Cell::Val(Value::Num(-1.0)); + globals[special::IGNORECASE as usize] = Cell::Val(Value::Num(0.0)); + globals[special::FILENAME as usize] = Cell::Val(Value::Str(String::new())); + let ranges = prog.ranges; + let mut interp = Interp { + prog, + lit_regex, + lit_regex_icase: vec![None; n_lit], + regex_cache: RegexCache::default(), + globals, + locals: Vec::new(), + frames: Vec::new(), + arrays: Vec::new(), + free_arrays: Vec::new(), + record: String::new(), + fields: Vec::new(), + nf: 0, + split_done: true, + record_stale: false, + convfmt: "%.6g".to_string(), + ofmt: "%.6g".to_string(), + ofs: " ".to_string(), + ors: "\n".to_string(), + subsep: "\x1c".to_string(), + splitter: Splitter::Space, + rs: RsMode::Newline, + csv: false, + ignorecase: false, + nr: 0.0, + fnr: 0.0, + range_active: vec![false; ranges], + io, + mem_bytes: 0, + limits, + total_loop: 0, + call_depth: 0, + exit_code: 0, + fatal: false, + cur_loc: None, + rand: super::funcs::Rand::new(), + budget_error: None, + }; + let mem: usize = interp + .globals + .iter() + .map(|c| match c { + Cell::Val(v) => v.heap_bytes(), + Cell::Arr(_) => 0, + }) + .sum(); + interp.mem_bytes = mem; + Ok(interp) + } + + // ----- diagnostics ----- + + /// Bashkit resource cap: `awk: fatal: msg`, exit 2 (TM-DOS-109). + pub(super) fn fatal(&mut self, msg: &str) -> Unwind { + if !self.fatal { + self.fatal = true; + self.err_line(&format!("awk: fatal: {}", truncate(msg, 1000))); + } + Unwind::Fatal + } + + /// gawk runtime fatal error with its source location. + pub(super) fn fatal_at(&mut self, msg: &str) -> Unwind { + if !self.fatal { + self.fatal = true; + let loc = self.location(); + self.err_line(&format!("awk: {loc}fatal: {}", truncate(msg, 1000))); + } + Unwind::Fatal + } + + /// Fatal error with a message that is printed as is after `awk: `. + pub(super) fn fatal_plain(&mut self, msg: &str) -> Unwind { + if !self.fatal { + self.fatal = true; + self.err_line(&format!("awk: {}", truncate(msg, 1000))); + } + Unwind::Fatal + } + + /// Stop because the shared request budget is exhausted; the builtin + /// dispatcher reports it. + pub(super) fn budget_exhausted(&mut self, e: crate::limits::LimitExceeded) -> Unwind { + self.fatal = true; + self.budget_error.get_or_insert(e); + Unwind::Fatal + } + + pub(super) fn location(&self) -> String { + match self.cur_loc.and_then(|l| self.prog.locs.get(l as usize)) { + Some((name, line)) => format!("{name}:{line}: "), + None => String::new(), + } + } + + pub(super) fn err_line(&mut self, s: &str) { + self.io.stderr.push_str(s); + self.io.stderr.push('\n'); + } + + // ----- budget and limits ----- + + fn tick(&mut self) -> R<()> { + if let Some(b) = &self.io.host.budget + && let Err(e) = b.consume_work(1) + { + return Err(self.budget_exhausted(e)); + } + self.check_mem() + } + + fn tick_loop(&mut self, iters: usize) -> R<()> { + self.total_loop += 1; + if iters > self.limits.max_loop { + let msg = format!("loop iteration limit ({}) exceeded", self.limits.max_loop); + return Err(self.fatal(&msg)); + } + if self.total_loop > self.limits.max_total_loop { + let msg = format!( + "total loop iteration limit ({}) exceeded", + self.limits.max_total_loop + ); + return Err(self.fatal(&msg)); + } + Ok(()) + } + + pub(super) fn string_fits(&mut self, len: usize) -> R<()> { + if len > AWK_MAX_STRING_BYTES { + return Err(self.fatal(&format!( + "string size limit ({AWK_MAX_STRING_BYTES} bytes) exceeded" + ))); + } + Ok(()) + } + + // ----- regex access ----- + + pub(super) fn lit_regex(&mut self, id: u32) -> Arc { + let i = id as usize; + if !self.ignorecase { + return self.lit_regex[i].clone(); + } + if let Some(r) = &self.lit_regex_icase[i] { + return r.clone(); + } + let r = AwkRegex::new(&self.prog.regexes[i], true) + .map(Arc::new) + .unwrap_or_else(|_| self.lit_regex[i].clone()); + self.lit_regex_icase[i] = Some(r.clone()); + r + } + + pub(super) fn dyn_regex(&mut self, pattern: &str) -> R> { + match self.regex_cache.get(pattern, self.ignorecase) { + Ok(r) => Ok(r), + Err(e) => Err(self.fatal_at(&format!("{e}: /{}/", truncate(pattern, 200)))), + } + } + + /// The regex an operand denotes: a literal, or a string's value. + pub(super) async fn regex_operand(&mut self, e: &Expr) -> R> { + match e { + Expr::Regex(id) => Ok(self.lit_regex(*id)), + Expr::Group(inner) if matches!(**inner, Expr::Regex(_)) => { + Box::pin(self.regex_operand(inner)).await + } + other => { + let v = self.eval(other).await?; + let s = self.to_str(&v); + self.dyn_regex(&s) + } + } + } + + // ----- conversions ----- + + pub(super) fn to_str(&self, v: &Value) -> String { + v.to_str_fmt(&self.convfmt) + } + + /// Output form of a value (`print` uses OFMT for numbers). + fn to_output(&self, v: &Value) -> String { + match v { + Value::Num(n) => format_number(*n, &self.ofmt), + other => other.to_str_fmt(&self.convfmt), + } + } + + // ----- cells ----- + + pub(super) fn cell(&self, v: VarRef) -> &Cell { + match v { + VarRef::Global(i) => &self.globals[i as usize], + VarRef::Local(i) => { + let base = self.frames.last().map_or(0, |f| f.base); + &self.locals[base + i as usize] + } + } + } + + fn cell_mut(&mut self, v: VarRef) -> &mut Cell { + match v { + VarRef::Global(i) => &mut self.globals[i as usize], + VarRef::Local(i) => { + let base = self.frames.last().map_or(0, |f| f.base); + &mut self.locals[base + i as usize] + } + } + } + + pub(super) fn var_name(&self, v: VarRef) -> String { + match v { + VarRef::Global(i) => self.prog.globals[i as usize].clone(), + VarRef::Local(i) => self + .frames + .last() + .and_then(|f| self.prog.funcs.get(f.func as usize)) + .and_then(|f| f.params.get(i as usize)) + .cloned() + .unwrap_or_default(), + } + } + + pub(super) fn get_var(&mut self, v: VarRef) -> R { + if v == VarRef::Global(special::NF) { + self.ensure_split()?; + return Ok(Value::Num(self.nf as f64)); + } + match self.cell(v) { + Cell::Val(x) => Ok(x.clone()), + Cell::Arr(_) => { + let name = self.var_name(v); + Err(self.fatal_at(&format!( + "attempt to use array `{name}' in a scalar context" + ))) + } + } + } + + pub(super) fn set_var(&mut self, v: VarRef, val: Value) -> R<()> { + let new_bytes = val.heap_bytes(); + let old_bytes = match self.cell(v) { + Cell::Val(x) => x.heap_bytes(), + Cell::Arr(_) => { + let name = self.var_name(v); + return Err(self.fatal_at(&format!( + "attempt to use array `{name}' in a scalar context" + ))); + } + }; + self.mem_bytes = self.mem_bytes + new_bytes - old_bytes.min(self.mem_bytes + new_bytes); + if let VarRef::Global(i) = v + && i < special::COUNT + { + return self.set_special(i, val); + } + *self.cell_mut(v) = Cell::Val(val); + Ok(()) + } + + fn set_special(&mut self, i: u32, val: Value) -> R<()> { + match i { + special::NF => { + let n = val.to_num(); + self.globals[i as usize] = Cell::Val(val); + return self.set_nf(n); + } + special::NR => self.nr = val.to_num(), + special::FNR => self.fnr = val.to_num(), + special::FS => { + let s = self.to_str(&val); + self.globals[i as usize] = Cell::Val(val); + return self.set_fs(&s); + } + special::RS => { + let s = self.to_str(&val); + self.globals[i as usize] = Cell::Val(val); + return self.set_rs(&s); + } + special::OFS => self.ofs = self.to_str(&val), + special::ORS => self.ors = self.to_str(&val), + special::SUBSEP => self.subsep = self.to_str(&val), + special::CONVFMT => self.convfmt = val.to_str_fmt("%.6g"), + special::OFMT => self.ofmt = self.to_str(&val), + special::IGNORECASE => { + let on = val.to_bool(); + self.globals[i as usize] = Cell::Val(val); + if on != self.ignorecase { + self.ignorecase = on; + let fs = self.special_str(special::FS); + self.set_fs(&fs)?; + let rs = self.special_str(special::RS); + self.set_rs(&rs)?; + } + return Ok(()); + } + special::FIELDWIDTHS => { + let s = self.to_str(&val); + self.globals[i as usize] = Cell::Val(val); + self.splitter = Splitter::Widths(parse_widths(&s)); + self.split_done = false; + return Ok(()); + } + special::FPAT => { + let s = self.to_str(&val); + self.globals[i as usize] = Cell::Val(val); + let re = self.dyn_regex(&s)?; + self.splitter = Splitter::Fpat(re); + self.split_done = false; + return Ok(()); + } + _ => {} + } + self.globals[i as usize] = Cell::Val(val); + Ok(()) + } + + fn special_str(&self, i: u32) -> String { + match &self.globals[i as usize] { + Cell::Val(v) => self.to_str(v), + Cell::Arr(_) => String::new(), + } + } + + pub(super) fn set_fs(&mut self, fs: &str) -> R<()> { + self.splitter = if self.csv && fs == "," { + Splitter::Csv + } else { + self.classify_sep(fs)? + }; + Ok(()) + } + + /// Splitter for a separator string (FS rules, shared with `split()`). + pub(super) fn classify_sep(&mut self, sep: &str) -> R { + let mut chars = sep.chars(); + Ok(match (chars.next(), chars.next()) { + (Some(' '), None) => Splitter::Space, + (None, _) => Splitter::Chars, + (Some(c), None) if c != '\\' && !(self.ignorecase && c.is_alphabetic()) => { + Splitter::Char(c) + } + _ => Splitter::Regex(self.dyn_regex(sep)?), + }) + } + + fn set_rs(&mut self, rs: &str) -> R<()> { + let mut chars = rs.chars(); + self.rs = match (chars.next(), chars.next()) { + (None, _) => RsMode::Paragraph, + (Some('\n'), None) => RsMode::Newline, + (Some(c), None) => RsMode::Char(c), + _ => RsMode::Regex(self.dyn_regex(rs)?), + }; + Ok(()) + } + + // ----- arrays ----- + + fn new_array(&mut self) -> ArrId { + if let Some(id) = self.free_arrays.pop() { + self.arrays[id] = Some(AwkArray::default()); + return id; + } + self.arrays.push(Some(AwkArray::default())); + self.arrays.len() - 1 + } + + /// Free an array and its subarrays, releasing their accounted bytes. + fn free_array(&mut self, id: ArrId) { + let mut stack = vec![id]; + while let Some(id) = stack.pop() { + let Some(mut arr) = self.arrays.get_mut(id).and_then(Option::take) else { + continue; + }; + for (k, e) in arr.drain() { + self.mem_bytes = self.mem_bytes.saturating_sub(elem_cost(&k, &e)); + if let Elem::Arr(sub) = e { + stack.push(sub); + } + } + self.free_arrays.push(id); + } + } + + /// Remove every element of an array (the array itself stays). + pub(super) fn clear_array(&mut self, id: ArrId) { + let Some(arr) = self.arrays.get_mut(id).and_then(Option::as_mut) else { + return; + }; + let drained = arr.drain(); + for (k, e) in drained { + self.mem_bytes = self.mem_bytes.saturating_sub(elem_cost(&k, &e)); + if let Elem::Arr(sub) = e { + self.free_array(sub); + } + } + } + + pub(super) fn array(&self, id: ArrId) -> Option<&AwkArray> { + self.arrays.get(id).and_then(Option::as_ref) + } + + /// Array behind a variable, creating it if the variable is untyped. + pub(super) fn array_of_var(&mut self, v: VarRef) -> R { + match self.cell(v) { + Cell::Arr(id) => Ok(*id), + Cell::Val(Value::Uninit) if !matches!(v, VarRef::Global(i) if i < special::COUNT && !matches!(i, special::ENVIRON | special::ARGV | special::PROCINFO | special::SYMTAB)) => + { + let id = self.new_array(); + *self.cell_mut(v) = Cell::Arr(id); + if let VarRef::Local(_) = v + && let Some(f) = self.frames.last_mut() + { + f.owned.push(id); + } + Ok(id) + } + _ => { + let name = self.var_name(v); + Err(self.fatal_at(&format!("attempt to use scalar `{name}' as an array"))) + } + } + } + + /// Array denoted by `name` or `name[i]...` (a subarray), creating + /// untyped parts. + pub(super) async fn array_of_expr(&mut self, e: &Expr) -> R { + match e { + Expr::Var(v) => self.array_of_var(*v), + Expr::Elem(v, groups) => { + let mut id = self.array_of_var(*v)?; + for g in groups { + let key = self.subscript(g).await?; + id = self.subarray(id, key)?; + } + Ok(id) + } + Expr::Group(inner) => Box::pin(self.array_of_expr(inner)).await, + _ => Err(self.fatal_at("attempt to use a scalar value as array")), + } + } + + /// The subarray at `arr[key]`, creating it when missing or untyped. + fn subarray(&mut self, arr: ArrId, key: String) -> R { + match self.array(arr).and_then(|a| a.get(&key)) { + Some(Elem::Arr(sub)) => return Ok(*sub), + Some(Elem::Val(Value::Uninit)) | None => {} + Some(Elem::Val(_)) => { + return Err(self.fatal_at(&format!( + "attempt to use scalar `[\"{}\"]' as array", + truncate(&key, 100) + ))); + } + } + let sub = self.new_array(); + self.elem_store(arr, key, Elem::Arr(sub)); + Ok(sub) + } + + /// Join a subscript group with SUBSEP. + pub(super) async fn subscript(&mut self, g: &[Expr]) -> R { + if g.len() == 1 { + let v = self.eval(&g[0]).await?; + return Ok(self.to_str(&v)); + } + let mut key = String::new(); + for (i, e) in g.iter().enumerate() { + if i > 0 { + key.push_str(&self.subsep.clone()); + } + let v = self.eval(e).await?; + let s = self.to_str(&v); + self.string_fits(key.len() + s.len())?; + key.push_str(&s); + } + Ok(key) + } + + pub(super) fn elem_store(&mut self, arr: ArrId, key: String, e: Elem) { + let add = elem_cost(&key, &e); + let Some(a) = self.arrays.get_mut(arr).and_then(Option::as_mut) else { + return; + }; + let k = key.clone(); + let old = a.insert(key, e); + self.mem_bytes += add; + if let Some(old) = old { + self.mem_bytes = self.mem_bytes.saturating_sub(elem_cost(&k, &old)); + if let Elem::Arr(sub) = old { + self.free_array(sub); + } + } + } + + pub(super) fn elem_delete(&mut self, arr: ArrId, key: &str) { + let Some(a) = self.arrays.get_mut(arr).and_then(Option::as_mut) else { + return; + }; + if let Some(old) = a.remove(key) { + self.mem_bytes = self.mem_bytes.saturating_sub(elem_cost(key, &old)); + if let Elem::Arr(sub) = old { + self.free_array(sub); + } + } + } + + /// Value of `arr[key]`; a reference creates the element (POSIX). + fn elem_load(&mut self, arr: ArrId, key: String) -> R { + match self.array(arr).and_then(|a| a.get(&key)) { + Some(Elem::Val(v)) => Ok(v.clone()), + Some(Elem::Arr(_)) => Err(self.fatal_at(&format!( + "attempt to use array `[\"{}\"]' in a scalar context", + truncate(&key, 100) + ))), + None => { + self.elem_store(arr, key, Elem::Val(Value::Uninit)); + Ok(Value::Uninit) + } + } + } + + /// Set every element of `arr` from `vals` (1-based), replacing it. + pub(super) fn fill_array(&mut self, arr: ArrId, vals: Vec) -> R<()> { + self.fill_array_from(arr, 1, vals) + } + + /// Like `fill_array`, with the first index `first`. + pub(super) fn fill_array_from(&mut self, arr: ArrId, first: usize, vals: Vec) -> R<()> { + self.clear_array(arr); + for (i, v) in vals.into_iter().enumerate() { + self.elem_store(arr, (i + first).to_string(), Elem::Val(v)); + } + self.check_mem() + } + + fn check_mem(&mut self) -> R<()> { + if self.mem_bytes > self.limits.max_mem { + let msg = format!("memory limit ({} bytes) exceeded", self.limits.max_mem); + return Err(self.fatal(&msg)); + } + Ok(()) + } + + // ----- records and fields ----- + + pub(super) fn set_record(&mut self, rec: String) { + self.mem_bytes = self.mem_bytes.saturating_sub(self.record.len()) + rec.len(); + self.record = rec; + self.split_done = false; + self.record_stale = false; + } + + pub(super) fn ensure_split(&mut self) -> R<()> { + if self.split_done { + return Ok(()); + } + let parts = self.split_record()?; + let field_bytes: usize = self.fields.iter().map(Value::heap_bytes).sum(); + self.mem_bytes = self.mem_bytes.saturating_sub(field_bytes); + let mut bytes = 0; + self.fields = parts + .into_iter() + .map(|p| { + bytes += p.len(); + Value::from_input(p) + }) + .collect(); + self.mem_bytes += bytes; + self.nf = self.fields.len(); + self.globals[special::NF as usize] = Cell::Val(Value::Num(self.nf as f64)); + self.split_done = true; + Ok(()) + } + + /// Most array elements or fields the memory cap leaves room for. + pub(super) fn split_budget(&self) -> usize { + self.limits.max_mem.saturating_sub(self.mem_bytes) / AWK_VARIABLE_OVERHEAD_BYTES + 1 + } + + pub(super) fn too_many(&mut self, _: TooMany) -> Unwind { + let msg = format!("memory limit ({} bytes) exceeded", self.limits.max_mem); + self.fatal(&msg) + } + + fn split_record(&mut self) -> R> { + let max = self.split_budget(); + let rec = std::mem::take(&mut self.record); + let splitter = self.splitter.clone(); + let out = if matches!(self.rs, RsMode::Paragraph) && !matches!(splitter, Splitter::Space) { + // POSIX: in paragraph mode newline always separates fields. + let mut v = Vec::new(); + for line in rec.split('\n') { + match split_text(&splitter, line, None, max.saturating_sub(v.len())) { + Ok(p) => v.extend(p), + Err(e) => { + self.record = rec; + return Err(self.too_many(e)); + } + } + } + Ok(v) + } else { + split_text(&splitter, &rec, None, max) + }; + self.record = rec; + out.map_err(|e| self.too_many(e)) + } + + fn rebuild_record(&mut self) -> R<()> { + if !self.record_stale { + return Ok(()); + } + let mut total = 0usize; + let strs: Vec = self + .fields + .iter() + .take(self.nf) + .map(|v| v.to_str_fmt(&self.convfmt)) + .collect(); + for s in &strs { + total += s.len() + self.ofs.len(); + } + self.string_fits(total)?; + let rec = strs.join(&self.ofs); + self.mem_bytes = self.mem_bytes.saturating_sub(self.record.len()) + rec.len(); + self.record = rec; + self.record_stale = false; + Ok(()) + } + + pub(super) fn get_field(&mut self, i: usize) -> R { + if i == 0 { + self.rebuild_record()?; + return Ok(Value::from_input(self.record.clone())); + } + self.ensure_split()?; + Ok(self.fields.get(i - 1).cloned().unwrap_or(Value::Uninit)) + } + + pub(super) fn set_field(&mut self, i: usize, v: Value) -> R<()> { + if i == 0 { + let s = self.to_str(&v); + self.set_record(s); + return Ok(()); + } + if i > AWK_MAX_FIELD_INDEX { + return Err(self.fatal(&format!( + "field index limit ({AWK_MAX_FIELD_INDEX}) exceeded" + ))); + } + self.ensure_split()?; + // The rebuilt record must fit before any field is created. + let est = self.record.len() + v.heap_bytes() + i.saturating_sub(self.nf) * self.ofs.len(); + self.string_fits(est)?; + if i > self.fields.len() { + self.fields.resize(i, Value::Str(String::new())); + } + if i > self.nf { + for f in self.fields.iter_mut().take(i).skip(self.nf) { + *f = Value::Str(String::new()); + } + self.nf = i; + self.globals[special::NF as usize] = Cell::Val(Value::Num(i as f64)); + } + self.mem_bytes = + self.mem_bytes + v.heap_bytes() - self.fields[i - 1].heap_bytes().min(self.mem_bytes); + self.fields[i - 1] = v; + self.record_stale = true; + Ok(()) + } + + fn set_nf(&mut self, n: f64) -> R<()> { + if n < 0.0 { + return Err(self.fatal_at("NF set to negative value")); + } + let n = n as usize; + if n > AWK_MAX_FIELD_INDEX { + return Err(self.fatal(&format!( + "field index limit ({AWK_MAX_FIELD_INDEX}) exceeded" + ))); + } + self.ensure_split()?; + self.string_fits(self.record.len() + n.saturating_sub(self.nf) * self.ofs.len())?; + if n > self.fields.len() { + self.fields.resize(n, Value::Str(String::new())); + } + for f in self.fields.iter_mut().take(n).skip(self.nf) { + *f = Value::Str(String::new()); + } + self.nf = n; + self.fields.truncate(n); + self.globals[special::NF as usize] = Cell::Val(Value::Num(n as f64)); + self.record_stale = true; + Ok(()) + } + + // ----- lvalues ----- + + pub(super) async fn resolve(&mut self, lv: &Expr) -> R { + match lv { + Expr::Var(v) => Ok(Loc::Var(*v)), + Expr::Elem(v, groups) => { + let mut id = self.array_of_var(*v)?; + let (last, path) = groups.split_last().ok_or(Unwind::Fatal)?; + for g in path { + let k = self.subscript(g).await?; + id = self.subarray(id, k)?; + } + let key = self.subscript(last).await?; + Ok(Loc::Elem(id, key)) + } + Expr::Field(e) => { + let i = self.eval(e).await?.to_num(); + Ok(Loc::Field(self.field_index(i)?)) + } + Expr::Group(inner) => Box::pin(self.resolve(inner)).await, + _ => Err(self.fatal_at("assignment to a non-lvalue")), + } + } + + pub(super) fn field_index(&mut self, i: f64) -> R { + if i < 0.0 || i.is_nan() { + return Err(self.fatal_at(&format!( + "attempt to access field {}", + format_number(i.trunc(), "%.6g") + ))); + } + if i > AWK_MAX_FIELD_INDEX as f64 { + return Err(self.fatal(&format!( + "field index limit ({AWK_MAX_FIELD_INDEX}) exceeded" + ))); + } + Ok(i as usize) + } + + pub(super) fn load(&mut self, loc: &Loc) -> R { + match loc { + Loc::Var(v) => self.get_var(*v), + Loc::Elem(id, key) => self.elem_load(*id, key.clone()), + Loc::Field(i) => self.get_field(*i), + } + } + + pub(super) fn store(&mut self, loc: Loc, v: Value) -> R<()> { + self.string_fits(v.heap_bytes())?; + match loc { + Loc::Var(var) => self.set_var(var, v), + Loc::Elem(id, key) => { + if let Some(Elem::Arr(_)) = self.array(id).and_then(|a| a.get(&key)) { + return Err(self.fatal_at(&format!( + "attempt to use array `[\"{}\"]' in a scalar context", + truncate(&key, 100) + ))); + } + self.elem_store(id, key, Elem::Val(v)); + Ok(()) + } + Loc::Field(i) => self.set_field(i, v), + } + } + + pub(super) async fn assign(&mut self, lv: &Expr, v: Value) -> R<()> { + let loc = self.resolve(lv).await?; + self.store(loc, v) + } + + // ----- expressions ----- + + /// Evaluate; leaves are handled inline, the rest through a boxed future. + pub(super) async fn eval(&mut self, e: &Expr) -> R { + match e { + Expr::Num(n) => Ok(Value::Num(*n)), + Expr::Str(s) => Ok(Value::Str(s.clone())), + Expr::Var(v) => self.get_var(*v), + _ => self.eval_boxed(e).await, + } + } + + /// Each arm is its own boxed future, so a recursive call keeps only + /// that arm's (small) poll frame on the stack, not one frame sized for + /// every arm. This bounds stack use per awk call level. + fn eval_boxed<'a>(&'a mut self, e: &'a Expr) -> BoxFut<'a, R> { + match e { + Expr::Num(n) => Box::pin(async move { Ok(Value::Num(*n)) }), + Expr::Str(s) => Box::pin(async move { Ok(Value::Str(s.clone())) }), + Expr::Var(v) => Box::pin(async move { self.get_var(*v) }), + Expr::Regex(id) => Box::pin(async move { + let re = self.lit_regex(*id); + self.rebuild_record()?; + Ok(bool_val(re.is_match(&self.record))) + }), + Expr::Elem(..) => Box::pin(async move { + let loc = self.resolve(e).await?; + self.load(&loc) + }), + Expr::Field(i) => Box::pin(async move { + let n = self.eval(i).await?.to_num(); + let idx = self.field_index(n)?; + self.get_field(idx) + }), + Expr::Group(inner) => Box::pin(async move { self.eval(inner).await }), + Expr::Assign(lv, op, rhs) => Box::pin(async move { + let loc = self.resolve(lv).await?; + let r = self.eval(rhs).await?; + let v = match op { + None => r, + Some(op) => { + let cur = self.load(&loc)?.to_num(); + Value::Num(self.arith(*op, cur, r.to_num())?) + } + }; + self.store(loc, v.clone())?; + Ok(v) + }), + Expr::Neg(x) => Box::pin(async move { Ok(Value::Num(-self.eval(x).await?.to_num())) }), + Expr::Plus(x) => Box::pin(async move { Ok(Value::Num(self.eval(x).await?.to_num())) }), + Expr::Not(x) => Box::pin(async move { Ok(bool_val(!self.eval(x).await?.to_bool())) }), + Expr::Binary(op, a, b) => Box::pin(async move { + let x = self.eval(a).await?.to_num(); + let y = self.eval(b).await?.to_num(); + Ok(Value::Num(self.arith(*op, x, y)?)) + }), + Expr::Cmp(op, a, b) => Box::pin(async move { + let x = self.eval(a).await?; + let y = self.eval(b).await?; + let ord = if self.ignorecase && !(x.is_numeric() && y.is_numeric()) { + let (xs, ys) = ( + self.to_str(&x).to_lowercase(), + self.to_str(&y).to_lowercase(), + ); + xs.as_bytes().cmp(ys.as_bytes()) + } else { + compare(&x, &y, &self.convfmt) + }; + use std::cmp::Ordering::*; + let nan = (x.is_numeric() && y.is_numeric()) + && (x.to_num().is_nan() || y.to_num().is_nan()); + let r = match op { + CmpOp::Lt => ord == Less, + CmpOp::Le => ord != Greater, + CmpOp::Gt => ord == Greater, + CmpOp::Ge => ord != Less, + CmpOp::Eq => ord == Equal, + CmpOp::Ne => ord != Equal, + }; + Ok(bool_val(if nan { matches!(op, CmpOp::Ne) } else { r })) + }), + Expr::Match(neg, a, b) => Box::pin(async move { + let s = self.eval(a).await?; + let s = self.to_str(&s); + let re = self.regex_operand(b).await?; + Ok(bool_val(re.is_match(&s) != *neg)) + }), + Expr::And(a, b) => { + Box::pin(async move { Ok(bool_val(self.cond(a).await? && self.cond(b).await?)) }) + } + Expr::Or(a, b) => { + Box::pin(async move { Ok(bool_val(self.cond(a).await? || self.cond(b).await?)) }) + } + Expr::Cond(c, a, b) => Box::pin(async move { + if self.cond(c).await? { + self.eval(a).await + } else { + self.eval(b).await + } + }), + Expr::Concat(a, b) => Box::pin(async move { + let x = self.eval(a).await?; + let mut s = self.to_str(&x); + let y = self.eval(b).await?; + let t = self.to_str(&y); + self.string_fits(s.len() + t.len())?; + s.push_str(&t); + Ok(Value::Str(s)) + }), + Expr::In(subs, arr) => Box::pin(async move { + let key = self.subscript(subs).await?; + let id = self.array_of_expr(arr).await?; + Ok(bool_val(self.array(id).is_some_and(|a| a.contains(&key)))) + }), + Expr::IncDec(lv, delta, prefix) => Box::pin(async move { + let loc = self.resolve(lv).await?; + let old = self.load(&loc)?.to_num(); + let new = old + delta; + self.store(loc, Value::Num(new))?; + Ok(Value::Num(if *prefix { new } else { old })) + }), + Expr::Call(f, args) => { + Box::pin(async move { Box::pin(self.call_user(*f, args)).await }) + } + Expr::Indirect(name, args) => Box::pin(async move { + let n = self.eval(name).await?; + let n = self.to_str(&n); + let prog = self.prog.clone(); + if let Some(i) = prog.funcs.iter().position(|f| f.name == n && f.defined) { + return Box::pin(self.call_user(i as u32, args)).await; + } + if let Some(bi) = Bi::from_name(&n) { + return Box::pin(self.call_builtin(bi, args)).await; + } + Err(self.fatal_at(&format!("function `{}' not defined", truncate(&n, 100)))) + }), + Expr::Builtin(bi, args) => { + Box::pin(async move { Box::pin(self.call_builtin(*bi, args)).await }) + } + Expr::Getline(src, var) => { + Box::pin(async move { Box::pin(self.getline(src, var.as_deref())).await }) + } + Expr::List(items) => Box::pin(async move { + let key = self.subscript(items).await?; + Ok(Value::Str(key)) + }), + } + } + + pub(super) async fn cond(&mut self, e: &Expr) -> R { + Ok(self.eval(e).await?.to_bool()) + } + + fn arith(&mut self, op: BinOp, x: f64, y: f64) -> R { + Ok(match op { + BinOp::Add => x + y, + BinOp::Sub => x - y, + BinOp::Mul => x * y, + BinOp::Div => { + if y == 0.0 { + return Err(self.fatal_at("division by zero attempted")); + } + x / y + } + BinOp::Mod => { + if y == 0.0 { + return Err(self.fatal_at("division by zero attempted in `%'")); + } + x % y + } + BinOp::Pow => fix_nan(x.powf(y), &[x, y]), + }) + } + + // ----- user functions ----- + + async fn call_user(&mut self, fid: u32, args: &[Expr]) -> R { + let prog = self.prog.clone(); + let f = &prog.funcs[fid as usize]; + if args.len() > f.params.len() { + return Err(self.fatal_at(&format!( + "function `{}' called with more arguments than declared", + f.name + ))); + } + if self.call_depth >= AWK_MAX_CALL_DEPTH { + return Err(self.fatal(&format!( + "function call depth limit ({AWK_MAX_CALL_DEPTH}) exceeded" + ))); + } + let mut cells = Vec::with_capacity(f.params.len()); + for (j, a) in args.iter().enumerate() { + let wants_array = f.array_params.get(j).copied().unwrap_or(false); + let cell = match a { + Expr::Var(v) if *v != VarRef::Global(special::NF) => match self.cell(*v) { + Cell::Arr(id) => Cell::Arr(*id), + Cell::Val(Value::Uninit) if wants_array => Cell::Arr(self.array_of_var(*v)?), + Cell::Val(x) => Cell::Val(x.clone()), + }, + Expr::Elem(v, groups) => { + let mut id = self.array_of_var(*v)?; + let (last, path) = groups.split_last().ok_or(Unwind::Fatal)?; + for g in path { + let k = self.subscript(g).await?; + id = self.subarray(id, k)?; + } + let key = self.subscript(last).await?; + match self.array(id).and_then(|a| a.get(&key)) { + Some(Elem::Arr(sub)) => Cell::Arr(*sub), + _ if wants_array => Cell::Arr(self.subarray(id, key)?), + _ => Cell::Val(self.elem_load(id, key)?), + } + } + other => Cell::Val(self.eval(other).await?), + }; + cells.push(cell); + } + cells.resize(f.params.len(), Cell::Val(Value::Uninit)); + let bytes: usize = cells + .iter() + .map(|c| match c { + Cell::Val(v) => v.heap_bytes(), + Cell::Arr(_) => 0, + }) + .sum(); + self.mem_bytes += bytes; + let base = self.locals.len(); + self.locals.extend(cells); + self.frames.push(Frame { + func: fid, + base, + owned: Vec::new(), + }); + self.call_depth += 1; + let saved_loc = self.cur_loc; + let r = self.exec_block(&f.body).await; + self.cur_loc = saved_loc; + self.call_depth -= 1; + if let Some(frame) = self.frames.pop() { + for c in self.locals.drain(frame.base..) { + if let Cell::Val(v) = c { + self.mem_bytes = self.mem_bytes.saturating_sub(v.heap_bytes()); + } + } + for id in frame.owned { + self.free_array(id); + } + } + match r? { + Flow::Return(v) => Ok(v), + _ => Ok(Value::Uninit), + } + } + + // ----- statements ----- + + pub(super) async fn exec_block(&mut self, b: &[Stmt]) -> R { + for s in b { + match self.exec_stmt(s).await? { + Flow::Normal => {} + other => return Ok(other), + } + } + Ok(Flow::Normal) + } + + /// One boxed future per statement kind (see `eval_boxed`). + fn exec_stmt<'a>(&'a mut self, s: &'a Stmt) -> BoxFut<'a, R> { + self.cur_loc = Some(s.loc); + if let Err(e) = self.tick() { + return Box::pin(std::future::ready(Err(e))); + } + match &s.kind { + StmtKind::Expr(e) => Box::pin(async move { + self.eval(e).await?; + Ok(Flow::Normal) + }), + StmtKind::Print(args, dest) => Box::pin(async move { + let mut line = String::new(); + if args.is_empty() { + self.rebuild_record()?; + line.push_str(&self.record); + } else { + for (i, a) in args.iter().enumerate() { + if i > 0 { + line.push_str(&self.ofs); + } + let v = self.eval(a).await?; + let t = self.to_output(&v); + self.string_fits(line.len() + t.len())?; + line.push_str(&t); + } + } + line.push_str(&self.ors); + Box::pin(self.write_out(dest.as_ref(), line)).await?; + Ok(Flow::Normal) + }), + StmtKind::Printf(args, dest) => Box::pin(async move { + let mut vals = Vec::with_capacity(args.len()); + for a in args { + vals.push(self.eval(a).await?); + } + let fmt = self.to_str(&vals[0]); + let out = match sprintf(&fmt, &vals[1..], &self.convfmt) { + Ok(s) => s, + Err(e) if e.located => return Err(self.fatal_at(&e.msg)), + Err(e) => return Err(self.fatal(&e.msg)), + }; + self.string_fits(out.len())?; + Box::pin(self.write_out(dest.as_ref(), out)).await?; + Ok(Flow::Normal) + }), + StmtKind::If(c, a, b) => Box::pin(async move { + if self.cond(c).await? { + return self.exec_block(a).await; + } else if let Some(b) = b { + return self.exec_block(b).await; + } + Ok(Flow::Normal) + }), + StmtKind::While(c, body) => Box::pin(async move { + let mut n = 0; + while self.cond(c).await? { + n += 1; + self.tick_loop(n)?; + match self.exec_block(body).await? { + Flow::Break => break, + Flow::Return(v) => return Ok(Flow::Return(v)), + _ => {} + } + } + Ok(Flow::Normal) + }), + StmtKind::Do(body, c) => Box::pin(async move { + let mut n = 0; + loop { + n += 1; + self.tick_loop(n)?; + match self.exec_block(body).await? { + Flow::Break => break, + Flow::Return(v) => return Ok(Flow::Return(v)), + _ => {} + } + if !self.cond(c).await? { + break; + } + } + Ok(Flow::Normal) + }), + StmtKind::For(init, c, incr, body) => Box::pin(async move { + if let Some(i) = init { + self.exec_stmt(i).await?; + } + let mut n = 0; + loop { + if let Some(c) = c + && !self.cond(c).await? + { + break; + } + n += 1; + self.tick_loop(n)?; + match self.exec_block(body).await? { + Flow::Break => break, + Flow::Return(v) => return Ok(Flow::Return(v)), + _ => {} + } + if let Some(i) = incr { + self.exec_stmt(i).await?; + } + } + Ok(Flow::Normal) + }), + StmtKind::ForIn(key, arr, body) => Box::pin(async move { + let id = self.array_of_expr(arr).await?; + let keys = self.for_in_keys(id); + let mut n = 0; + for k in keys { + if !self.array(id).is_some_and(|a| a.contains(&k)) { + continue; + } + n += 1; + self.tick_loop(n)?; + self.assign(key, Value::from_input(k)).await?; + match self.exec_block(body).await? { + Flow::Break => break, + Flow::Return(v) => return Ok(Flow::Return(v)), + _ => {} + } + } + Ok(Flow::Normal) + }), + StmtKind::Block(b) => Box::pin(async move { self.exec_block(b).await }), + StmtKind::Next => Box::pin(async move { Err(Unwind::Next) }), + StmtKind::NextFile => Box::pin(async move { Err(Unwind::NextFile) }), + StmtKind::Exit(e) => Box::pin(async move { + if let Some(e) = e { + let v = self.eval(e).await?.to_num(); + self.exit_code = if v.is_finite() { + (v as i64 & 0xff) as i32 + } else { + 0 + }; + } + Err(Unwind::Exit) + }), + StmtKind::Return(e) => Box::pin(async move { + let v = match e { + Some(e) => self.eval(e).await?, + None => Value::Uninit, + }; + Ok(Flow::Return(v)) + }), + StmtKind::Break => Box::pin(async move { Ok(Flow::Break) }), + StmtKind::Continue => Box::pin(async move { Ok(Flow::Continue) }), + StmtKind::Delete(target) => Box::pin(async move { + match target { + Expr::Var(v) => { + let id = self.array_of_var(*v)?; + self.clear_array(id); + } + Expr::Elem(v, groups) => { + let mut id = self.array_of_var(*v)?; + let (last, path) = groups.split_last().ok_or(Unwind::Fatal)?; + for g in path { + let k = self.subscript(g).await?; + match self.array(id).and_then(|a| a.get(&k)) { + Some(Elem::Arr(sub)) => id = *sub, + _ => return Ok(Flow::Normal), + } + } + let key = self.subscript(last).await?; + self.elem_delete(id, &key); + } + _ => {} + }; + Ok(Flow::Normal) + }), + StmtKind::Switch(subject, cases) => Box::pin(async move { + let v = self.eval(subject).await?; + let mut matched = None; + for (i, (label, _)) in cases.iter().enumerate() { + let hit = match label { + CaseLabel::Default => false, + CaseLabel::Regex(id) => { + let re = self.lit_regex(*id); + re.is_match(&self.to_str(&v)) + } + CaseLabel::Value(e) => { + let c = self.eval(e).await?; + match c { + Value::Num(n) if v.is_numeric() => v.to_num() == n, + _ => self.to_str(&v) == self.to_str(&c), + } + } + }; + if hit { + matched = Some(i); + break; + } + } + if matched.is_none() { + matched = cases + .iter() + .position(|(l, _)| matches!(l, CaseLabel::Default)); + } + if let Some(start) = matched { + for (_, body) in &cases[start..] { + match self.exec_block(body).await? { + Flow::Normal => {} + Flow::Break => break, + other => return Ok(other), + } + } + } + Ok(Flow::Normal) + }), + } + } + + /// Keys for `for (k in a)`, in `PROCINFO["sorted_in"]` order if set. + fn for_in_keys(&self, id: ArrId) -> Vec { + let Some(arr) = self.array(id) else { + return Vec::new(); + }; + let order = match &self.globals[special::PROCINFO as usize] { + Cell::Arr(p) => match self.array(*p).and_then(|a| a.get("sorted_in")) { + Some(Elem::Val(v)) => self.to_str(v), + _ => String::new(), + }, + Cell::Val(_) => String::new(), + }; + let mut keys = arr.keys(); + if order == "@unsorted" { + return keys; + } + if order.is_empty() { + // Default: numeric keys ascending, then the rest by bytes. + // Deterministic; gawk's own order follows its hash tables. + // TODO: reproduce gawk's str/int/cint array iteration order + // so unsorted `for (k in a)` output matches Debian awk. + keys.sort_by(|a, b| match (key_num(a), key_num(b)) { + (Some(x), Some(y)) => x.total_cmp(&y).then_with(|| a.cmp(b)), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => a.cmp(b), + }); + return keys; + } + let val = |k: &String| match arr.get(k) { + Some(Elem::Val(v)) => v.clone(), + _ => Value::Uninit, + }; + match order.as_str() { + "@ind_str_asc" => keys.sort(), + "@ind_str_desc" => keys.sort_by(|a, b| b.cmp(a)), + "@ind_num_asc" => keys.sort_by(|a, b| num_cmp(a, b)), + "@ind_num_desc" => keys.sort_by(|a, b| num_cmp(b, a)), + "@val_str_asc" | "@val_type_asc" => keys.sort_by(|a, b| { + self.to_str(&val(a)) + .cmp(&self.to_str(&val(b))) + .then_with(|| a.cmp(b)) + }), + "@val_str_desc" | "@val_type_desc" => keys.sort_by(|a, b| { + self.to_str(&val(b)) + .cmp(&self.to_str(&val(a))) + .then_with(|| b.cmp(a)) + }), + "@val_num_asc" => keys.sort_by(|a, b| { + val(a) + .to_num() + .total_cmp(&val(b).to_num()) + .then_with(|| a.cmp(b)) + }), + "@val_num_desc" => keys.sort_by(|a, b| { + val(b) + .to_num() + .total_cmp(&val(a).to_num()) + .then_with(|| b.cmp(a)) + }), + _ => {} + } + keys + } + + // ----- program driving ----- + + async fn pattern_matches(&mut self, rule: &Rule) -> R { + Ok(match &rule.pattern { + Pattern::All => true, + Pattern::Expr(e) => self.cond(e).await?, + Pattern::Range(a, b, id) => { + if !self.range_active[*id] { + if self.cond(a).await? { + if !self.cond(b).await? { + self.range_active[*id] = true; + } + true + } else { + false + } + } else { + if self.cond(b).await? { + self.range_active[*id] = false; + } + true + } + } + }) + } + + async fn run_rules(&mut self, prog: &Program) -> R<()> { + for rule in &prog.rules { + if !self.pattern_matches(rule).await? { + continue; + } + match &rule.body { + None => { + self.rebuild_record()?; + let mut line = self.record.clone(); + line.push_str(&self.ors); + self.write_out(None, line).await?; + } + Some(b) => { + self.exec_block(b).await?; + } + } + } + Ok(()) + } + + pub(super) async fn run_blocks(&mut self, blocks: &[Block]) -> R<()> { + for b in blocks { + self.exec_block(b).await?; + } + Ok(()) + } + + /// Run the whole program; returns the exit status. + pub(super) async fn run(&mut self) -> i32 { + let prog = self.prog.clone(); + let mut exited = false; + match self.run_blocks(&prog.begin).await { + Ok(()) => {} + Err(Unwind::Exit) => exited = true, + Err(Unwind::Fatal) => return self.finish().await, + Err(Unwind::Next | Unwind::NextFile) => {} + } + let reads_input = !prog.rules.is_empty() + || !prog.end.is_empty() + || !prog.beginfile.is_empty() + || !prog.endfile.is_empty(); + if !exited && reads_input { + loop { + let rec = match self.next_main_record().await { + Ok(Some(r)) => r, + Ok(None) => break, + Err(Unwind::Exit) => { + exited = true; + break; + } + Err(Unwind::Fatal) => return self.finish().await, + Err(_) => continue, + }; + self.set_record(rec); + match self.run_rules(&prog).await { + Ok(()) | Err(Unwind::Next) => {} + Err(Unwind::NextFile) => { + if let Err(e) = self.skip_file().await { + match e { + Unwind::Exit => { + exited = true; + break; + } + Unwind::Fatal => return self.finish().await, + _ => {} + } + } + } + Err(Unwind::Exit) => { + exited = true; + break; + } + Err(Unwind::Fatal) => return self.finish().await, + } + } + } + let _ = exited; + match self.run_blocks(&prog.end).await { + Ok(()) | Err(Unwind::Exit) | Err(Unwind::Next) | Err(Unwind::NextFile) => {} + Err(Unwind::Fatal) => return self.finish().await, + } + self.finish().await + } + + async fn finish(&mut self) -> i32 { + // Flush redirections and run pending output pipes, even after a + // fatal error (gawk closes its files on exit too). + self.close_all().await; + if self.fatal { 2 } else { self.exit_code } + } +} + +pub(super) fn bool_val(b: bool) -> Value { + Value::Num(if b { 1.0 } else { 0.0 }) +} + +/// A NaN produced from non-NaN operands prints as `-nan`, like gawk on +/// x86-64 (the hardware default NaN has its sign bit set). +pub(super) fn fix_nan(r: f64, inputs: &[f64]) -> f64 { + if r.is_nan() && !inputs.iter().any(|x| x.is_nan()) { + -f64::NAN + } else { + r + } +} + +fn elem_cost(key: &str, e: &Elem) -> usize { + key.len() + + AWK_VARIABLE_OVERHEAD_BYTES + + match e { + Elem::Val(v) => v.heap_bytes(), + Elem::Arr(_) => 0, + } +} + +fn key_num(k: &str) -> Option { + k.parse::().ok().filter(|n| n.is_finite()) +} + +fn num_cmp(a: &str, b: &str) -> std::cmp::Ordering { + let (x, y) = (super::value::str_to_num(a), super::value::str_to_num(b)); + x.total_cmp(&y).then_with(|| a.cmp(b)) +} + +pub(super) fn truncate(s: &str, max: usize) -> &str { + if s.len() <= max { + return s; + } + let mut end = max; + while !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] +} + +fn parse_widths(s: &str) -> Vec<(usize, Option)> { + let mut out = Vec::new(); + for item in s.split_whitespace() { + let (skip, w) = match item.split_once(':') { + Some((a, b)) => (a.parse().unwrap_or(0), b), + None => (0, item), + }; + let width = if w == "*" { None } else { w.parse().ok() }; + if width.is_none() && w != "*" { + continue; + } + out.push((skip, width)); + } + out +} + +/// Split `text` with `sp`; `seps` collects the separators when given. +pub(super) fn split_text( + sp: &Splitter, + text: &str, + mut seps: Option<&mut Vec>, + max: usize, +) -> Result, TooMany> { + let mut out = Parts { + v: Vec::new(), + max, + over: false, + }; + match sp { + Splitter::Space => { + let is_blank = |c: char| c == ' ' || c == '\t' || c == '\n'; + let trimmed_start = text.trim_start_matches(is_blank); + // seps[0] is the leading blank run (gawk), possibly empty. + if let Some(s) = seps.as_deref_mut() { + s.push(text[..text.len() - trimmed_start.len()].to_string()); + } + let mut rest = trimmed_start; + while !rest.is_empty() && !out.over { + let end = rest.find(is_blank).unwrap_or(rest.len()); + out.push(rest[..end].to_string()); + let after = rest[end..].trim_start_matches(is_blank); + if let Some(s) = seps.as_deref_mut() { + let sep = &rest[end..rest.len() - after.len()]; + if !sep.is_empty() { + s.push(sep.to_string()); + } + } + rest = after; + } + } + _ if text.is_empty() => {} + Splitter::Char(c) => { + for part in text.split(*c) { + out.push(part.to_string()); + if out.over { + break; + } + } + if let Some(s) = seps.as_deref_mut() { + for _ in 1..out.v.len() { + s.push(c.to_string()); + } + } + } + Splitter::Chars => { + for c in text.chars() { + out.push(c.to_string()); + if out.over { + break; + } + } + } + Splitter::Regex(re) => { + let mut last = 0; + let mut pos = 0; + while pos <= text.len() && !out.over { + let Some((s, e)) = re.find_at(text, pos) else { + break; + }; + if s == e { + // An empty match separates nothing. + pos = next_char(text, s); + continue; + } + out.push(text[last..s].to_string()); + if let Some(v) = seps.as_deref_mut() { + v.push(text[s..e].to_string()); + } + last = e; + pos = e; + } + out.push(text[last..].to_string()); + } + Splitter::Csv => { + for f in super::csv_split_fields(text) { + out.push(f); + } + } + Splitter::Widths(ws) => { + let chars: Vec = text.chars().collect(); + let mut i = 0; + for (skip, w) in ws { + if out.over { + break; + } + i += skip; + if i >= chars.len() { + break; + } + let end = match w { + Some(w) => (i + w).min(chars.len()), + None => chars.len(), + }; + out.push(chars[i..end].iter().collect()); + i = end; + } + } + Splitter::Fpat(re) => { + let mut pos = 0; + let mut last_end: Option = None; + while pos <= text.len() && !out.over { + let Some((s, e)) = re.find_at(text, pos) else { + break; + }; + if s == e { + if last_end != Some(s) { + out.push(String::new()); + last_end = Some(s); + } + pos = next_char(text, s); + continue; + } + if let Some(v) = seps.as_deref_mut() + && let Some(l) = last_end + { + v.push(text[l..s].to_string()); + } + out.push(text[s..e].to_string()); + last_end = Some(e); + pos = e; + } + } + } + if out.over { Err(TooMany) } else { Ok(out.v) } +} + +/// More fields than the memory cap leaves room for. +pub(super) struct TooMany; + +/// Split output that stops at `max` parts, so a huge string cannot build +/// a huge vector before the memory cap is checked (TM-DOS-110). +struct Parts { + v: Vec, + max: usize, + over: bool, +} + +impl Parts { + fn push(&mut self, s: String) { + if self.v.len() >= self.max { + self.over = true; + } else { + self.v.push(s); + } + } +} + +/// Byte offset just past the character at `i` (or `len + 1` at the end). +pub(super) fn next_char(s: &str, i: usize) -> usize { + match s[i..].chars().next() { + Some(c) => i + c.len_utf8(), + None => s.len() + 1, + } +} diff --git a/crates/bashkit/src/builtins/awk/interpreter.rs b/crates/bashkit/src/builtins/awk/interpreter.rs deleted file mode 100644 index 1c75502d2..000000000 --- a/crates/bashkit/src/builtins/awk/interpreter.rs +++ /dev/null @@ -1,1871 +0,0 @@ -//! AWK interpreter - executes a parsed `AwkProgram`. - -use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; -use std::sync::Arc; - -use super::{ - AwkAction, AwkExpr, AwkFunctionDef, AwkOutputTarget, AwkPattern, AwkState, AwkValue, FieldSep, -}; -use crate::builtins::MAX_FORMAT_WIDTH; -use crate::builtins::limits::{ - AWK_MAX_CALL_DEPTH as MAX_AWK_CALL_DEPTH, AWK_MAX_FIELD_INDEX, - AWK_MAX_GETLINE_CACHE_BYTES as MAX_GETLINE_CACHE_BYTES, - AWK_MAX_GETLINE_CACHED_FILES as MAX_GETLINE_CACHED_FILES, - AWK_MAX_GETLINE_FILE_BYTES as MAX_GETLINE_FILE_BYTES, - AWK_MAX_OUTPUT_BYTES as MAX_AWK_OUTPUT_BYTES, AWK_MAX_OUTPUT_TARGETS as MAX_AWK_OUTPUT_TARGETS, - AWK_MAX_STRING_BYTES, -}; -use crate::builtins::search_common::RuntimeRegexCache; -use crate::fs::{FileSystem, normalize_path}; -use crate::limits::ExecutionLimits; -// On wasm32 there is no OS thread to bridge the sync AWK evaluator to the async -// VFS, so redirected reads/writes drive the VFS future inline with now_or_never. -use crate::fs::vfs_join; -#[cfg(target_family = "wasm")] -use futures_util::FutureExt; - -/// Flow control signal from action execution -#[derive(Debug, PartialEq)] -pub(super) enum AwkFlow { - Continue, // Normal execution - Next, // Skip to next record - Break, // Break out of loop - LoopContinue, // Continue to next loop iteration - Exit(Option), // Exit program with optional code - Return(AwkValue), // Return from user-defined function -} - -// Awk runtime limits (TM-DOS-027, TM-DOS-028) live in `super::limits`: -// - AWK_MAX_CALL_DEPTH (user-function recursion) -// - AWK_MAX_OUTPUT_BYTES (total stdout+stderr+file redirects, 10 MB) -// - AWK_MAX_OUTPUT_TARGETS (distinct redirected output files). -// - AWK_MAX_GETLINE_CACHED_FILES (distinct files held open by `getline`). -// - AWK_MAX_GETLINE_FILE_BYTES / AWK_MAX_GETLINE_CACHE_BYTES (retained input bytes). -// - The request execution budget counts every `getline` load monotonically; -// `close()` releases retained memory but cannot recycle read/work quotas. -// THREAT[TM-DOS-023]: Runtime regex operands share one bounded cache. Invalid -// patterns are cached too, preventing repeated compilation failures from -// becoming a CPU sink. - -/// One redirected VFS write, dispatched to the [`VfsWriter`] thread. -#[cfg(not(target_family = "wasm"))] -struct WriteJob { - path: PathBuf, - bytes: Vec, - append: bool, - resp: std::sync::mpsc::Sender>, -} - -/// A single long-lived OS thread owning one Tokio runtime that serializes -/// redirected VFS writes. -/// -/// AWK executes synchronously inside the outer async runtime, so each -/// redirected write must hop to a thread with its own runtime to call the async -/// VFS. Spawning a fresh thread + runtime per write would let a tight -/// `print > file` loop create thousands of threads/runtimes (DoS). Reusing one -/// writer for the whole run keeps writes incremental (so VFS quotas are still -/// enforced as they happen) at O(1) threads. -#[cfg(not(target_family = "wasm"))] -struct VfsWriter { - tx: Option>, - handle: Option>, -} - -#[cfg(not(target_family = "wasm"))] -impl VfsWriter { - fn new(fs: Arc) -> Self { - let (tx, rx) = std::sync::mpsc::channel::(); - let handle = std::thread::spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("awk vfs writer runtime"); - runtime.block_on(async move { - while let Ok(job) = rx.recv() { - let res = if job.append { - fs.append_file(&job.path, &job.bytes).await - } else { - fs.write_file(&job.path, &job.bytes).await - }; - let _ = job.resp.send(res); - } - }); - }); - Self { - tx: Some(tx), - handle: Some(handle), - } - } - - /// Perform one write synchronously on the writer thread. Returns the VFS - /// result, or `Err(())` if the writer thread is gone. - fn write( - &self, - path: PathBuf, - bytes: Vec, - append: bool, - ) -> Result, ()> { - let (resp_tx, resp_rx) = std::sync::mpsc::channel(); - let job = WriteJob { - path, - bytes, - append, - resp: resp_tx, - }; - self.tx.as_ref().ok_or(())?.send(job).map_err(|_| ())?; - resp_rx.recv().map_err(|_| ()) - } -} - -#[cfg(not(target_family = "wasm"))] -impl Drop for VfsWriter { - fn drop(&mut self) { - // Drop the sender first so the writer loop's `rx.recv()` returns Err and - // the thread exits, then join it to flush any in-flight write cleanly. - drop(self.tx.take()); - if let Some(handle) = self.handle.take() { - let _ = handle.join(); - } - } -} - -/// wasm32 has no OS threads (`std::thread::spawn` returns `Unsupported`), so the -/// thread-hop writer above can't exist. The browser build only ever runs over -/// the in-memory VFS, whose writes resolve in a single poll, so we drive the -/// async VFS future to completion inline with `now_or_never` — same VFS, same -/// quota accounting, no thread. -#[cfg(target_family = "wasm")] -struct VfsWriter { - fs: Arc, -} - -#[cfg(target_family = "wasm")] -impl VfsWriter { - fn new(fs: Arc) -> Self { - Self { fs } - } - - /// Perform one write by polling the VFS future once. Returns the VFS result, - /// or `Err(())` if the write would suspend (an async-backed mount, which the - /// browser build does not use). - fn write( - &self, - path: PathBuf, - bytes: Vec, - append: bool, - ) -> Result, ()> { - let fut = async { - if append { - self.fs.append_file(&path, &bytes).await - } else { - self.fs.write_file(&path, &bytes).await - } - }; - fut.now_or_never().ok_or(()) - } -} - -pub(super) struct AwkInterpreter { - pub(super) state: AwkState, - pub(super) output: String, - /// Stderr output buffer for `/dev/stderr` redirection - pub(super) stderr_output: String, - /// Lines of current input file (set before main loop) - pub(super) input_lines: Vec, - /// Current line index within input_lines - pub(super) line_index: usize, - /// User-defined functions - pub(super) functions: HashMap, - /// Current function call depth for recursion limiting - call_depth: usize, - /// Paths already seen as `>`/`>>` redirect targets in this AWK run. - /// First write truncates (for `>`); later writes stream as appends. Also - /// serves as the distinct-target set for the output-target DoS guard. - pub(super) file_truncates: HashSet, - /// Total bytes streamed to real file redirects, excluding discarded devices. - redirected_file_output_bytes: usize, - /// Single reusable writer thread for redirected VFS writes (lazily started - /// on the first redirect). Joined when the interpreter is dropped. - vfs_writer: Option, - /// Cached file inputs for `getline var < file` redirection. - /// Maps normalized resolved path -> (lines, current_position). - /// Maps normalized resolved path -> (lines, current_position, raw_bytes). - file_inputs: HashMap, usize, usize)>, - /// Approximate raw input bytes retained by `file_inputs`. - file_input_bytes: usize, - /// VFS reference for lazy file reads (getline < file). - pub(super) fs: Option>, - /// Working directory for resolving relative paths. - pub(super) cwd: PathBuf, - /// Tracks active state for range patterns (rule index -> is_active). - /// When start pattern matches, range becomes active. When end pattern - /// matches, that line is included but range becomes inactive. - range_active: HashMap, - /// Max iterations for a single loop, inherited from execution limits. - pub(super) max_loop_iterations: usize, - /// Max iterations across all loops of one awk run (nested loops share it). - pub(super) max_total_loop_iterations: usize, - total_loop_iterations: usize, - regex_cache: RuntimeRegexCache, - /// Shared request budget; poisoning is surfaced by the builtin dispatcher. - pub(super) execution_budget: Option, - /// Set once a resource limit aborted the program (see `fatal`). - fatal: bool, - /// Cap on accounted variable bytes; the host's - /// `max_live_intermediate_bytes`. THREAT[TM-DOS-110]. - pub(super) max_state_bytes: usize, - /// Accounted bytes of values parked in user-function call frames. - pinned_bytes: usize, -} - -/// Which matches `replace_checked` substitutes. -#[derive(Clone, Copy)] -enum Replace { - All, - Nth(usize), -} - -/// `format_string` failure: a user-facing diagnostic, or the string cap. -enum FormatError { - Message(String), - TooLarge, -} - -impl AwkInterpreter { - pub(super) fn new() -> Self { - Self { - state: AwkState::default(), - output: String::new(), - stderr_output: String::new(), - input_lines: Vec::new(), - line_index: 0, - functions: HashMap::new(), - file_truncates: HashSet::new(), - redirected_file_output_bytes: 0, - vfs_writer: None, - file_inputs: HashMap::new(), - file_input_bytes: 0, - call_depth: 0, - fs: None, - cwd: PathBuf::from("/"), - range_active: HashMap::new(), - max_loop_iterations: ExecutionLimits::default().max_loop_iterations, - max_total_loop_iterations: ExecutionLimits::default().max_total_loop_iterations, - total_loop_iterations: 0, - regex_cache: RuntimeRegexCache::default(), - execution_budget: None, - fatal: false, - max_state_bytes: ExecutionLimits::default().max_live_intermediate_bytes as usize, - pinned_bytes: 0, - } - } - - pub(super) fn is_fatal(&self) -> bool { - self.fatal - } - - /// THREAT[TM-DOS-110]: a single string may not exceed - /// `AWK_MAX_STRING_BYTES`. Callers check a size before allocating it. - fn string_fits(&mut self, len: usize) -> bool { - if len > AWK_MAX_STRING_BYTES { - self.fatal(&format!( - "string size limit ({AWK_MAX_STRING_BYTES} bytes) exceeded" - )); - return false; - } - true - } - - /// THREAT[TM-DOS-110]: variables, array elements and parked call-frame - /// values together stay within the host's live-bytes limit. Checked at - /// every expression, so growth overshoots by at most one assignment. - /// - /// Decision: a local cap, not an `ExecutionBudgetLease`. A lease that - /// runs out poisons the whole request, while this fails only the awk - /// command (fatal, exit 2) and the script carries on, like every other - /// awk cap (TM-DOS-109). - fn memory_fits(&mut self) -> bool { - if self.fatal { - return false; - } - if self.state.mem_bytes + self.pinned_bytes > self.max_state_bytes { - let msg = format!("memory limit ({} bytes) exceeded", self.max_state_bytes); - self.fatal(&msg); - return false; - } - true - } - - /// `sub`/`gsub`/`gensub` core: substitute matches of `re` in `target`, - /// checking the result against the string cap as it grows (a 10 KB - /// target and replacement would otherwise build 100 MB first). - /// `expand` interprets `$N` group references in `rep`. - fn replace_checked( - &mut self, - re: ®ex::Regex, - target: &str, - rep: &str, - which: Replace, - expand: bool, - ) -> Option<(String, usize)> { - let mut out = String::new(); - let mut last = 0; - let mut count = 0; - for caps in re.captures_iter(target) { - let Some(m) = caps.get(0) else { continue }; - count += 1; - out.push_str(&target[last..m.start()]); - let hit = match which { - Replace::All => true, - Replace::Nth(n) => count == n, - }; - if !hit { - out.push_str(m.as_str()); - } else if expand { - caps.expand(rep, &mut out); - } else { - out.push_str(rep); - } - last = m.end(); - if !self.string_fits(out.len()) { - return None; - } - if matches!(which, Replace::Nth(n) if count >= n) { - break; - } - } - if !self.string_fits(out.len() + target.len() - last) { - return None; - } - out.push_str(&target[last..]); - Some((out, count)) - } - - /// Abort the program because a resource limit was reached. - /// THREAT[TM-DOS-109]: caps are reported, never silent. - /// - /// Decision (#2446): hitting a cap is never silent. Like a gawk fatal - /// error, the message goes to stderr, no further actions (END included) - /// run, and awk exits 2. Output produced so far is kept. - fn fatal(&mut self, msg: &str) -> AwkFlow { - if !self.fatal { - self.fatal = true; - self.stderr_output.push_str("awk: fatal: "); - self.stderr_output.push_str(msg); - self.stderr_output.push('\n'); - } - AwkFlow::Exit(Some(2)) - } - - /// Count one iteration of a loop that has run `iters` times so far. - /// THREAT[TM-DOS-033]: per-loop and whole-program caps, so nested loops - /// cannot multiply past `max_total_loop_iterations`. - fn tick_loop(&mut self, iters: usize) -> Option { - self.total_loop_iterations += 1; - if iters > self.max_loop_iterations { - let msg = format!( - "loop iteration limit ({}) exceeded", - self.max_loop_iterations - ); - return Some(self.fatal(&msg)); - } - if self.total_loop_iterations > self.max_total_loop_iterations { - let msg = format!( - "total loop iteration limit ({}) exceeded", - self.max_total_loop_iterations - ); - return Some(self.fatal(&msg)); - } - None - } - - fn runtime_regex(&mut self, pattern: &str) -> Option { - self.regex_cache.get_or_compile(pattern) - } - - fn resolve_getline_path(&self, path_str: &str) -> String { - let resolved = if path_str.starts_with('/') { - PathBuf::from(path_str) - } else { - vfs_join(&self.cwd, path_str) - }; - normalize_path(&resolved).to_string_lossy().to_string() - } - - /// Load a file into the `file_inputs` cache if not already present. - /// Uses a separate thread + tokio runtime to bridge async VFS → sync context. - /// Returns true on success, false on error. An unreadable file is the - /// normal awk `-1` case; exceeding a getline cap is fatal (#2446). - fn ensure_file_loaded(&mut self, resolved: &str) -> bool { - if self.file_inputs.contains_key(resolved) { - return true; - } - // Guard: cap cache entries and retained bytes before whole-file reads. - if self.file_inputs.len() >= MAX_GETLINE_CACHED_FILES { - self.fatal(&format!( - "getline open file limit ({MAX_GETLINE_CACHED_FILES}) exceeded" - )); - return false; - } - let Some(fs) = &self.fs else { - return false; - }; - let fs = fs.clone(); - let p = PathBuf::from(resolved); - - // `Ok(None)` means the file exceeds MAX_GETLINE_FILE_BYTES. - // Native: spawn a thread with its own runtime to bridge the sync AWK - // evaluator to the async VFS without blocking the outer runtime. - #[cfg(not(target_family = "wasm"))] - let result = std::thread::spawn(move || -> crate::error::Result>> { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let meta = runtime.block_on(fs.stat(&p))?; - if meta.size > MAX_GETLINE_FILE_BYTES as u64 { - return Ok(None); - } - runtime.block_on(fs.read_file(&p)).map(Some) - }) - .join(); - - // wasm32: no threads. Drive the in-memory VFS reads inline (they resolve - // in one poll). The outer Ok mirrors a thread that didn't panic. - #[cfg(target_family = "wasm")] - let result: std::result::Result>>, ()> = Ok((|| { - let meta = match fs.stat(&p).now_or_never() { - Some(m) => m?, - None => return Err(std::io::Error::other("awk getline: vfs read suspended").into()), - }; - if meta.size > MAX_GETLINE_FILE_BYTES as u64 { - return Ok(None); - } - match fs.read_file(&p).now_or_never() { - Some(bytes) => bytes.map(Some), - None => Err(std::io::Error::other("awk getline: vfs read suspended").into()), - } - })( - )); - - let bytes = match result { - Ok(Ok(Some(bytes))) if bytes.len() <= MAX_GETLINE_FILE_BYTES => bytes, - Ok(Ok(_)) => { - self.fatal(&format!( - "getline input file size limit ({MAX_GETLINE_FILE_BYTES} bytes) exceeded" - )); - return false; - } - _ => return false, - }; - // THREAT[TM-DOS-116]: `close()` must not recycle the request-wide - // getline budget. Charge both materialized input and byte-proportional - // work before allocating the decoded text and its line copies. - let loaded = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - if self.execution_budget.as_ref().is_some_and(|budget| { - budget.consume_input(bytes.len()).is_err() || budget.consume_work(loaded).is_err() - }) { - // The builtin dispatcher reports the poisoned shared budget. - self.fatal = true; - return false; - } - let total = self.file_input_bytes.saturating_add(bytes.len()); - if total > MAX_GETLINE_CACHE_BYTES { - self.fatal(&format!( - "getline total input limit ({MAX_GETLINE_CACHE_BYTES} bytes) exceeded" - )); - return false; - } - let text = String::from_utf8_lossy(&bytes).into_owned(); - let lines: Vec = text.lines().map(|l| l.to_string()).collect(); - self.file_inputs - .insert(resolved.to_string(), (lines, 0, bytes.len())); - self.file_input_bytes = total; - true - } - - /// A regex operand: a `/re/` constant gives its pattern text, anything - /// else is evaluated as a dynamic regex string. - fn eval_pattern_arg(&mut self, expr: &AwkExpr) -> String { - match expr { - AwkExpr::Regex(pattern) => pattern.clone(), - other => self.eval_expr(other).as_string(), - } - } - - /// Evaluate an expression as a boolean, with special handling for regex - /// literals: `/regex/` is matched against $0 in boolean context (e.g. && / ||). - fn eval_expr_as_bool(&mut self, expr: &AwkExpr) -> bool { - if let AwkExpr::Regex(pattern) = expr { - let line = self.state.get_field(0).as_string(); - if let Some(re) = self.runtime_regex(pattern) { - return re.is_match(&line); - } - return false; - } - self.eval_expr(expr).as_bool() - } - - fn eval_expr(&mut self, expr: &AwkExpr) -> AwkValue { - if !self.memory_fits() { - return AwkValue::Uninitialized; - } - match expr { - AwkExpr::Number(n) => AwkValue::Number(*n), - AwkExpr::String(s) => AwkValue::String(s.clone()), - AwkExpr::Field(index) => { - let n = self.eval_expr(index).as_number() as usize; - self.state.get_field(n) - } - AwkExpr::Variable(name) => self.state.get_variable(name), - AwkExpr::Assign(name, val) => { - let value = self.eval_expr(val); - self.state.set_variable(name, value.clone()); - value - } - AwkExpr::BinOp(left, op, right) => { - if op == "&&" { - let lb = self.eval_expr_as_bool(left); - if !lb { - return AwkValue::Number(0.0); - } - let rb = self.eval_expr_as_bool(right); - return AwkValue::Number(if rb { 1.0 } else { 0.0 }); - } - if op == "||" { - let lb = self.eval_expr_as_bool(left); - if lb { - return AwkValue::Number(1.0); - } - let rb = self.eval_expr_as_bool(right); - return AwkValue::Number(if rb { 1.0 } else { 0.0 }); - } - - let l = self.eval_expr(left); - let r = if op == "~" || op == "!~" { - AwkValue::String(self.eval_pattern_arg(right)) - } else { - self.eval_expr(right) - }; - - match op.as_str() { - "+" => AwkValue::Number(l.as_number() + r.as_number()), - "-" => AwkValue::Number(l.as_number() - r.as_number()), - "*" => AwkValue::Number(l.as_number() * r.as_number()), - "/" => AwkValue::Number(l.as_number() / r.as_number()), - "%" => AwkValue::Number(l.as_number() % r.as_number()), - "^" => AwkValue::Number(l.as_number().powf(r.as_number())), - "==" => AwkValue::Number(if l.as_string() == r.as_string() { - 1.0 - } else { - 0.0 - }), - "!=" => AwkValue::Number(if l.as_string() != r.as_string() { - 1.0 - } else { - 0.0 - }), - "<" => AwkValue::Number(if l.as_number() < r.as_number() { - 1.0 - } else { - 0.0 - }), - ">" => AwkValue::Number(if l.as_number() > r.as_number() { - 1.0 - } else { - 0.0 - }), - "<=" => AwkValue::Number(if l.as_number() <= r.as_number() { - 1.0 - } else { - 0.0 - }), - ">=" => AwkValue::Number(if l.as_number() >= r.as_number() { - 1.0 - } else { - 0.0 - }), - "~" => { - if let Some(re) = self.runtime_regex(&r.as_string()) { - AwkValue::Number(if re.is_match(&l.as_string()) { - 1.0 - } else { - 0.0 - }) - } else { - AwkValue::Number(0.0) - } - } - "!~" => { - if let Some(re) = self.runtime_regex(&r.as_string()) { - AwkValue::Number(if !re.is_match(&l.as_string()) { - 1.0 - } else { - 0.0 - }) - } else { - AwkValue::Number(1.0) - } - } - "SUBSEP_CONCAT" => { - let subsep = self.state.get_variable("SUBSEP").as_string(); - let (l, r) = (l.as_string(), r.as_string()); - if !self.string_fits(l.len() + subsep.len() + r.len()) { - return AwkValue::Uninitialized; - } - AwkValue::String(format!("{l}{subsep}{r}")) - } - _ => AwkValue::Uninitialized, - } - } - AwkExpr::UnaryOp(op, expr) => match op.as_str() { - "-" => { - let v = self.eval_expr(expr); - AwkValue::Number(-v.as_number()) - } - "!" => { - let b = self.eval_expr_as_bool(expr); - AwkValue::Number(if b { 0.0 } else { 1.0 }) - } - _ => self.eval_expr(expr), - }, - AwkExpr::Concat(parts) => { - let parts: Vec = parts - .iter() - .map(|p| self.eval_expr(p).as_string()) - .collect(); - if !self.string_fits(parts.iter().map(String::len).sum()) { - return AwkValue::Uninitialized; - } - AwkValue::String(parts.concat()) - } - AwkExpr::ArrayAssign(name, key, val) => { - let k = self.eval_expr(key).as_string(); - let v = self.eval_expr(val); - let full_key = format!("{}[{}]", name, k); - self.state.set_variable(&full_key, v.clone()); - v - } - AwkExpr::CompoundArrayAssign(name, key, op, val) => { - let k = self.eval_expr(key).as_string(); - let full_key = format!("{}[{}]", name, k); - let current = self.state.get_variable(&full_key).as_number(); - let rhs = self.eval_expr(val).as_number(); - let result = match op.as_str() { - "+" => current + rhs, - "-" => current - rhs, - "*" => current * rhs, - "/" => current / rhs, - "%" => current % rhs, - _ => rhs, - }; - let v = AwkValue::Number(result); - self.state.set_variable(&full_key, v.clone()); - v - } - AwkExpr::FieldAssign(index, val) => { - let n = self.eval_expr(index).as_number() as usize; - let v = self.eval_expr(val); - if n == 0 { - self.state.set_variable("$0", v.clone()); - } else { - if n > AWK_MAX_FIELD_INDEX { - self.fatal(&format!( - "field index limit ({AWK_MAX_FIELD_INDEX}) exceeded" - )); - return AwkValue::Uninitialized; - } - // The rebuilt $0 must fit before any field is created. - let value_len = v.as_string().len(); - let nf = n.max(self.state.fields.len()); - let kept: usize = self - .state - .fields - .iter() - .enumerate() - .filter(|(i, _)| *i != n - 1) - .map(|(_, f)| f.len()) - .sum(); - let seps = (nf - 1).saturating_mul(self.state.ofs.len()); - if !self.string_fits(kept + value_len + seps) { - return AwkValue::Uninitialized; - } - // Extend fields if needed - while self.state.fields.len() < n { - self.state.fields.push(String::new()); - } - self.state.fields[n - 1] = v.as_string(); - self.state.nf = self.state.fields.len(); - // Rebuild $0 - let new_line = self.state.fields.join(&self.state.ofs); - self.state.set_variable("$0", AwkValue::String(new_line)); - } - v - } - AwkExpr::PostIncrement(name) => { - let current = self.state.get_variable(name).as_number(); - self.state - .set_variable(name, AwkValue::Number(current + 1.0)); - AwkValue::Number(current) // Return old value - } - AwkExpr::PostDecrement(name) => { - let current = self.state.get_variable(name).as_number(); - self.state - .set_variable(name, AwkValue::Number(current - 1.0)); - AwkValue::Number(current) // Return old value - } - AwkExpr::PreIncrement(name) => { - let current = self.state.get_variable(name).as_number(); - let new_val = current + 1.0; - self.state.set_variable(name, AwkValue::Number(new_val)); - AwkValue::Number(new_val) // Return new value - } - AwkExpr::PreDecrement(name) => { - let current = self.state.get_variable(name).as_number(); - let new_val = current - 1.0; - self.state.set_variable(name, AwkValue::Number(new_val)); - AwkValue::Number(new_val) // Return new value - } - AwkExpr::InArray(key, arr_name) => { - let k = self.eval_expr(key).as_string(); - let full_key = format!("{}[{}]", arr_name, k); - let exists = !matches!(self.state.get_variable(&full_key), AwkValue::Uninitialized); - AwkValue::Number(if exists { 1.0 } else { 0.0 }) - } - AwkExpr::FuncCall(name, args) => self.call_function(name, args), - AwkExpr::Regex(pattern) => { - // A standalone /regex/ is `$0 ~ /regex/` (`x = /a/`, `if (/a/)`). - // Where it names a pattern (`~` operand, sub/gsub/match/split/ - // gensub argument) callers read it via `eval_pattern_arg`. - let line = self.state.get_field(0).as_string(); - let hit = self - .runtime_regex(pattern) - .is_some_and(|re| re.is_match(&line)); - AwkValue::Number(if hit { 1.0 } else { 0.0 }) - } - AwkExpr::Match(expr, pattern) => { - let s = self.eval_expr(expr).as_string(); - if let Some(re) = self.runtime_regex(pattern) { - AwkValue::Number(if re.is_match(&s) { 1.0 } else { 0.0 }) - } else { - AwkValue::Number(0.0) - } - } - AwkExpr::GetlineFile { var, file } => { - let path_str = self.eval_expr(file).as_string(); - let resolved = self.resolve_getline_path(&path_str); - - if !self.ensure_file_loaded(&resolved) { - return AwkValue::Number(-1.0); - } - - let entry = self.file_inputs.get_mut(&resolved).unwrap(); - if entry.1 < entry.0.len() { - let line = entry.0[entry.1].clone(); - entry.1 += 1; - match var { - Some(v) => { - self.state - .variables - .insert(v.clone(), AwkValue::String(line)); - } - None => { - self.state.set_line(&line); - } - } - AwkValue::Number(1.0) // success - } else { - AwkValue::Number(0.0) // EOF - } - } - } - } - - fn call_function(&mut self, name: &str, args: &[AwkExpr]) -> AwkValue { - match name { - "length" => { - if args.is_empty() { - AwkValue::Number(self.state.get_field(0).as_string().len() as f64) - } else { - // Check if the argument is an array name - if so, return element count - if let AwkExpr::Variable(ref arr_name) = args[0] { - let prefix = format!("{}[", arr_name); - let count = self - .state - .variables - .keys() - .filter(|k| k.starts_with(&prefix)) - .count(); - if count > 0 { - return AwkValue::Number(count as f64); - } - } - AwkValue::Number(self.eval_expr(&args[0]).as_string().len() as f64) - } - } - "substr" => { - if args.len() < 2 { - return AwkValue::Uninitialized; - } - let s = self.eval_expr(&args[0]).as_string(); - let start = (self.eval_expr(&args[1]).as_number() as usize).saturating_sub(1); - let len = if args.len() > 2 { - self.eval_expr(&args[2]).as_number() as usize - } else { - s.len() - }; - let end = (start + len).min(s.len()); - AwkValue::String(s.chars().skip(start).take(end - start).collect()) - } - "index" => { - if args.len() < 2 { - return AwkValue::Number(0.0); - } - let s = self.eval_expr(&args[0]).as_string(); - let t = self.eval_expr(&args[1]).as_string(); - match s.find(&t) { - Some(i) => AwkValue::Number((i + 1) as f64), - None => AwkValue::Number(0.0), - } - } - "split" => { - if args.len() < 2 { - return AwkValue::Number(0.0); - } - let s = self.eval_expr(&args[0]).as_string(); - // A regex constant (`/re/`) is always an ERE; a string follows - // the same rules as FS (see `FieldSep`). - let (sep, force_regex) = match args.get(2) { - Some(AwkExpr::Regex(pattern)) => (pattern.clone(), true), - Some(expr) => (self.eval_expr(expr).as_string(), false), - None => (self.state.fs.clone(), false), - }; - let regex = if (force_regex && !sep.is_empty()) || FieldSep::is_regex(&sep) { - self.runtime_regex(&sep) - } else { - None - }; - let parts = FieldSep::classify(&sep, regex.as_ref()).split(&s); - - // Store in array variable, replacing any previous contents. - if let AwkExpr::Variable(arr_name) = &args[1] { - self.state.clear_array(arr_name); - for (i, part) in parts.iter().enumerate() { - // One call can create millions of elements. - if !self.memory_fits() { - return AwkValue::Uninitialized; - } - let key = format!("{}[{}]", arr_name, i + 1); - self.state - .set_variable(&key, AwkValue::String(part.to_string())); - } - } - - AwkValue::Number(parts.len() as f64) - } - "sprintf" => { - if args.is_empty() { - return AwkValue::String(String::new()); - } - let format = self.eval_expr(&args[0]).as_string(); - let values: Vec = args[1..].iter().map(|a| self.eval_expr(a)).collect(); - // THREAT[TM-DOS-109]: expression errors must poison the run; - // returning an empty string with exit 0 hides a failed cap. - match self.format_string(&format, &values) { - Ok(s) => AwkValue::String(s), - Err(FormatError::Message(e)) => { - self.fatal(&e); - AwkValue::Uninitialized - } - Err(FormatError::TooLarge) => { - self.string_fits(usize::MAX); - AwkValue::Uninitialized - } - } - } - "toupper" => { - if args.is_empty() { - return AwkValue::Uninitialized; - } - AwkValue::String(self.eval_expr(&args[0]).as_string().to_uppercase()) - } - "tolower" => { - if args.is_empty() { - return AwkValue::Uninitialized; - } - AwkValue::String(self.eval_expr(&args[0]).as_string().to_lowercase()) - } - "gsub" | "sub" => { - // gsub(regexp, replacement, target) - if args.len() < 2 { - return AwkValue::Number(0.0); - } - let pattern = self.eval_pattern_arg(&args[0]); - let replacement = self.eval_expr(&args[1]).as_string(); - - let target_expr = if args.len() > 2 { - args[2].clone() - } else { - AwkExpr::Field(Box::new(AwkExpr::Number(0.0))) - }; - - let target = self.eval_expr(&target_expr).as_string(); - - if let Some(re) = self.runtime_regex(&pattern) { - let which = if name == "gsub" { - Replace::All - } else { - Replace::Nth(1) - }; - let Some((result, count)) = - self.replace_checked(&re, &target, &replacement, which, true) - else { - return AwkValue::Number(0.0); - }; - - // Update the target variable or field - match &target_expr { - AwkExpr::Variable(name) => { - self.state.set_variable(name, AwkValue::String(result)); - } - AwkExpr::Field(index) => { - let n = self.eval_expr(index).as_number() as usize; - if n == 0 { - // $0 is stored as a variable - self.state.set_variable("$0", AwkValue::String(result)); - } - // For other fields, we'd need to update the fields vec - // and rebuild $0, but for now we just support $0 - } - _ => {} - } - - AwkValue::Number(count as f64) - } else { - AwkValue::Number(0.0) - } - } - "int" => { - if args.is_empty() { - return AwkValue::Number(0.0); - } - AwkValue::Number(self.eval_expr(&args[0]).as_number().trunc()) - } - "sqrt" => { - if args.is_empty() { - return AwkValue::Number(0.0); - } - AwkValue::Number(self.eval_expr(&args[0]).as_number().sqrt()) - } - "sin" => { - if args.is_empty() { - return AwkValue::Number(0.0); - } - AwkValue::Number(self.eval_expr(&args[0]).as_number().sin()) - } - "cos" => { - if args.is_empty() { - return AwkValue::Number(0.0); - } - AwkValue::Number(self.eval_expr(&args[0]).as_number().cos()) - } - "log" => { - if args.is_empty() { - return AwkValue::Number(0.0); - } - AwkValue::Number(self.eval_expr(&args[0]).as_number().ln()) - } - "exp" => { - if args.is_empty() { - return AwkValue::Number(0.0); - } - AwkValue::Number(self.eval_expr(&args[0]).as_number().exp()) - } - "match" => { - if args.len() < 2 { - return AwkValue::Number(0.0); - } - let s = self.eval_expr(&args[0]).as_string(); - let pattern = self.eval_pattern_arg(&args[1]); - // Extract capture array name from 3rd arg (gawk extension) - let arr_name = if args.len() >= 3 { - if let AwkExpr::Variable(name) = &args[2] { - Some(name.clone()) - } else { - None - } - } else { - None - }; - if let Some(re) = self.runtime_regex(&pattern) { - if let Some(caps) = re.captures(&s) { - let m = caps.get(0).unwrap(); - let rstart = m.start() + 1; // awk is 1-indexed - let rlength = m.end() - m.start(); - self.state - .set_variable("RSTART", AwkValue::Number(rstart as f64)); - self.state - .set_variable("RLENGTH", AwkValue::Number(rlength as f64)); - // Populate capture array if 3rd arg provided - if let Some(ref arr) = arr_name { - // arr[0] = entire match - let full_key = format!("{}[0]", arr); - self.state - .set_variable(&full_key, AwkValue::String(m.as_str().to_string())); - // arr[1..N] = capture groups - for i in 1..caps.len() { - let key = format!("{}[{}]", arr, i); - let val = caps - .get(i) - .map(|c| c.as_str().to_string()) - .unwrap_or_default(); - self.state.set_variable(&key, AwkValue::String(val)); - } - } - AwkValue::Number(rstart as f64) - } else { - self.state.set_variable("RSTART", AwkValue::Number(0.0)); - self.state.set_variable("RLENGTH", AwkValue::Number(-1.0)); - AwkValue::Number(0.0) - } - } else { - AwkValue::Number(0.0) - } - } - "gensub" => { - // gensub(regexp, replacement, how [, target]) - if args.len() < 3 { - return AwkValue::Uninitialized; - } - let pattern = self.eval_pattern_arg(&args[0]); - let replacement = self.eval_expr(&args[1]).as_string(); - let how = self.eval_expr(&args[2]).as_string(); - let target = if args.len() > 3 { - self.eval_expr(&args[3]).as_string() - } else { - self.state.get_field(0).as_string() - }; - if let Some(re) = self.runtime_regex(&pattern) { - let (which, expand) = if how == "g" || how == "G" { - (Replace::All, true) - } else { - // Replace nth occurrence (default 1st) - (Replace::Nth(how.parse::().unwrap_or(1)), false) - }; - match self.replace_checked(&re, &target, &replacement, which, expand) { - Some((result, _)) => AwkValue::String(result), - None => AwkValue::Uninitialized, - } - } else { - AwkValue::String(target) - } - } - "__getline" => { - // Plain getline as expression — advance to next input line, return 1/0 - self.line_index += 1; - if self.line_index < self.input_lines.len() { - let line = self.input_lines[self.line_index].clone(); - self.state.set_line(&line); - AwkValue::Number(1.0) - } else { - AwkValue::Number(0.0) - } - } - "__array_access" => { - // Internal function for array indexing: arr[index] - if args.len() < 2 { - return AwkValue::Uninitialized; - } - let arr_name = if let AwkExpr::Variable(name) = &args[0] { - name.clone() - } else { - return AwkValue::Uninitialized; - }; - let index = self.eval_expr(&args[1]); - let key = format!("{}[{}]", arr_name, index.as_string()); - self.state.get_variable(&key) - } - "__ternary" => { - // Ternary operator: cond ? then : else - if args.len() < 3 { - return AwkValue::Uninitialized; - } - let cond = self.eval_expr(&args[0]); - if cond.as_bool() { - self.eval_expr(&args[1]) - } else { - self.eval_expr(&args[2]) - } - } - "close" => { - // close(name): release a `getline < file` input (freeing its - // share of the getline caps) and make the next `>` write to - // the same path truncate again. 0 if something was open. - let Some(arg) = args.first() else { - return AwkValue::Number(-1.0); - }; - let name = self.eval_expr(arg).as_string(); - let resolved = self.resolve_getline_path(&name); - let mut closed = false; - if let Some((_, _, bytes)) = self.file_inputs.remove(&resolved) { - self.file_input_bytes = self.file_input_bytes.saturating_sub(bytes); - closed = true; - } - let out_key = if name.starts_with('/') { - PathBuf::from(&name) - } else { - vfs_join(&self.cwd, &name) - } - .to_string_lossy() - .into_owned(); - closed |= self.file_truncates.remove(&out_key); - AwkValue::Number(if closed { 0.0 } else { -1.0 }) - } - _ => { - // Check for user-defined function - if let Some(func) = self.functions.get(name).cloned() { - self.call_user_function(&func, args) - } else { - AwkValue::Uninitialized - } - } - } - } - - fn call_user_function(&mut self, func: &AwkFunctionDef, args: &[AwkExpr]) -> AwkValue { - // THREAT[TM-DOS-027]: Limit recursion depth to prevent stack overflow - if self.call_depth >= MAX_AWK_CALL_DEPTH { - self.fatal(&format!( - "function call depth limit ({MAX_AWK_CALL_DEPTH}) exceeded" - )); - return AwkValue::Uninitialized; - } - self.call_depth += 1; - - // Save current local variables that will be shadowed. Stored values - // move out of the map and stay accounted as pinned bytes, so deep - // recursion cannot duplicate memory past the cap (TM-DOS-110). - let mut saved: Vec<(String, AwkValue, bool)> = Vec::new(); - for param in &func.params { - match self.state.remove_var(param) { - Some(val) => { - self.pinned_bytes += super::var_cost(param, &val); - saved.push((param.clone(), val, true)); - } - None => saved.push((param.clone(), self.state.get_variable(param), false)), - } - } - - // Bind arguments to parameters - for (i, param) in func.params.iter().enumerate() { - let val = if i < args.len() { - self.eval_expr(&args[i]) - } else { - AwkValue::Uninitialized - }; - self.state.set_variable(param, val); - } - - // Execute function body, capture return value - let mut return_value = AwkValue::Uninitialized; - for action in &func.body.clone() { - match self.exec_action(action) { - AwkFlow::Return(val) => { - return_value = val; - break; - } - AwkFlow::Exit(_) => break, - _ => {} - } - } - - // Restore saved variables - for (name, val, pinned) in saved { - if pinned { - self.pinned_bytes -= super::var_cost(&name, &val); - } - self.state.set_variable(&name, val); - } - - self.call_depth -= 1; - return_value - } - - /// Max width/precision for format specifiers to prevent memory exhaustion - const MAX_FORMAT_WIDTH: usize = MAX_FORMAT_WIDTH; - - fn format_string( - &self, - format: &str, - values: &[AwkValue], - ) -> std::result::Result { - let mut result = String::new(); - let mut chars = format.chars().peekable(); - let mut value_idx = 0; - - while let Some(c) = chars.next() { - if c == '\\' { - // Handle escape sequences in format strings - match chars.peek() { - Some('n') => { - chars.next(); - result.push('\n'); - } - Some('t') => { - chars.next(); - result.push('\t'); - } - Some('r') => { - chars.next(); - result.push('\r'); - } - Some('\\') => { - chars.next(); - result.push('\\'); - } - _ => result.push('\\'), - } - } else if c == '%' { - if chars.peek() == Some(&'%') { - chars.next(); - result.push('%'); - continue; - } - - // Parse format specifier: %[flags][width][.precision]type - let mut left_align = false; - let mut zero_pad = false; - let mut plus_sign = false; - let mut width: Option = None; - let mut precision: Option = None; - let mut conversion = ' '; - - // Parse flags - loop { - match chars.peek() { - Some(&'-') => { - left_align = true; - chars.next(); - } - Some(&'0') if width.is_none() => { - zero_pad = true; - chars.next(); - } - Some(&'+') => { - plus_sign = true; - chars.next(); - } - _ => break, - } - } - - // Parse width - let mut w = String::new(); - while let Some(&c) = chars.peek() { - if c.is_ascii_digit() { - w.push(c); - chars.next(); - } else { - break; - } - } - if !w.is_empty() - && let Ok(w_val) = w.parse::() - { - if w_val > Self::MAX_FORMAT_WIDTH { - return Err(FormatError::Message(format!( - "format width {} exceeds maximum ({})", - w_val, - Self::MAX_FORMAT_WIDTH - ))); - } - width = Some(w_val); - } - - // Parse precision - if chars.peek() == Some(&'.') { - chars.next(); - let mut p = String::new(); - while let Some(&c) = chars.peek() { - if c.is_ascii_digit() { - p.push(c); - chars.next(); - } else { - break; - } - } - precision = if p.is_empty() { - Some(0) - } else if let Ok(p_val) = p.parse::() { - if p_val > Self::MAX_FORMAT_WIDTH { - return Err(FormatError::Message(format!( - "format precision {} exceeds maximum ({})", - p_val, - Self::MAX_FORMAT_WIDTH - ))); - } - Some(p_val) - } else { - None - }; - } - - // Parse conversion character - if let Some(&c) = chars.peek() - && c.is_ascii_alphabetic() - { - conversion = c; - chars.next(); - } - - if value_idx < values.len() { - let val = &values[value_idx]; - value_idx += 1; - - let formatted = match conversion { - 'd' | 'i' => { - let n = val.as_number() as i64; - if plus_sign && n >= 0 { - format!("+{}", n) - } else { - format!("{}", n) - } - } - 'f' => { - let n = val.as_number(); - let prec = precision.unwrap_or(6); - format!("{:.prec$}", n) - } - 'g' => { - let n = val.as_number(); - let prec = precision.unwrap_or(6); - // %g: use shorter of %e or %f, strip trailing zeros - let s = format!("{:.prec$e}", n); - let f = format!("{:.prec$}", n); - if s.len() < f.len() { s } else { f } - } - 'e' | 'E' => { - let n = val.as_number(); - let prec = precision.unwrap_or(6); - format!("{:.prec$e}", n) - } - 's' => { - let mut s = val.as_string(); - if let Some(p) = precision { - s = s.chars().take(p).collect(); - } - s - } - 'c' => { - // %c: print character from ASCII code or first char of string - let n = val.as_number(); - if n > 0.0 && n < 128.0 { - String::from(n as u8 as char) - } else { - let s = val.as_string(); - s.chars().next().map(String::from).unwrap_or_default() - } - } - 'x' | 'X' => { - let n = val.as_number() as i64; - if conversion == 'X' { - format!("{:X}", n) - } else { - format!("{:x}", n) - } - } - 'o' => { - let n = val.as_number() as i64; - format!("{:o}", n) - } - _ => val.as_string(), - }; - - // Apply width and alignment - if let Some(w) = width { - if formatted.len() < w { - let padding = w - formatted.len(); - if left_align { - result.push_str(&formatted); - for _ in 0..padding { - result.push(' '); - } - } else if zero_pad - && matches!(conversion, 'd' | 'i' | 'f' | 'x' | 'X' | 'o') - { - for _ in 0..padding { - result.push('0'); - } - result.push_str(&formatted); - } else { - for _ in 0..padding { - result.push(' '); - } - result.push_str(&formatted); - } - } else { - result.push_str(&formatted); - } - } else { - result.push_str(&formatted); - } - } - } else { - result.push(c); - } - // THREAT[TM-DOS-110]: each conversion adds at most - // MAX_FORMAT_WIDTH (or one value); stop before the next one. - if result.len() > AWK_MAX_STRING_BYTES { - return Err(FormatError::TooLarge); - } - } - - Ok(result) - } - - /// Total bytes accounted across all output streams (O(1): stdout/stderr - /// lengths plus the running streamed-redirect counter). - fn total_output_bytes(&self) -> usize { - self.output.len() + self.stderr_output.len() + self.redirected_file_output_bytes - } - - fn has_output_capacity(&self, len: usize) -> bool { - len <= MAX_AWK_OUTPUT_BYTES.saturating_sub(self.total_output_bytes()) - } - - /// Stream text to stdout, stderr, or the VFS-backed redirect target. - /// Returns `false` if output limits or VFS validation reject the write. - fn write_output(&mut self, text: &str, target: &Option) -> bool { - if !self.has_output_capacity(text.len()) { - self.stderr_output - .push_str("awk: output limit exceeded (max 10MB)\n"); - return false; - } - match target { - None => { - self.output.push_str(text); - true - } - Some(AwkOutputTarget::Truncate(expr)) | Some(AwkOutputTarget::Append(expr)) => { - let path = self.eval_expr(expr).as_string(); - // Intercept special devices before VFS work so /dev/null never buffers. - if path == "/dev/null" { - true - } else if path == "/dev/stderr" { - self.stderr_output.push_str(text); - true - } else if path == "/dev/stdout" { - self.output.push_str(text); - true - } else { - let append = matches!(target, Some(AwkOutputTarget::Append(_))); - self.write_file_output(&path, text, append) - } - } - } - } - - /// Write redirected output immediately through the VFS so sandbox quotas are - /// enforced during AWK execution instead of after unbounded buffering. - fn write_file_output(&mut self, path: &str, text: &str, append: bool) -> bool { - let resolved = if path.starts_with('/') { - PathBuf::from(path) - } else { - vfs_join(&self.cwd, path) - }; - let key = resolved.to_string_lossy().into_owned(); - // Bound the number of distinct redirect targets (DoS guard, parity with - // the previous buffered implementation). file_truncates doubles as the - // distinct-target set, so reject new targets before inserting them. - if !self.file_truncates.contains(&key) - && self.file_truncates.len() >= MAX_AWK_OUTPUT_TARGETS - { - self.stderr_output - .push_str("awk: too many output redirection targets\n"); - return false; - } - let fs = match &self.fs { - Some(fs) => fs.clone(), - None => { - self.stderr_output - .push_str("awk: no filesystem available\n"); - return false; - } - }; - let bytes = text.as_bytes().to_vec(); - // First write to a path truncates (for `>`); later writes append. `>>` - // always appends. insert() returns false when the path was already seen. - let first_write = self.file_truncates.insert(key); - let append = append || !first_write; - - // Start the single reusable writer thread on first use, then dispatch - // every redirected write through it (one thread/runtime per AWK run, not - // per write). - let writer = self.vfs_writer.get_or_insert_with(|| VfsWriter::new(fs)); - match writer.write(resolved, bytes, append) { - Ok(Ok(())) => { - self.redirected_file_output_bytes += text.len(); - true - } - Ok(Err(e)) => { - self.stderr_output - .push_str(&format!("awk: cannot write {path}: {e}\n")); - false - } - Err(()) => { - self.stderr_output - .push_str(&format!("awk: cannot write {path}: write worker failed\n")); - false - } - } - } - - /// Execute action. Returns flow control signal. - pub(super) fn exec_action(&mut self, action: &AwkAction) -> AwkFlow { - if self - .execution_budget - .as_ref() - .is_some_and(|budget| budget.consume_work(1).is_err()) - { - // The builtin dispatcher reports the poisoned budget. - self.fatal = true; - } - if self.fatal || !self.memory_fits() { - return AwkFlow::Exit(Some(2)); - } - match action { - AwkAction::Print(exprs, target) => { - let parts: Vec = exprs - .iter() - .map(|e| self.eval_expr(e).as_string()) - .collect(); - if self.fatal { - // A limit fired while evaluating the arguments. - return AwkFlow::Exit(Some(2)); - } - let len = parts.iter().map(String::len).sum::() - + parts.len().saturating_sub(1) * self.state.ofs.len(); - if !self.string_fits(len) { - return AwkFlow::Exit(Some(2)); - } - let mut text = parts.join(&self.state.ofs); - text.push_str(&self.state.ors); - if !self.write_output(&text, target) { - // write_output already reported why; stop like a fatal error. - self.fatal = true; - return AwkFlow::Exit(Some(2)); - } - AwkFlow::Continue - } - AwkAction::Printf(format_expr, args, target) => { - let format_str = self.eval_expr(format_expr).as_string(); - let values: Vec = args.iter().map(|a| self.eval_expr(a)).collect(); - if self.fatal { - return AwkFlow::Exit(Some(2)); - } - match self.format_string(&format_str, &values) { - Ok(text) => { - if !self.write_output(&text, target) { - // write_output already reported why; stop like a fatal error. - self.fatal = true; - return AwkFlow::Exit(Some(2)); - } - AwkFlow::Continue - } - Err(FormatError::Message(e)) => self.fatal(&e), - Err(FormatError::TooLarge) => { - self.string_fits(usize::MAX); - AwkFlow::Exit(Some(2)) - } - } - } - AwkAction::Assign(name, expr) => { - let value = self.eval_expr(expr); - self.state.set_variable(name, value); - AwkFlow::Continue - } - AwkAction::ArrayAssign(name, key, val) => { - let k = self.eval_expr(key).as_string(); - let v = self.eval_expr(val); - let full_key = format!("{}[{}]", name, k); - self.state.set_variable(&full_key, v); - AwkFlow::Continue - } - AwkAction::If(cond, then_actions, else_actions) => { - let actions = if self.eval_expr(cond).as_bool() { - then_actions - } else { - else_actions - }; - for action in actions { - match self.exec_action(action) { - AwkFlow::Continue => {} - flow => return flow, - } - } - AwkFlow::Continue - } - AwkAction::While(cond, actions) => { - let mut iters = 0; - while self.eval_expr(cond).as_bool() { - iters += 1; - if let Some(flow) = self.tick_loop(iters) { - return flow; - } - let mut do_break = false; - for action in actions { - match self.exec_action(action) { - AwkFlow::Continue => {} - AwkFlow::Break => { - do_break = true; - break; - } - AwkFlow::LoopContinue => break, - flow => return flow, - } - } - if do_break { - break; - } - } - AwkFlow::Continue - } - AwkAction::DoWhile(cond, actions) => { - let mut iters = 0; - loop { - iters += 1; - if let Some(flow) = self.tick_loop(iters) { - return flow; - } - let mut do_break = false; - for action in actions { - match self.exec_action(action) { - AwkFlow::Continue => {} - AwkFlow::Break => { - do_break = true; - break; - } - AwkFlow::LoopContinue => break, - flow => return flow, - } - } - if do_break || !self.eval_expr(cond).as_bool() { - break; - } - } - AwkFlow::Continue - } - AwkAction::For(init, cond, update, actions) => { - self.exec_action(init); - let mut iters = 0; - while self.eval_expr(cond).as_bool() { - iters += 1; - if let Some(flow) = self.tick_loop(iters) { - return flow; - } - let mut do_break = false; - for action in actions { - match self.exec_action(action) { - AwkFlow::Continue => {} - AwkFlow::Break => { - do_break = true; - break; - } - AwkFlow::LoopContinue => break, - flow => return flow, - } - } - if do_break { - break; - } - self.exec_action(update); - } - AwkFlow::Continue - } - AwkAction::ForIn(var, arr_name, actions) => { - // Collect array keys matching the pattern arr_name[*] - let prefix = format!("{}[", arr_name); - let mut keys: Vec = self - .state - .variables - .keys() - .filter(|k| k.starts_with(&prefix) && k.ends_with(']')) - .map(|k| k[prefix.len()..k.len() - 1].to_string()) - .collect(); - // Sort for deterministic iteration: numeric keys first, then lexical - keys.sort_by(|a, b| match (a.parse::(), b.parse::()) { - (Ok(na), Ok(nb)) => na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal), - _ => a.cmp(b), - }); - - let mut iters = 0; - for key in keys { - iters += 1; - if let Some(flow) = self.tick_loop(iters) { - return flow; - } - self.state.set_variable(var, AwkValue::String(key)); - let mut do_break = false; - for action in actions { - match self.exec_action(action) { - AwkFlow::Continue => {} - AwkFlow::Break => { - do_break = true; - break; - } - AwkFlow::LoopContinue => break, - flow => return flow, - } - } - if do_break { - break; - } - } - AwkFlow::Continue - } - AwkAction::Delete(arr_name, key) => { - let k = self.eval_expr(key).as_string(); - if k == "*" { - self.state.clear_array(arr_name); - } else { - let full_key = format!("{}[{}]", arr_name, k); - self.state.remove_var(&full_key); - } - AwkFlow::Continue - } - AwkAction::Next => AwkFlow::Next, - AwkAction::Getline => { - // Advance to next input line and update $0, NR, NF, FNR - self.line_index += 1; - if self.line_index < self.input_lines.len() { - let line = self.input_lines[self.line_index].clone(); - self.state.set_line(&line); - } - AwkFlow::Continue - } - AwkAction::GetlineFile { var, file } => { - // Read next line from file (action context — return value discarded). - let path_str = self.eval_expr(file).as_string(); - let resolved = self.resolve_getline_path(&path_str); - - if !self.ensure_file_loaded(&resolved) { - return AwkFlow::Continue; - } - - let entry = self.file_inputs.get_mut(&resolved).unwrap(); - if entry.1 < entry.0.len() { - let line = entry.0[entry.1].clone(); - entry.1 += 1; - match var { - Some(v) => { - self.state - .variables - .insert(v.clone(), AwkValue::String(line)); - } - None => { - self.state.set_line(&line); - } - } - } - AwkFlow::Continue - } - AwkAction::Break => AwkFlow::Break, - AwkAction::Continue => AwkFlow::LoopContinue, - AwkAction::Exit(expr) => { - let code = expr.as_ref().map(|e| self.eval_expr(e).as_number() as i32); - AwkFlow::Exit(code) - } - AwkAction::Return(expr) => { - let val = expr - .as_ref() - .map(|e| self.eval_expr(e)) - .unwrap_or(AwkValue::Uninitialized); - AwkFlow::Return(val) - } - AwkAction::Expression(expr) => { - self.eval_expr(expr); - AwkFlow::Continue - } - } - } - - fn matches_pattern(&mut self, pattern: &AwkPattern) -> bool { - match pattern { - AwkPattern::Regex(re) => { - let line = self.state.get_field(0).as_string(); - re.is_match(&line) - } - AwkPattern::Expression(expr) => self.eval_expr(expr).as_bool(), - // Range patterns are handled specially via matches_pattern_with_index - // which tracks state. This arm shouldn't normally be reached for ranges - // in the main loop, but handle it defensively. - AwkPattern::Range(_, _) => false, - } - } - - /// Check if a rule's pattern matches, with range state tracking by rule index. - pub(super) fn matches_pattern_with_index( - &mut self, - pattern: &AwkPattern, - rule_idx: usize, - ) -> bool { - match pattern { - AwkPattern::Range(start, end) => { - let active = *self.range_active.get(&rule_idx).unwrap_or(&false); - if active { - // Already in range — check if end pattern matches - if self.matches_pattern(end) { - // End pattern matched: include this line, deactivate range - self.range_active.insert(rule_idx, false); - } - true - } else { - // Not in range — check if start pattern matches - if self.matches_pattern(start) { - // If end also matches this same line, range closes immediately. - let end_matches = self.matches_pattern(end); - self.range_active.insert(rule_idx, !end_matches); - true - } else { - false - } - } - } - other => self.matches_pattern(other), - } - } -} - -#[cfg(test)] -mod regex_cache_tests { - use super::*; - - #[test] - fn repeated_match_expression_compiles_regex_once() { - let mut interp = AwkInterpreter::new(); - interp.state.set_line("bytes=123"); - let expr = AwkExpr::BinOp( - Box::new(AwkExpr::Field(Box::new(AwkExpr::Number(0.0)))), - "~".to_string(), - Box::new(AwkExpr::Regex("^bytes=".to_string())), - ); - - for _ in 0..300_000 { - assert!(interp.eval_expr(&expr).as_bool()); - } - - assert_eq!(interp.regex_cache.compile_count(), 1); - } - - #[test] - fn repeated_invalid_regex_compiles_once() { - let mut interp = AwkInterpreter::new(); - - for _ in 0..100 { - assert!(interp.runtime_regex("[").is_none()); - } - - assert_eq!(interp.regex_cache.compile_count(), 1); - } -} diff --git a/crates/bashkit/src/builtins/awk/io.rs b/crates/bashkit/src/builtins/awk/io.rs new file mode 100644 index 000000000..361fb1f2e --- /dev/null +++ b/crates/bashkit/src/builtins/awk/io.rs @@ -0,0 +1,792 @@ +//! awk input and output: records, `getline`, redirections, pipes and +//! `system()`. +//! +//! Decisions: +//! - Main input follows `ARGV` at run time (so BEGIN can change it): +//! `var=value` operands are assignments, `-` is stdin, a missing file is +//! a gawk fatal error. +//! - Commands (`system()`, `print | cmd`, `cmd | getline`) run in the +//! sandbox shell through the builtin's execution-plan driver: the +//! evaluator posts a request on a shared slot and waits; the driver +//! hands it to the interpreter and returns the result. Without a driver +//! (direct `execute`), commands fail with a message. +//! - `print | cmd` buffers the command's stdin and runs it at `close()` or +//! exit (the command sees all input at once, as with a real pipe). +//! - File redirections are buffered in small chunks and flushed through the +//! VFS, so sandbox quotas apply while awk runs. +//! - THREAT[TM-DOS-028]: output (stdout, stderr, files, pipes) is capped at +//! `AWK_MAX_OUTPUT_BYTES` and distinct redirection targets at +//! `AWK_MAX_OUTPUT_TARGETS`. +//! - THREAT[TM-DOS-116]: `getline` inputs are capped per file, in number +//! and in retained bytes; every load is charged to the request budget, +//! which `close()` cannot refund. + +use std::collections::HashMap; +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context as TaskContext, Poll}; + +use super::ast::{Expr, GetlineSrc, Redirect, special}; +use super::interp::{Cell, Elem, Interp, R, Unwind, next_char}; +use super::regex::AwkRegex; +use super::value::Value; +use crate::builtins::Date; +use crate::builtins::limits::{ + AWK_MAX_GETLINE_CACHE_BYTES, AWK_MAX_GETLINE_CACHED_FILES, AWK_MAX_GETLINE_FILE_BYTES, + AWK_MAX_OUTPUT_BYTES, AWK_MAX_OUTPUT_TARGETS, +}; +use crate::fs::{FileSystem, vfs_join}; +use crate::interpreter::ExecResult; + +/// File redirections flush once this many bytes are buffered. +const FILE_FLUSH_BYTES: usize = 8 * 1024; + +/// Record separator mode. +#[derive(Clone)] +pub(super) enum RsMode { + Newline, + Char(char), + /// `RS = ""`: records separated by blank lines. + Paragraph, + Regex(Arc), +} + +/// An input source held in memory. +pub(super) struct Reader { + data: String, + pos: usize, + /// Exit status of the command that produced it (`cmd | getline`). + status: i32, +} + +impl Reader { + pub(super) fn new(data: String) -> Self { + Reader { + data, + pos: 0, + status: 0, + } + } + + /// Next record and its terminator (`RT`). + pub(super) fn read(&mut self, rs: &RsMode) -> Option<(String, String)> { + let d = &self.data; + let p = self.pos; + let len = d.len(); + match rs { + RsMode::Newline | RsMode::Char(_) => { + if p >= len { + return None; + } + let sep = match rs { + RsMode::Char(c) => *c, + _ => '\n', + }; + match d[p..].find(sep) { + Some(i) => { + let rec = d[p..p + i].to_string(); + self.pos = p + i + sep.len_utf8(); + Some((rec, sep.to_string())) + } + None => { + let rec = d[p..].to_string(); + self.pos = len; + Some((rec, String::new())) + } + } + } + RsMode::Paragraph => { + let mut s = p; + while s < len && d.as_bytes()[s] == b'\n' { + s += 1; + } + if s >= len { + self.pos = len; + return None; + } + match d[s..].find("\n\n") { + Some(i) => { + let end = s + i; + let mut j = end; + while j < len && d.as_bytes()[j] == b'\n' { + j += 1; + } + let r = (d[s..end].to_string(), d[end..j].to_string()); + self.pos = j; + Some(r) + } + None => { + let rest = &d[s..]; + let trimmed = rest.trim_end_matches('\n'); + let r = (trimmed.to_string(), rest[trimmed.len()..].to_string()); + self.pos = len; + Some(r) + } + } + } + RsMode::Regex(re) => { + if p >= len { + return None; + } + let mut from = p; + while from <= len { + match re.find_at(d, from) { + Some((s, e)) if s == e => from = next_char(d, s), + Some((s, e)) => { + let r = (d[p..s].to_string(), d[s..e].to_string()); + self.pos = e; + return Some(r); + } + None => break, + } + } + let rec = d[p..].to_string(); + self.pos = len; + Some((rec, String::new())) + } + } + } +} + +/// A command for the shell, posted by the evaluator to the plan driver. +pub(super) struct CmdRequest { + /// Capture the output for awk (`cmd | getline`) instead of streaming it. + pub(super) capture: bool, + pub(super) command: String, + pub(super) stdin: Option, + /// awk output to stream before the command runs (keeps order). + pub(super) emit_stdout: String, + pub(super) emit_stderr: String, +} + +#[derive(Default)] +pub(super) struct BridgeState { + pub(super) request: Option, + pub(super) response: Option, +} + +pub(super) type Bridge = Arc>; + +/// Resolves when the driver has stored a response. The driver polls the +/// awk future again only after storing one, so no waker is needed. +struct WaitResponse(Bridge); + +impl Future for WaitResponse { + type Output = ExecResult; + fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll { + match self.0.lock().ok().and_then(|mut s| s.response.take()) { + Some(r) => Poll::Ready(r), + None => Poll::Pending, + } + } +} + +/// What awk needs from the shell around it. +pub(super) struct Host { + pub(super) fs: Arc, + pub(super) cwd: PathBuf, + pub(super) env: Vec<(String, String)>, + pub(super) stdin: Option, + pub(super) budget: Option, + pub(super) clock: Date, + pub(super) bridge: Option, +} + +enum OutKind { + File { + path: PathBuf, + append: bool, + written: bool, + }, + Pipe { + command: String, + }, +} + +struct Output { + kind: OutKind, + buf: String, +} + +#[derive(Default)] +struct MainInput { + /// Next `ARGV` index to look at. + argi: usize, + cur: Option, + /// Some file operand (or stdin) has been opened. + used: bool, +} + +pub(super) struct Io { + pub(super) host: Host, + pub(super) stdout: String, + pub(super) stderr: String, + /// Bytes of stdout/stderr already streamed by the driver. + pub(super) emitted: (usize, usize), + /// Bytes sent to files and pipes (output cap accounting). + redirected: usize, + outputs: HashMap, + out_order: Vec, + readers: HashMap, + reader_bytes: usize, + main: MainInput, + stdin_taken: bool, + warned_no_commands: bool, +} + +impl Io { + pub(super) fn new(host: Host) -> Self { + Io { + host, + stdout: String::new(), + stderr: String::new(), + emitted: (0, 0), + redirected: 0, + outputs: HashMap::new(), + out_order: Vec::new(), + readers: HashMap::new(), + reader_bytes: 0, + main: MainInput::default(), + stdin_taken: false, + warned_no_commands: false, + } + } + + fn take_stdin(&mut self) -> String { + if self.stdin_taken { + return String::new(); + } + self.stdin_taken = true; + self.host.stdin.take().unwrap_or_default() + } + + fn resolve(&self, name: &str) -> PathBuf { + if name.starts_with('/') { + PathBuf::from(name) + } else { + vfs_join(&self.host.cwd, name) + } + } + + fn total_out(&self) -> usize { + self.stdout.len() + self.stderr.len() + self.redirected + } +} + +/// `name=value` command-line assignment operand. +pub(super) fn cmdline_assignment(arg: &str) -> Option<(&str, &str)> { + let (name, value) = arg.split_once('=')?; + let mut chars = name.chars(); + let first = chars.next()?; + if !(first.is_ascii_alphabetic() || first == '_') + || !chars.all(|c| c.is_ascii_alphanumeric() || c == '_') + { + return None; + } + Some((name, value)) +} + +impl Interp { + // ----- main input ----- + + pub(super) async fn next_main_record(&mut self) -> R> { + loop { + if let Some(r) = self.io.main.cur.as_mut() { + let rs = self.rs.clone(); + if let Some((rec, rt)) = r.read(&rs) { + if let Some(b) = &self.io.host.budget + && let Err(e) = b.consume_work(1) + { + return Err(self.budget_exhausted(e)); + } + self.nr += 1.0; + self.fnr += 1.0; + self.globals[special::NR as usize] = Cell::Val(Value::Num(self.nr)); + self.globals[special::FNR as usize] = Cell::Val(Value::Num(self.fnr)); + self.globals[special::RT as usize] = Cell::Val(Value::Str(rt)); + return Ok(Some(rec)); + } + self.end_main_file().await?; + } + if !self.open_next_main().await? { + return Ok(None); + } + } + } + + async fn end_main_file(&mut self) -> R<()> { + if self.io.main.cur.take().is_some() { + let prog = self.prog.clone(); + self.run_blocks(&prog.endfile).await?; + } + Ok(()) + } + + /// `nextfile`: stop reading the current file. + pub(super) async fn skip_file(&mut self) -> R<()> { + self.end_main_file().await + } + + fn argv_elem(&self, i: usize) -> Option { + let Cell::Arr(id) = self.globals[special::ARGV as usize] else { + return None; + }; + match self.array(id)?.get(&i.to_string())? { + Elem::Val(v) => Some(self.to_str(v)), + Elem::Arr(_) => None, + } + } + + async fn open_next_main(&mut self) -> R { + // ARGV[0] is the program name. + self.io.main.argi = self.io.main.argi.max(1); + loop { + let argc = match &self.globals[special::ARGC as usize] { + Cell::Val(v) => v.to_num(), + Cell::Arr(_) => 0.0, + }; + let argc = if argc.is_finite() && argc > 0.0 { + argc as usize + } else { + 0 + }; + if self.io.main.argi >= argc { + break; + } + let i = self.io.main.argi; + self.io.main.argi += 1; + let Some(arg) = self.argv_elem(i) else { + continue; + }; + if arg.is_empty() { + continue; + } + if let Some((name, value)) = cmdline_assignment(&arg) { + let value = super::lexer::unescape(value); + self.assign_by_name(name, Value::from_input(value))?; + continue; + } + self.io.main.used = true; + let data = if arg == "-" || arg == "/dev/stdin" { + self.io.take_stdin() + } else { + let path = self.io.resolve(&arg); + if let Ok(meta) = self.io.host.fs.stat(&path).await + && meta.file_type.is_dir() + { + let msg = format!( + "awk: warning: command line argument `{arg}' is a directory: skipped" + ); + self.err_line(&msg); + continue; + } + match self.io.host.fs.read_file(&path).await { + Ok(bytes) => { + if let Some(b) = &self.io.host.budget + && let Err(e) = b.consume_input(bytes.len()) + { + return Err(self.budget_exhausted(e)); + } + String::from_utf8_lossy(&bytes).into_owned() + } + Err(e) => { + let reason = crate::error::io_error_reason(&e); + return Err(self + .fatal_at(&format!("cannot open file `{arg}' for reading: {reason}"))); + } + } + }; + self.start_main_file(arg, data).await?; + return Ok(true); + } + if !self.io.main.used { + self.io.main.used = true; + let data = self.io.take_stdin(); + self.start_main_file(String::new(), data).await?; + return Ok(true); + } + Ok(false) + } + + async fn start_main_file(&mut self, name: String, data: String) -> R<()> { + self.set_var( + super::ast::VarRef::Global(special::FILENAME), + Value::Str(name), + )?; + self.fnr = 0.0; + self.globals[special::FNR as usize] = Cell::Val(Value::Num(0.0)); + self.io.main.cur = Some(Reader::new(data)); + let prog = self.prog.clone(); + self.run_blocks(&prog.beginfile).await + } + + pub(super) fn assign_by_name(&mut self, name: &str, v: Value) -> R<()> { + if let Some(i) = self.prog.globals.iter().position(|g| g == name) { + self.set_var(super::ast::VarRef::Global(i as u32), v)?; + } + Ok(()) + } + + // ----- getline ----- + + pub(super) async fn getline(&mut self, src: &GetlineSrc, var: Option<&Expr>) -> R { + match src { + GetlineSrc::Main => match self.next_main_record().await? { + Some(rec) => { + self.getline_store(var, rec).await?; + Ok(Value::Num(1.0)) + } + None => Ok(Value::Num(0.0)), + }, + GetlineSrc::File(f) => { + let name = self.eval(f).await?; + let name = self.to_str(&name); + if !self.io.readers.contains_key(&name) && !self.open_file_reader(&name).await? { + return Ok(Value::Num(-1.0)); + } + let rs = self.rs.clone(); + let rec = self.io.readers.get_mut(&name).and_then(|r| r.read(&rs)); + match rec { + Some((rec, rt)) => { + self.globals[special::RT as usize] = Cell::Val(Value::Str(rt)); + self.getline_store(var, rec).await?; + Ok(Value::Num(1.0)) + } + None => Ok(Value::Num(0.0)), + } + } + GetlineSrc::Cmd(c) => { + let cmd = self.eval(c).await?; + let cmd = self.to_str(&cmd); + if !self.io.readers.contains_key(&cmd) { + self.check_reader_slots()?; + let r = self.run_command(true, &cmd, None).await?; + let data = r.stdout.text_lossy().into_owned(); + self.charge_reader(data.len())?; + let mut reader = Reader::new(data); + reader.status = r.exit_code; + self.io.readers.insert(cmd.clone(), reader); + } + let rs = self.rs.clone(); + let rec = self.io.readers.get_mut(&cmd).and_then(|r| r.read(&rs)); + match rec { + // Like gawk, redirected getline leaves NR and FNR alone. + Some((rec, rt)) => { + self.globals[special::RT as usize] = Cell::Val(Value::Str(rt)); + self.getline_store(var, rec).await?; + Ok(Value::Num(1.0)) + } + None => Ok(Value::Num(0.0)), + } + } + } + } + + async fn getline_store(&mut self, var: Option<&Expr>, rec: String) -> R<()> { + match var { + Some(v) => self.assign(v, Value::from_input(rec)).await, + None => { + self.set_record(rec); + Ok(()) + } + } + } + + fn check_reader_slots(&mut self) -> R<()> { + if self.io.readers.len() >= AWK_MAX_GETLINE_CACHED_FILES { + return Err(self.fatal(&format!( + "getline open file limit ({AWK_MAX_GETLINE_CACHED_FILES}) exceeded" + ))); + } + Ok(()) + } + + /// Charge a loaded input to the request budget and the retained cap. + fn charge_reader(&mut self, len: usize) -> R<()> { + if let Some(b) = &self.io.host.budget + && let Err(e) = b + .consume_input(len) + .and_then(|()| b.consume_work(len as u64)) + { + return Err(self.budget_exhausted(e)); + } + let total = self.io.reader_bytes.saturating_add(len); + if total > AWK_MAX_GETLINE_CACHE_BYTES { + return Err(self.fatal(&format!( + "getline total input limit ({AWK_MAX_GETLINE_CACHE_BYTES} bytes) exceeded" + ))); + } + self.io.reader_bytes = total; + Ok(()) + } + + /// Open `name` for `getline <`; `false` when it cannot be read. + /// gawk's `ERRNO`: why the last `getline` or `close` failed. + fn set_errno(&mut self, reason: &str) { + let reason = super::interp::truncate(reason, 200).to_string(); + self.globals[special::ERRNO as usize] = Cell::Val(Value::Str(reason)); + } + + async fn open_file_reader(&mut self, name: &str) -> R { + self.check_reader_slots()?; + let data = if name == "-" || name == "/dev/stdin" { + self.io.take_stdin() + } else { + let path = self.io.resolve(name); + let fs = self.io.host.fs.clone(); + match fs.stat(&path).await { + Ok(meta) if meta.file_type.is_dir() => { + self.set_errno("Is a directory"); + return Ok(false); + } + Ok(meta) if meta.size > AWK_MAX_GETLINE_FILE_BYTES as u64 => { + return Err(self.fatal(&format!( + "getline input file size limit ({AWK_MAX_GETLINE_FILE_BYTES} bytes) exceeded" + ))); + } + Ok(_) => {} + Err(e) => { + self.set_errno(&crate::error::io_error_reason(&e)); + return Ok(false); + } + } + match fs.read_file(&path).await { + Ok(bytes) if bytes.len() > AWK_MAX_GETLINE_FILE_BYTES => { + return Err(self.fatal(&format!( + "getline input file size limit ({AWK_MAX_GETLINE_FILE_BYTES} bytes) exceeded" + ))); + } + Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(), + Err(e) => { + self.set_errno(&crate::error::io_error_reason(&e)); + return Ok(false); + } + } + }; + self.charge_reader(data.len())?; + self.io.readers.insert(name.to_string(), Reader::new(data)); + Ok(true) + } + + // ----- output ----- + + pub(super) async fn write_out(&mut self, dest: Option<&Redirect>, text: String) -> R<()> { + if text.len() > AWK_MAX_OUTPUT_BYTES.saturating_sub(self.io.total_out()) { + return Err(self.fatal_plain("output limit exceeded (max 10MB)")); + } + let Some(dest) = dest else { + self.io.stdout.push_str(&text); + return Ok(()); + }; + let (target, append, pipe) = match dest { + Redirect::File(e) => (e, false, false), + Redirect::Append(e) => (e, true, false), + Redirect::Pipe(e) => (e, false, true), + }; + let name = self.eval(target).await?; + let name = self.to_str(&name); + if !pipe { + match name.as_str() { + "/dev/stdout" | "/dev/fd/1" => { + self.io.stdout.push_str(&text); + return Ok(()); + } + "/dev/stderr" | "/dev/fd/2" => { + self.io.stderr.push_str(&text); + return Ok(()); + } + "/dev/null" => return Ok(()), + _ => {} + } + } + if !self.io.outputs.contains_key(&name) { + if self.io.outputs.len() >= AWK_MAX_OUTPUT_TARGETS { + return Err(self.fatal_plain("too many output redirection targets")); + } + let kind = if pipe { + OutKind::Pipe { + command: name.clone(), + } + } else { + OutKind::File { + path: self.io.resolve(&name), + append, + written: false, + } + }; + self.io.outputs.insert( + name.clone(), + Output { + kind, + buf: String::new(), + }, + ); + self.io.out_order.push(name.clone()); + } + self.io.redirected += text.len(); + let flush = { + let out = self.io.outputs.get_mut(&name).ok_or(Unwind::Fatal)?; + out.buf.push_str(&text); + matches!(out.kind, OutKind::File { .. }) && out.buf.len() >= FILE_FLUSH_BYTES + }; + if flush { + self.flush_output(&name).await?; + } + Ok(()) + } + + /// Write a file redirection's buffer through the VFS. + async fn flush_output(&mut self, name: &str) -> R<()> { + let fs = self.io.host.fs.clone(); + let Some(out) = self.io.outputs.get_mut(name) else { + return Ok(()); + }; + let OutKind::File { + path, + append, + written, + } = &mut out.kind + else { + return Ok(()); + }; + if *written && out.buf.is_empty() { + return Ok(()); + } + let bytes = std::mem::take(&mut out.buf); + let mut truncate = !(*append || *written); + *written = true; + let path = path.clone(); + let write = |truncate: bool, data: &str| { + let fs = fs.clone(); + let path = path.clone(); + let data = data.as_bytes().to_vec(); + async move { + if truncate { + fs.write_file(&path, &data).await + } else { + fs.append_file(&path, &data).await + } + } + }; + let mut res = write(truncate, &bytes).await; + if res.is_err() && bytes.trim_end_matches('\n').contains('\n') { + // A quota rejected the chunk: keep the lines that fit, as if + // each print had been written on its own. + for line in bytes.split_inclusive('\n') { + res = write(truncate, line).await; + if res.is_err() { + break; + } + truncate = false; + } + } + if let Err(e) = res { + let reason = crate::error::io_error_reason(&e); + return Err(self.fatal_plain(&format!("cannot write {name}: {reason}"))); + } + Ok(()) + } + + pub(super) async fn flush_all(&mut self) -> R<()> { + let names: Vec = self.io.out_order.clone(); + for n in names { + self.flush_output(&n).await?; + } + Ok(()) + } + + /// `close(name)`: the command's exit status, 0 for files, -1 if not + /// open. + pub(super) async fn close_stream(&mut self, name: &str) -> R { + if self.io.outputs.contains_key(name) { + let r = self.flush_output(name).await; + let out = self.io.outputs.remove(name); + self.io.out_order.retain(|n| n != name); + r?; + if let Some(Output { + kind: OutKind::Pipe { command }, + buf, + }) = out + { + let res = self.run_command(false, &command, Some(buf)).await?; + return Ok(res.exit_code as f64); + } + return Ok(0.0); + } + if let Some(r) = self.io.readers.remove(name) { + // Retained bytes are freed; the request budget is not refunded. + self.io.reader_bytes = self.io.reader_bytes.saturating_sub(r.data.len()); + return Ok(r.status as f64); + } + Ok(-1.0) + } + + /// Flush files and run pending output pipes at exit. + pub(super) async fn close_all(&mut self) { + let names: Vec = self.io.out_order.clone(); + for n in names { + if let Err(Unwind::Fatal) = Box::pin(self.close_stream(&n)).await + && self.fatal + { + // Keep closing the rest; the first error is reported. + continue; + } + } + } + + // ----- commands ----- + + /// Run `command` with `sh -c` through the plan driver. + pub(super) async fn run_command( + &mut self, + capture: bool, + command: &str, + stdin: Option, + ) -> R { + let Some(bridge) = self.io.host.bridge.clone() else { + if !self.io.warned_no_commands { + self.io.warned_no_commands = true; + self.err_line("awk: running commands is not supported in this context"); + } + return Ok(ExecResult::with_code(String::new(), 127)); + }; + // Commands see redirected output written so far (gawk flushes). + self.flush_all().await?; + let (emit_stdout, emit_stderr) = if capture { + (String::new(), String::new()) + } else { + let out = self.io.stdout[self.io.emitted.0..].to_string(); + let err = self.io.stderr[self.io.emitted.1..].to_string(); + self.io.emitted = (self.io.stdout.len(), self.io.stderr.len()); + (out, err) + }; + if let Ok(mut s) = bridge.lock() { + s.request = Some(CmdRequest { + capture, + command: command.to_string(), + stdin, + emit_stdout, + emit_stderr, + }); + } + let res = WaitResponse(bridge).await; + if !capture { + // The shell streamed this output already; keep it in ours for + // callers that capture the whole result. + let out = res.stdout.text_lossy(); + let err = res.stderr.text_lossy(); + if out.len() + err.len() > AWK_MAX_OUTPUT_BYTES.saturating_sub(self.io.total_out()) { + return Err(self.fatal_plain("output limit exceeded (max 10MB)")); + } + self.io.stdout.push_str(&out); + self.io.stderr.push_str(&err); + self.io.emitted = (self.io.stdout.len(), self.io.stderr.len()); + } else { + let err = res.stderr.text_lossy().into_owned(); + self.io.stderr.push_str(&err); + } + Ok(res) + } +} diff --git a/crates/bashkit/src/builtins/awk/lexer.rs b/crates/bashkit/src/builtins/awk/lexer.rs new file mode 100644 index 000000000..7fb6df9bd --- /dev/null +++ b/crates/bashkit/src/builtins/awk/lexer.rs @@ -0,0 +1,667 @@ +//! awk lexer. +//! +//! Decisions: +//! - Lazy, parser-driven: `/` is lexed as division; the parser asks for a +//! regex literal (`read_regex`) when it expects an operand and sees `/` or +//! `/=`. That is the only way to tell `a / b / c` from `/re/`. +//! - Newlines are tokens (statement terminators); the parser skips them +//! where the grammar allows (`opt_nls`). Comments and backslash-newline +//! continuations are skipped here. +//! - An identifier immediately followed by `(` is a `FuncName` token: user +//! function calls must not have a space before `(` (POSIX), which is what +//! tells `f (x)` (concatenation) from `f(x)`. +//! - Numeric literals in source accept gawk's hex (`0x1A`) and octal (`011`) +//! forms; strings converted at runtime never do (see `value.rs`). + +#[derive(Debug, Clone, PartialEq)] +pub(super) enum Tok { + Num(f64), + Str(String), + /// Regex literal body (escapes kept for the regex translator). + Regex(String), + Name(String), + FuncName(String), + Builtin(String), + // Keywords + Begin, + End, + BeginFile, + EndFile, + Function, + If, + Else, + While, + For, + Do, + Break, + Continue, + Next, + NextFile, + Exit, + Return, + Delete, + Getline, + Print, + Printf, + In, + Switch, + Case, + Default, + // Punctuation + LBrace, + RBrace, + LParen, + RParen, + LBracket, + RBracket, + Semi, + Newline, + Comma, + Plus, + Minus, + Star, + Slash, + Percent, + Caret, + Not, + Gt, + Lt, + Pipe, + PipeAmp, + Question, + Colon, + Tilde, + NoMatch, + Dollar, + At, + Assign, + AddAssign, + SubAssign, + MulAssign, + DivAssign, + ModAssign, + PowAssign, + Eq, + Le, + Ge, + Ne, + Incr, + Decr, + And, + Or, + Append, + Eof, +} + +pub(super) const BUILTIN_FUNCS: &[&str] = &[ + "length", "substr", "index", "split", "sub", "gsub", "gensub", "match", "sprintf", "sin", + "cos", "atan2", "exp", "log", "sqrt", "int", "rand", "srand", "tolower", "toupper", "system", + "close", "fflush", "strtonum", "systime", "strftime", "mktime", "asort", "asorti", "typeof", + "isarray", "and", "or", "xor", "lshift", "rshift", "compl", "patsplit", +]; + +pub(super) struct Lexer<'a> { + src: &'a str, + pos: usize, +} + +/// Lexing failure: message and byte offset. +pub(super) struct LexError { + pub(super) msg: String, + pub(super) pos: usize, +} + +impl<'a> Lexer<'a> { + pub(super) fn new(src: &'a str) -> Self { + Self { src, pos: 0 } + } + + pub(super) fn pos(&self) -> usize { + self.pos + } + + pub(super) fn reset(&mut self, pos: usize) { + self.pos = pos; + } + + fn peek_byte(&self) -> Option { + self.src.as_bytes().get(self.pos).copied() + } + + fn byte_at(&self, i: usize) -> Option { + self.src.as_bytes().get(i).copied() + } + + /// Skip blanks, comments and backslash-newline continuations. + fn skip_blank(&mut self) { + while let Some(b) = self.peek_byte() { + match b { + b' ' | b'\t' | b'\r' => self.pos += 1, + b'\\' if self.byte_at(self.pos + 1) == Some(b'\n') => self.pos += 2, + b'\\' + if self.byte_at(self.pos + 1) == Some(b'\r') + && self.byte_at(self.pos + 2) == Some(b'\n') => + { + self.pos += 3 + } + b'#' => { + while let Some(c) = self.peek_byte() { + if c == b'\n' { + break; + } + self.pos += 1; + } + } + _ => break, + } + } + } + + /// Next token and its start offset. + pub(super) fn next(&mut self) -> Result<(Tok, usize), LexError> { + self.skip_blank(); + let start = self.pos; + let Some(b) = self.peek_byte() else { + return Ok((Tok::Eof, start)); + }; + let two = |l: &Self, c: u8| l.byte_at(l.pos + 1) == Some(c); + let tok = match b { + b'\n' => { + self.pos += 1; + Tok::Newline + } + b'{' => self.one(Tok::LBrace), + b'}' => self.one(Tok::RBrace), + b'(' => self.one(Tok::LParen), + b')' => self.one(Tok::RParen), + b'[' => self.one(Tok::LBracket), + b']' => self.one(Tok::RBracket), + b';' => self.one(Tok::Semi), + b',' => self.one(Tok::Comma), + b'?' => self.one(Tok::Question), + b':' => self.one(Tok::Colon), + b'~' => self.one(Tok::Tilde), + b'$' => self.one(Tok::Dollar), + b'@' => self.one(Tok::At), + b'+' if two(self, b'+') => self.two(Tok::Incr), + b'+' if two(self, b'=') => self.two(Tok::AddAssign), + b'+' => self.one(Tok::Plus), + b'-' if two(self, b'-') => self.two(Tok::Decr), + b'-' if two(self, b'=') => self.two(Tok::SubAssign), + b'-' => self.one(Tok::Minus), + b'*' if two(self, b'*') => { + if self.byte_at(self.pos + 2) == Some(b'=') { + self.pos += 3; + Tok::PowAssign + } else { + self.two(Tok::Caret) + } + } + b'*' if two(self, b'=') => self.two(Tok::MulAssign), + b'*' => self.one(Tok::Star), + b'/' if two(self, b'=') => self.two(Tok::DivAssign), + b'/' => self.one(Tok::Slash), + b'%' if two(self, b'=') => self.two(Tok::ModAssign), + b'%' => self.one(Tok::Percent), + b'^' if two(self, b'=') => self.two(Tok::PowAssign), + b'^' => self.one(Tok::Caret), + b'!' if two(self, b'=') => self.two(Tok::Ne), + b'!' if two(self, b'~') => self.two(Tok::NoMatch), + b'!' => self.one(Tok::Not), + b'>' if two(self, b'=') => self.two(Tok::Ge), + b'>' if two(self, b'>') => self.two(Tok::Append), + b'>' => self.one(Tok::Gt), + b'<' if two(self, b'=') => self.two(Tok::Le), + b'<' => self.one(Tok::Lt), + b'=' if two(self, b'=') => self.two(Tok::Eq), + b'=' => self.one(Tok::Assign), + b'|' if two(self, b'|') => self.two(Tok::Or), + b'|' if two(self, b'&') => self.two(Tok::PipeAmp), + b'|' => self.one(Tok::Pipe), + b'&' if two(self, b'&') => self.two(Tok::And), + b'"' => self.string()?, + b'0'..=b'9' | b'.' => self.number()?, + c if c.is_ascii_alphabetic() || c == b'_' => self.word(), + _ => { + let c = self.src[self.pos..].chars().next().unwrap_or('?'); + return Err(LexError { + msg: format!("invalid char '{c}' in expression"), + pos: start, + }); + } + }; + Ok((tok, start)) + } + + fn one(&mut self, t: Tok) -> Tok { + self.pos += 1; + t + } + + fn two(&mut self, t: Tok) -> Tok { + self.pos += 2; + t + } + + fn word(&mut self) -> Tok { + let start = self.pos; + while let Some(b) = self.peek_byte() { + if b.is_ascii_alphanumeric() || b == b'_' { + self.pos += 1; + } else { + break; + } + } + let w = &self.src[start..self.pos]; + let kw = match w { + "BEGIN" => Some(Tok::Begin), + "END" => Some(Tok::End), + "BEGINFILE" => Some(Tok::BeginFile), + "ENDFILE" => Some(Tok::EndFile), + "function" | "func" => Some(Tok::Function), + "if" => Some(Tok::If), + "else" => Some(Tok::Else), + "while" => Some(Tok::While), + "for" => Some(Tok::For), + "do" => Some(Tok::Do), + "break" => Some(Tok::Break), + "continue" => Some(Tok::Continue), + "next" => Some(Tok::Next), + "nextfile" => Some(Tok::NextFile), + "exit" => Some(Tok::Exit), + "return" => Some(Tok::Return), + "delete" => Some(Tok::Delete), + "getline" => Some(Tok::Getline), + "print" => Some(Tok::Print), + "printf" => Some(Tok::Printf), + "in" => Some(Tok::In), + "switch" => Some(Tok::Switch), + "case" => Some(Tok::Case), + "default" => Some(Tok::Default), + _ => None, + }; + if let Some(k) = kw { + return k; + } + if BUILTIN_FUNCS.contains(&w) { + return Tok::Builtin(w.to_string()); + } + if self.peek_byte() == Some(b'(') { + Tok::FuncName(w.to_string()) + } else { + Tok::Name(w.to_string()) + } + } + + fn number(&mut self) -> Result { + let start = self.pos; + let b = self.src.as_bytes(); + // Hex literal. + if b[start] == b'0' + && matches!(self.byte_at(start + 1), Some(b'x' | b'X')) + && self + .byte_at(start + 2) + .is_some_and(|c| c.is_ascii_hexdigit()) + { + let mut i = start + 2; + while i < b.len() && b[i].is_ascii_hexdigit() { + i += 1; + } + self.pos = i; + let v = u64::from_str_radix(&self.src[start + 2..i], 16) + .map(|v| v as f64) + .unwrap_or(f64::INFINITY); + return Ok(Tok::Num(v)); + } + let mut i = start; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let int_end = i; + let mut is_int = true; + if i < b.len() && b[i] == b'.' { + is_int = false; + i += 1; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + } + if i == start + 1 && b[start] == b'.' { + return Err(LexError { + msg: "syntax error".to_string(), + pos: start, + }); + } + if i < b.len() && (b[i] == b'e' || b[i] == b'E') { + let mut j = i + 1; + if j < b.len() && (b[j] == b'+' || b[j] == b'-') { + j += 1; + } + if j < b.len() && b[j].is_ascii_digit() { + is_int = false; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + i = j; + } + } + self.pos = i; + let text = &self.src[start..i]; + // gawk: a leading-zero integer of octal digits is octal. + if is_int + && int_end - start > 1 + && b[start] == b'0' + && text.bytes().all(|c| (b'0'..=b'7').contains(&c)) + { + let v = u64::from_str_radix(text, 8) + .map(|v| v as f64) + .unwrap_or(f64::INFINITY); + return Ok(Tok::Num(v)); + } + Ok(Tok::Num(text.parse::().unwrap_or(0.0))) + } + + fn string(&mut self) -> Result { + let start = self.pos; + self.pos += 1; + let mut out = String::new(); + loop { + let Some(c) = self.src[self.pos..].chars().next() else { + return Err(LexError { + msg: "unterminated string".to_string(), + pos: start, + }); + }; + self.pos += c.len_utf8(); + match c { + '"' => break, + '\n' => { + return Err(LexError { + msg: "unterminated string".to_string(), + pos: start, + }); + } + '\\' => { + let Some(e) = self.src[self.pos..].chars().next() else { + out.push('\\'); + continue; + }; + self.pos += e.len_utf8(); + match e { + 'n' => out.push('\n'), + 't' => out.push('\t'), + 'r' => out.push('\r'), + '\\' => out.push('\\'), + '"' => out.push('"'), + '/' => out.push('/'), + 'a' => out.push('\x07'), + 'b' => out.push('\x08'), + 'f' => out.push('\x0c'), + 'v' => out.push('\x0b'), + '\n' => {} + '0'..='7' => { + let mut v = e as u32 - '0' as u32; + for _ in 0..2 { + match self.peek_byte() { + Some(d @ b'0'..=b'7') => { + v = v * 8 + (d - b'0') as u32; + self.pos += 1; + } + _ => break, + } + } + push_byte_char(&mut out, v); + } + 'x' => { + let mut v = 0u32; + let mut n = 0; + while n < 2 { + match self.peek_byte() { + Some(d) if d.is_ascii_hexdigit() => { + v = v * 16 + (d as char).to_digit(16).unwrap_or(0); + self.pos += 1; + n += 1; + } + _ => break, + } + } + if n == 0 { + out.push_str("\\x"); + } else { + push_byte_char(&mut out, v); + } + } + // gawk 5.3: `\u` takes up to 8 hex digits of a + // code point; with none it stays as written. + 'u' => { + let mut v = 0u32; + let mut n = 0; + while n < 8 { + match self.peek_byte() { + Some(d) if d.is_ascii_hexdigit() => { + v = v + .saturating_mul(16) + .saturating_add((d as char).to_digit(16).unwrap_or(0)); + self.pos += 1; + n += 1; + } + _ => break, + } + } + if n == 0 { + out.push_str("\\u"); + } else { + push_byte_char(&mut out, v); + } + } + // gawk: an unknown escape keeps the character and + // drops the backslash (gawk also warns; we do not). + other => out.push(other), + } + } + c => out.push(c), + } + } + Ok(Tok::Str(out)) + } + + /// Read a regex literal; `start` is the offset of the opening `/`. + pub(super) fn read_regex(&mut self, start: usize) -> Result { + self.pos = start + 1; + let mut out = String::new(); + let mut in_bracket = false; + loop { + let Some(c) = self.src[self.pos..].chars().next() else { + return Err(LexError { + msg: "unterminated regexp".to_string(), + pos: start, + }); + }; + self.pos += c.len_utf8(); + match c { + '\n' => { + return Err(LexError { + msg: "unterminated regexp".to_string(), + pos: start, + }); + } + '/' if !in_bracket => break, + '\\' => { + let Some(e) = self.src[self.pos..].chars().next() else { + out.push('\\'); + continue; + }; + self.pos += e.len_utf8(); + if e == '/' { + out.push('/'); + } else if e == '\n' { + // Line continuation inside a regex. + } else { + out.push('\\'); + out.push(e); + } + } + '[' if !in_bracket => { + in_bracket = true; + out.push('['); + // `]` right after `[` or `[^` is literal. + if self.peek_byte() == Some(b'^') { + out.push('^'); + self.pos += 1; + } + if self.peek_byte() == Some(b']') { + out.push(']'); + self.pos += 1; + } + } + '[' if in_bracket && matches!(self.peek_byte(), Some(b':' | b'.' | b'=')) => { + // Character class like [:alpha:] inside a bracket. + let delim = self.peek_byte().unwrap_or(b':') as char; + out.push('['); + let close = format!("{delim}]"); + if let Some(end) = self.src[self.pos..].find(&close) { + out.push_str(&self.src[self.pos..self.pos + end + 2]); + self.pos += end + 2; + } + } + ']' if in_bracket => { + in_bracket = false; + out.push(']'); + } + c => out.push(c), + } + } + Ok(Tok::Regex(out)) + } +} + +/// Process awk string escapes in command-line text (`-v`, `var=value` +/// operands, `-F`), as in a string literal. +pub(super) fn unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + let Some(e) = chars.next() else { + out.push('\\'); + break; + }; + match e { + 'n' => out.push('\n'), + 't' => out.push('\t'), + 'r' => out.push('\r'), + '\\' => out.push('\\'), + '"' => out.push('"'), + '/' => out.push('/'), + 'a' => out.push('\x07'), + 'b' => out.push('\x08'), + 'f' => out.push('\x0c'), + 'v' => out.push('\x0b'), + '0'..='7' => { + let mut v = e as u32 - '0' as u32; + for _ in 0..2 { + match chars.peek() { + Some(d @ '0'..='7') => { + v = v * 8 + (*d as u32 - '0' as u32); + chars.next(); + } + _ => break, + } + } + push_byte_char(&mut out, v); + } + other => { + out.push('\\'); + out.push(other); + } + } + } + out +} + +/// Push a byte-valued escape: ASCII as is, other values as the Unicode +/// character with that code (the input is handled as UTF-8 text). +fn push_byte_char(out: &mut String, v: u32) { + if let Some(c) = char::from_u32(v) { + out.push(c); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn toks(src: &str) -> Vec { + let mut l = Lexer::new(src); + let mut out = Vec::new(); + loop { + let (t, _) = l.next().ok().unwrap(); + if t == Tok::Eof { + break; + } + out.push(t); + } + out + } + + #[test] + fn numbers() { + assert_eq!( + toks("0x1A 011 018 1.5e2 .5"), + vec![ + Tok::Num(26.0), + Tok::Num(9.0), + Tok::Num(18.0), + Tok::Num(150.0), + Tok::Num(0.5) + ] + ); + } + + #[test] + fn strings_and_escapes() { + assert_eq!( + toks(r#""a\tb\101\x41\.""#), + vec![Tok::Str("a\tbAA.".into())] + ); + } + + #[test] + fn operators_and_words() { + assert_eq!( + toks("x **= 2; f(1) g (2) length"), + vec![ + Tok::Name("x".into()), + Tok::PowAssign, + Tok::Num(2.0), + Tok::Semi, + Tok::FuncName("f".into()), + Tok::LParen, + Tok::Num(1.0), + Tok::RParen, + Tok::Name("g".into()), + Tok::LParen, + Tok::Num(2.0), + Tok::RParen, + Tok::Builtin("length".into()) + ] + ); + } + + #[test] + fn regex_literal_with_slash_in_class() { + let mut l = Lexer::new("/[/]x\\/y/ z"); + assert_eq!(l.read_regex(0).ok().unwrap(), Tok::Regex("[/]x/y".into())); + } + + #[test] + fn comments_and_continuations() { + assert_eq!( + toks("a # c\n\\\nb"), + vec![Tok::Name("a".into()), Tok::Newline, Tok::Name("b".into())] + ); + } +} diff --git a/crates/bashkit/src/builtins/awk/mod.rs b/crates/bashkit/src/builtins/awk/mod.rs index 5eee10663..e0e72939a 100644 --- a/crates/bashkit/src/builtins/awk/mod.rs +++ b/crates/bashkit/src/builtins/awk/mod.rs @@ -1,270 +1,119 @@ -//! awk - Pattern scanning and processing builtin -//! -//! Implements basic AWK functionality. +//! awk - pattern scanning and processing builtin (gawk-compatible) //! //! Usage: //! awk '{print $1}' file -//! awk -F: '{print $1}' /etc/passwd -//! echo "a b c" | awk '{print $2}' -//! awk 'BEGIN{print "start"} {print} END{print "end"}' file -//! awk '/pattern/{print}' file -//! awk 'NR==2{print}' file - -// Parser invariant: `pos` is always a byte offset on a UTF-8 char boundary. -// Move across user-controlled text with `advance()`/`consume_while()`, and use -// raw `pos += N` only for known ASCII tokens and delimiters. +//! awk -F: -v OFS=- '{$1=$1; print}' /etc/passwd +//! awk -f prog.awk data.txt +//! awk 'BEGIN { while (("ls" | getline f) > 0) print f }' +//! +//! Decisions: +//! - The target is gawk 5 (Debian's `awk`): its option set, its error +//! message shapes (`awk: cmd. line:1: ...` with a caret for syntax +//! errors), its output for numbers, and extensions agents use (`gensub`, +//! `asort`, `strftime`, `BEGINFILE`, `switch`, `--csv`, arrays of arrays). +//! - Pipeline: `lexer` -> `parser` (resolves names to slots, `ast`) -> +//! `interp` (async evaluator) with `io` (records, getline, redirections, +//! commands) and `funcs` (builtin functions). `regex` translates EREs and +//! gives POSIX leftmost-longest matches. +//! - Commands run in the sandbox shell through an execution-plan driver. +//! The driver is used only when the program can run commands (a `|` or +//! `system` in the arguments, or a `-f` program); otherwise `execute` +//! runs awk directly and a command attempt fails with a message. +//! - Resource limits (TM-DOS-027/028/033/109/110/116) are enforced in the +//! parser and evaluator; see the THREAT notes there. + #![allow(clippy::unwrap_used)] +mod ast; +mod format; +mod funcs; +mod interp; +mod io; +mod lexer; +mod parser; +mod regex; +mod value; + +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + use async_trait::async_trait; -use regex::Regex; -use std::collections::HashMap; -use super::limits::AWK_VARIABLE_OVERHEAD_BYTES; -use super::{Builtin, Context, read_text_file}; -use crate::error::{Error, Result}; +use self::ast::{Source, VarRef, special}; +use self::interp::{Interp, Limits}; +use self::io::{Bridge, BridgeState, Host, Io}; +use self::parser::{ParseError, ParseErrorKind, Parser}; +use self::value::Value; +use super::{Builtin, Context, Date, ExecutionPlan, PlanDriver, PlanStep, SubCommand}; +use crate::StreamData; +use crate::error::Result; +use crate::fs::vfs_join; use crate::interpreter::ExecResult; use crate::limits::ExecutionLimits; /// awk command - pattern scanning and processing -pub struct Awk; - -#[derive(Debug)] -struct AwkProgram { - begin_actions: Vec, - main_rules: Vec, - end_actions: Vec, - functions: HashMap, -} - -#[derive(Debug, Clone)] -struct AwkFunctionDef { - params: Vec, - body: Vec, -} - -#[derive(Debug)] -struct AwkRule { - pattern: Option, - actions: Vec, -} - -#[derive(Debug)] -enum AwkPattern { - Regex(Regex), - Expression(AwkExpr), - /// Range pattern: /start/,/end/ — matches from start to end inclusive. - /// Each sub-pattern can be a Regex or Expression. - Range(Box, Box), -} - -#[derive(Debug, Clone)] -enum AwkExpr { - Number(f64), - String(String), - Field(Box), // $n - Variable(String), // var - BinOp(Box, String, Box), - UnaryOp(String, Box), - Assign(String, Box), - ArrayAssign(String, Box, Box), // arr[key] = val - CompoundArrayAssign(String, Box, String, Box), // arr[key] += val - Concat(Vec), - FuncCall(String, Vec), - Regex(String), - #[allow(dead_code)] // matched in eval but construction deferred to pattern expansion - Match(Box, String), // expr ~ /pattern/ - PostIncrement(String), // var++ - PostDecrement(String), // var-- - PreIncrement(String), // ++var - PreDecrement(String), // --var - InArray(Box, String), // key in arr - FieldAssign(Box, Box), // $n = val - /// getline [var] < file as expression — returns 1 on success, 0 on EOF, -1 on error - GetlineFile { - var: Option, - file: Box, - }, -} - -/// Output target for print/printf redirection (e.g., `> file`, `>> file`). -/// Pipe (`| cmd`) is not supported and returns a clear error. -#[derive(Debug, Clone)] -enum AwkOutputTarget { - /// Truncate/create file: `> file` - Truncate(AwkExpr), - /// Append to file: `>> file` - Append(AwkExpr), -} - -#[derive(Debug, Clone)] -enum AwkAction { - Print(Vec, Option), - Printf(AwkExpr, Vec, Option), - Assign(String, AwkExpr), - ArrayAssign(String, AwkExpr, AwkExpr), // arr[key] = val - If(AwkExpr, Vec, Vec), - While(AwkExpr, Vec), - DoWhile(AwkExpr, Vec), - For(Box, AwkExpr, Box, Vec), - ForIn(String, String, Vec), // for (key in arr) { body } - Next, - Break, - Continue, - Delete(String, AwkExpr), // delete arr[key] - Getline, // getline — read next input record into $0 - /// getline [var] < file — read next line from file - GetlineFile { - var: Option, - file: AwkExpr, - }, - Exit(Option), - Return(Option), - Expression(AwkExpr), -} - -struct AwkState { - variables: HashMap, - fields: Vec, - fs: String, - /// Compiled ERE for `fs` when it is used as a regex (see `FieldSep`). - fs_regex: Option, - ofs: String, - ors: String, - nr: usize, - nf: usize, - fnr: usize, - /// When true, fields are split per RFC 4180 CSV rules (--csv flag) - csv_mode: bool, - /// Accounted bytes held by `variables` (keys, string values, and a fixed - /// per-entry overhead). THREAT[TM-DOS-110]: checked by the interpreter - /// against the live-bytes limit; mutate `variables` only through - /// `insert_var` / `remove_var` so this stays exact. - mem_bytes: usize, -} - -fn var_cost(key: &str, value: &AwkValue) -> usize { - key.len() + value.heap_bytes() + AWK_VARIABLE_OVERHEAD_BYTES -} - -#[derive(Debug, Clone, PartialEq)] -enum AwkValue { - Number(f64), - String(String), - Uninitialized, -} - -impl AwkValue { - fn heap_bytes(&self) -> usize { - match self { - AwkValue::String(s) => s.len(), - _ => 0, - } - } +#[derive(Clone, Copy, Default)] +pub struct Awk { + clock: Date, } -/// Format number using AWK's OFMT (%.6g): 6 significant digits, trim trailing zeros. -fn format_awk_number(n: f64) -> String { - if n.is_nan() { - return "nan".to_string(); - } - if n.is_infinite() { - return if n > 0.0 { "inf" } else { "-inf" }.to_string(); - } - // Integers: no decimal point - if n.fract() == 0.0 && n.abs() < 1e16 { - return format!("{}", n as i64); - } - // %.6g: use 6 significant digits - let abs = n.abs(); - let exp = abs.log10().floor() as i32; - if !(-4..6).contains(&exp) { - // Scientific notation: 5 decimal places = 6 sig digits - let mut s = format!("{:.*e}", 5, n); - // Trim trailing zeros in mantissa - if let Some(e_pos) = s.find('e') { - let (mantissa, exp_part) = s.split_at(e_pos); - let trimmed = mantissa.trim_end_matches('0').trim_end_matches('.'); - s = format!("{}{}", trimmed, exp_part); - } - // Normalize exponent format: e1 -> e+01 etc. to match C printf - // Actually AWK uses e+06 style. Rust uses e6. Fix: - if let Some(e_pos) = s.find('e') { - let exp_str = &s[e_pos + 1..]; - let exp_val: i32 = exp_str.parse().unwrap_or(0); - let mantissa = &s[..e_pos]; - s = format!("{}e{:+03}", mantissa, exp_val); - } - s - } else { - // Fixed notation - let decimal_places = (5 - exp).max(0) as usize; - let mut s = format!("{:.*}", decimal_places, n); - if s.contains('.') { - s = s.trim_end_matches('0').trim_end_matches('.').to_string(); - } - s - } -} - -impl AwkValue { - fn as_number(&self) -> f64 { - match self { - AwkValue::Number(n) => *n, - AwkValue::String(s) => s.parse().unwrap_or(0.0), - AwkValue::Uninitialized => 0.0, - } - } - - fn as_string(&self) -> String { - match self { - AwkValue::Number(n) => format_awk_number(*n), - AwkValue::String(s) => s.clone(), - AwkValue::Uninitialized => String::new(), - } - } - - fn as_bool(&self) -> bool { - match self { - AwkValue::Number(n) => *n != 0.0, - AwkValue::String(s) => { - if s.is_empty() { - return false; - } - // In awk, numeric strings evaluate as numbers in boolean context - if let Ok(n) = s.parse::() { - n != 0.0 - } else { - true - } - } - AwkValue::Uninitialized => false, - } - } -} - -impl Default for AwkState { - fn default() -> Self { - let mut state = Self { - variables: HashMap::new(), - fields: Vec::new(), - fs: " ".to_string(), - fs_regex: None, - ofs: " ".to_string(), - ors: "\n".to_string(), - nr: 0, - nf: 0, - fnr: 0, - csv_mode: false, - mem_bytes: 0, - }; - // POSIX SUBSEP: subscript separator for multi-dimensional arrays - state.insert_var("SUBSEP".to_string(), AwkValue::String("\x1c".to_string())); - state +impl Awk { + /// awk whose `systime()`, `strftime()` and `mktime()` use `clock`. + pub fn with_clock(clock: Date) -> Self { + Self { clock } } } -/// Parse a CSV line per RFC 4180: handle quoted fields, embedded commas, -/// and double-quote escaping. +const USAGE: &str = "\ +Usage: awk [POSIX or GNU style options] -f progfile [--] file ... +Usage: awk [POSIX or GNU style options] [--] 'program' file ... +POSIX options:\t\tGNU long options: (standard) +\t-f progfile\t\t--file=progfile +\t-F fs\t\t\t--field-separator=fs +\t-v var=val\t\t--assign=var=val +Short options:\t\tGNU long options: (extensions) +\t-e 'program-text'\t--source='program-text' +\t-k\t\t\t--csv +"; + +const VERSION: &str = "GNU Awk 5.2.1 (bashkit)"; + +/// gawk options that change nothing here. +const IGNORED_FLAGS: &[&str] = &[ + "-b", + "--characters-as-bytes", + "-c", + "--traditional", + "-C", + "--copyright", + "-g", + "--gen-pot", + "-M", + "--bignum", + "-n", + "--non-decimal-data", + "-N", + "--use-lc-numeric", + "-O", + "--optimize", + "-P", + "--posix", + "-r", + "--re-interval", + "-s", + "--no-optimize", + "-S", + "--sandbox", + "-t", + "--lint-old", + "-L", + "--lint", +]; + +/// Parse a CSV record per RFC 4180: quoted fields, embedded commas and +/// doubled quotes. fn csv_split_fields(line: &str) -> Vec { let mut fields = Vec::new(); let mut field = String::new(); @@ -275,7 +124,6 @@ fn csv_split_fields(line: &str) -> Vec { if in_quotes { if c == '"' { if chars.peek() == Some(&'"') { - // Escaped quote field.push('"'); chars.next(); } else { @@ -296,612 +144,474 @@ fn csv_split_fields(line: &str) -> Vec { fields } -/// How a field separator string splits text (POSIX awk "Regular -/// Expressions as Field Separators"). -/// -/// Decision (#2445): one classification drives record splitting (`FS`) and -/// `split()`, so both follow the same rules: -/// - `" "`: runs of blanks/newlines, leading and trailing ones ignored -/// - `""`: every character is a field (gawk/mawk extension) -/// - any other single character: that character, literally -/// - anything longer: an ERE -enum FieldSep<'a> { - Whitespace, - Chars, - Literal(&'a str), - Regex(&'a Regex), +enum ProgSrc { + File(String), + Text(String), } -impl<'a> FieldSep<'a> { - /// Returns `true` when `sep` must be matched as an ERE. - fn is_regex(sep: &str) -> bool { - sep != " " && sep.chars().nth(1).is_some() - } - - /// Classify `sep`; `regex` is its compiled ERE when `is_regex(sep)`. - /// An ERE that failed to compile falls back to a literal match. - fn classify(sep: &'a str, regex: Option<&'a Regex>) -> Self { - match sep { - " " => Self::Whitespace, - "" => Self::Chars, - _ => match regex { - Some(re) => Self::Regex(re), - None => Self::Literal(sep), - }, - } - } - - fn split(&self, text: &str) -> Vec { - if text.is_empty() { - // An empty record/string has no fields in every mode. - return Vec::new(); - } - match self { - Self::Whitespace => text.split_whitespace().map(String::from).collect(), - Self::Chars => text.chars().map(String::from).collect(), - Self::Literal(sep) => text.split(sep).map(String::from).collect(), - Self::Regex(re) => split_regex_nonempty(re, text), - } - } +struct Opts { + fs: Option, + assigns: Vec, + progs: Vec, + csv: bool, + operands: Vec, } -/// Split on non-empty ERE matches. An empty match separates nothing, as in -/// gawk (`FS = "x*"` does not split between every character). -fn split_regex_nonempty(re: &Regex, text: &str) -> Vec { - let mut fields = Vec::new(); - let mut last = 0; - for m in re.find_iter(text).filter(|m| !m.as_str().is_empty()) { - fields.push(text[last..m.start()].to_string()); - last = m.end(); - } - fields.push(text[last..].to_string()); - fields +fn usage_error(msg: Option) -> Box { + let mut err = msg.map(|m| format!("{m}\n")).unwrap_or_default(); + err.push_str(USAGE); + Box::new(exec_err(err, 1)) } -impl AwkState { - /// Set `FS`, compiling it once when it is an ERE. - fn set_fs(&mut self, fs: String) { - // THREAT[TM-DOS-023]: `build_regex` enforces the shared regex size limits. - self.fs_regex = if FieldSep::is_regex(&fs) { - crate::builtins::search_common::build_regex(&fs).ok() +fn parse_opts(args: &[String]) -> std::result::Result> { + let mut o = Opts { + fs: None, + assigns: Vec::new(), + progs: Vec::new(), + csv: false, + operands: Vec::new(), + }; + let mut i = 0; + // Option value: attached (`-F:`, `--file=x`) or the next argument. + let value = |i: &mut usize, attached: Option<&str>, opt: &str| { + if let Some(v) = attached { + return Ok(v.to_string()); + } + *i += 1; + args.get(*i).cloned().ok_or_else(|| { + usage_error(Some(format!( + "awk: option requires an argument -- '{}'", + opt.trim_start_matches('-') + ))) + }) + }; + while i < args.len() { + let arg = args[i].as_str(); + if arg == "--" { + i += 1; + break; + } + if arg == "-" || !arg.starts_with('-') { + break; + } + if IGNORED_FLAGS.contains(&arg) || arg.starts_with("--lint=") || arg.starts_with("-L") { + i += 1; + continue; + } + let (name, attached) = if let Some(long) = arg.strip_prefix("--") { + match long.split_once('=') { + Some((n, v)) => (format!("--{n}"), Some(v)), + None => (arg.to_string(), None), + } } else { - None + let rest = &arg[2..]; + (arg[..2].to_string(), (!rest.is_empty()).then_some(rest)) }; - self.fs = fs; - } - - /// Delete every element of array `name`. - fn clear_array(&mut self, name: &str) { - let prefix = format!("{name}["); - let mut released = 0; - self.variables.retain(|k, v| { - let keep = !k.starts_with(&prefix); - if !keep { - released += var_cost(k, v); + match name.as_str() { + "-F" | "--field-separator" => { + let v = value(&mut i, attached, "F")?; + o.fs = Some(if v == "t" { "\t".to_string() } else { v }); + } + "-v" | "--assign" => o.assigns.push(value(&mut i, attached, "v")?), + "-f" | "--file" => o.progs.push(ProgSrc::File(value(&mut i, attached, "f")?)), + "-e" | "--source" => o.progs.push(ProgSrc::Text(value(&mut i, attached, "e")?)), + "-k" | "--csv" if attached.is_none() => o.csv = true, + _ if name.starts_with("--") => { + return Err(usage_error(Some(format!( + "awk: unrecognized option '{arg}'" + )))); + } + _ => { + let c = arg.chars().nth(1).unwrap_or('-'); + return Err(usage_error(Some(format!("awk: invalid option -- '{c}'")))); } - keep - }); - self.mem_bytes -= released; - } - - /// Split a line into fields based on current mode (CSV or FS) - fn split_fields(&self, line: &str) -> Vec { - if self.csv_mode { - csv_split_fields(line) - } else { - FieldSep::classify(&self.fs, self.fs_regex.as_ref()).split(line) } + i += 1; } - - fn set_line(&mut self, line: &str) { - self.nr += 1; - self.fnr += 1; - - // Split by field separator - self.fields = self.split_fields(line); - - self.nf = self.fields.len(); - - // Set built-in variables - self.insert_var("NR".to_string(), AwkValue::Number(self.nr as f64)); - self.insert_var("NF".to_string(), AwkValue::Number(self.nf as f64)); - self.insert_var("FNR".to_string(), AwkValue::Number(self.fnr as f64)); - self.insert_var("$0".to_string(), AwkValue::String(line.to_string())); - } - - fn insert_var(&mut self, key: String, value: AwkValue) { - let added = var_cost(&key, &value); - if let Some(old) = self.variables.get(&key) { - self.mem_bytes -= var_cost(&key, old); + let mut rest = args[i..].iter().cloned(); + if o.progs.is_empty() { + match rest.next() { + Some(p) => o.progs.push(ProgSrc::Text(p)), + None => return Err(usage_error(None)), } - self.mem_bytes += added; - self.variables.insert(key, value); - } - - fn remove_var(&mut self, key: &str) -> Option { - let old = self.variables.remove(key)?; - self.mem_bytes -= var_cost(key, &old); - Some(old) } + o.operands = rest.collect(); + Ok(o) +} - fn get_field(&self, n: usize) -> AwkValue { - if n == 0 { - // $0 is the whole line - self.variables - .get("$0") - .cloned() - .unwrap_or(AwkValue::Uninitialized) - } else if n <= self.fields.len() { - AwkValue::String(self.fields[n - 1].clone()) - } else { - AwkValue::Uninitialized +/// gawk's message for a parse error, and the exit status. +fn format_parse_error(src: &str, sources: &[Source], e: &ParseError) -> (String, i32) { + let pos = e.pos.min(src.len()); + let si = sources.iter().rposition(|s| s.start <= pos).unwrap_or(0); + let (name, start) = sources + .get(si) + .map_or(("cmd. line", 0), |s| (s.name.as_str(), s.start)); + let line_no = src[start..pos].bytes().filter(|&b| b == b'\n').count() + 1; + let loc = format!("{}:{line_no}", interp::truncate(name, 200)); + match e.kind { + ParseErrorKind::Syntax => { + let line_start = src[..pos].rfind('\n').map_or(0, |p| p + 1).max(start); + let line_end = src[pos..].find('\n').map_or(src.len(), |p| pos + p); + let line = interp::truncate(&src[line_start..line_end], 400); + let col = src[line_start..pos] + .chars() + .count() + .min(line.chars().count()); + ( + format!( + "awk: {loc}: {line}\nawk: {loc}: {}^ {}\n", + " ".repeat(col), + interp::truncate(&e.msg, 300) + ), + 1, + ) } + ParseErrorKind::Error => (format!("awk: {loc}: error: {}\n", e.msg), 1), + ParseErrorKind::Fatal => (format!("awk: {loc}: fatal: {}\n", e.msg), 2), + ParseErrorKind::Limit => (format!("awk: fatal: {}\n", e.msg), 2), } +} - fn get_variable(&self, name: &str) -> AwkValue { - match name { - "NR" => AwkValue::Number(self.nr as f64), - "NF" => AwkValue::Number(self.nf as f64), - "FNR" => AwkValue::Number(self.fnr as f64), - "FS" => AwkValue::String(self.fs.clone()), - "OFS" => AwkValue::String(self.ofs.clone()), - "ORS" => AwkValue::String(self.ors.clone()), - _ => self - .variables - .get(name) - .cloned() - .unwrap_or(AwkValue::Uninitialized), - } - } +fn exec_err(stderr: String, code: i32) -> ExecResult { + let mut r = ExecResult::with_code(String::new(), code); + r.stderr = stderr.into(); + r +} - fn set_variable(&mut self, name: &str, value: AwkValue) { - match name { - "FS" => self.set_fs(value.as_string()), - "OFS" => self.ofs = value.as_string(), - "ORS" => self.ors = value.as_string(), - "$0" => { - let s = value.as_string(); - // Re-split fields when $0 is modified - self.fields = self.split_fields(&s); - self.nf = self.fields.len(); - self.insert_var("NF".to_string(), AwkValue::Number(self.nf as f64)); - self.insert_var(name.to_string(), value); - } - _ => { - self.insert_var(name.to_string(), value); - } - } +fn limits_from(ctx: &Context<'_>) -> Limits { + let (max_loop, max_total_loop, max_live) = ctx + .execution_extension::() + .and_then(|limits| { + limits + .try_with(|l| { + ( + l.max_loop_iterations, + l.max_total_loop_iterations, + l.max_live_intermediate_bytes, + ) + }) + .ok() + }) + .unwrap_or_else(|| { + let d = ExecutionLimits::default(); + ( + d.max_loop_iterations, + d.max_total_loop_iterations, + d.max_live_intermediate_bytes, + ) + }); + Limits { + max_loop, + max_total_loop, + max_mem: usize::try_from(max_live).unwrap_or(usize::MAX), } } -// THREAT[TM-DOS-027]: parser-depth limit lives in -// `super::limits::AWK_MAX_PARSER_DEPTH` (guards against deeply nested -// expressions). - -/// Preprocess awk program: replace newlines with semicolons inside action blocks. -/// This makes newlines act as statement separators per POSIX awk spec. -/// Respects string literals, regex literals, and nested braces. A newline -/// that POSIX lets a statement continue across (`opt_nls`: after the `)` of -/// an `if`/`for`/`while` header, after `else`/`do`, `&&`, `||` or `,`) -/// becomes a space instead. -fn normalize_awk_newlines(input: &str) -> String { - let mut result = String::with_capacity(input.len()); - let chars: Vec = input.chars().collect(); - let mut i = 0; - let mut brace_depth = 0; - // One entry per open `(`: whether it opened an if/for/while header. - let mut parens: Vec = Vec::new(); - // `result.len()` right after the `)` that closed a control header. - let mut header_end: Option = None; - let separator = |result: &str, header_end: Option| -> char { - let t = result.trim_end_matches([' ', '\t']); - let ends_with_word = |w: &str| { - t.strip_suffix(w).is_some_and(|rest| { - !rest - .chars() - .next_back() - .is_some_and(|c| c.is_alphanumeric() || c == '_') - }) - }; - if header_end == Some(t.len()) - || ends_with_word("else") - || ends_with_word("do") - || t.ends_with("&&") - || t.ends_with("||") - || t.ends_with(',') - { - ' ' - } else { - ';' +enum Setup { + Done(Box), + Run(Box), +} + +impl Awk { + /// Parse arguments and the program and build a ready interpreter. + async fn setup(&self, ctx: &Context<'_>, bridge: Option) -> Result { + if let Some(r) = super::check_help_version(ctx.args, USAGE, Some(VERSION)) { + return Ok(Setup::Done(Box::new(r))); } - }; + let opts = match parse_opts(ctx.args) { + Ok(o) => o, + Err(r) => return Ok(Setup::Done(r)), + }; - while i < chars.len() { - match chars[i] { - '{' => { - brace_depth += 1; - result.push('{'); - i += 1; - } - '}' => { - if brace_depth > 0 { - brace_depth -= 1; - } - result.push('}'); - i += 1; + // Join all program sources; each keeps its name for messages. + let mut program = String::new(); + let mut sources = Vec::new(); + for p in &opts.progs { + if !program.is_empty() { + program.push('\n'); } - '(' => { - let t = result.trim_end_matches([' ', '\t']); - let word_start = t - .rfind(|c: char| !(c.is_alphanumeric() || c == '_')) - .map_or(0, |p| p + 1); - parens.push(matches!(&t[word_start..], "if" | "for" | "while")); - result.push('('); - i += 1; - } - ')' => { - result.push(')'); - if parens.pop() == Some(true) { - header_end = Some(result.len()); - } - i += 1; - } - '"' => { - // String literal — pass through unchanged - result.push('"'); - i += 1; - while i < chars.len() && chars[i] != '"' { - if chars[i] == '\\' && i + 1 < chars.len() { - result.push(chars[i]); - i += 1; - } - result.push(chars[i]); - i += 1; - } - if i < chars.len() { - result.push(chars[i]); // closing " - i += 1; - } - } - '/' => { - // Regex literal — pass through unchanged (both pattern and expression context) - result.push('/'); - i += 1; - while i < chars.len() && chars[i] != '/' { - if chars[i] == '\\' && i + 1 < chars.len() { - result.push(chars[i]); - i += 1; - } - result.push(chars[i]); - i += 1; - } - if i < chars.len() { - result.push(chars[i]); // closing / - i += 1; - } - } - '#' => { - // Comment — skip to end of line, replace with newline/semicolon - while i < chars.len() && chars[i] != '\n' { - i += 1; - } - if i < chars.len() { - if brace_depth > 0 { - result.push(separator(&result, header_end)); + let (name, text) = match p { + ProgSrc::Text(t) => ("cmd. line".to_string(), t.clone()), + ProgSrc::File(f) => { + let path = if f.starts_with('/') { + std::path::PathBuf::from(f) } else { - result.push('\n'); + vfs_join(ctx.cwd, f) + }; + match ctx.fs.read_file(&path).await { + Ok(bytes) => { + ctx.consume_budget_input(bytes.len())?; + (f.clone(), String::from_utf8_lossy(&bytes).into_owned()) + } + Err(e) => { + return Ok(Setup::Done(Box::new(exec_err( + format!( + "awk: fatal: can't open source file `{}' for reading: {}\n", + interp::truncate(f, 200), + crate::error::io_error_reason(&e) + ), + 2, + )))); + } } - i += 1; } + }; + sources.push(Source { + name, + start: program.len(), + }); + program.push_str(&text); + } + + let prog = match Parser::new(&program, &sources).parse_program() { + Ok(p) => p, + Err(e) => { + let (msg, code) = format_parse_error(&program, &sources, &e); + return Ok(Setup::Done(Box::new(exec_err(msg, code)))); } - '\\' if i + 1 < chars.len() && chars[i + 1] == '\n' => { - // Backslash-newline: line continuation — join lines - i += 2; - } - '\n' if brace_depth > 0 => { - // Inside action block: replace newline with semicolon - result.push(separator(&result, header_end)); - i += 1; - } - _ => { - result.push(chars[i]); - i += 1; + }; + + let mut env: Vec<(String, String)> = ctx + .env + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + env.sort(); + let host = Host { + fs: ctx.fs.clone(), + cwd: ctx.cwd.clone(), + env, + stdin: ctx.stdin.map(|s| s.text_lossy().into_owned()), + budget: ctx + .execution_budget() + .and_then(|b| b.try_with(Clone::clone).ok()), + clock: self.clock, + bridge, + }; + let mut interp = match Interp::new(Arc::new(prog), Io::new(host), limits_from(ctx)) { + Ok(i) => Box::new(i), + Err(msg) => { + return Ok(Setup::Done(Box::new(exec_err( + format!("awk: fatal: {msg}\n"), + 2, + )))); } + }; + + if let Err(code) = Self::init_globals(&mut interp, &opts) { + return Ok(Setup::Done(Box::new(Self::result(&mut interp, code)?))); } + Ok(Setup::Run(interp)) } - result -} -mod interpreter; -mod parser; -use crate::fs::vfs_join; -use interpreter::{AwkFlow, AwkInterpreter}; -use parser::AwkParser; + /// ENVIRON, ARGV, ARGC, PROCINFO, `-F`, `--csv` and `-v` assignments. + fn init_globals(interp: &mut Interp, opts: &Opts) -> std::result::Result<(), i32> { + let fail = |interp: &mut Interp| if interp.fatal { 2 } else { 1 }; + let env = interp.io.host.env.clone(); + let Ok(arr) = interp.array_of_var(VarRef::Global(special::ENVIRON)) else { + return Err(fail(interp)); + }; + for (k, v) in env { + interp.elem_store(arr, k, interp::Elem::Val(Value::from_input(v))); + } + let Ok(arr) = interp.array_of_var(VarRef::Global(special::ARGV)) else { + return Err(fail(interp)); + }; + interp.elem_store(arr, "0".into(), interp::Elem::Val(Value::Str("awk".into()))); + for (i, a) in opts.operands.iter().enumerate() { + interp.elem_store( + arr, + (i + 1).to_string(), + interp::Elem::Val(Value::from_input(a.clone())), + ); + } + let argc = (opts.operands.len() + 1) as f64; + let Ok(arr) = interp.array_of_var(VarRef::Global(special::PROCINFO)) else { + return Err(fail(interp)); + }; + for (k, v) in [ + ("version", "5.2.1"), + ("strftime", "%a %b %e %H:%M:%S %Z %Y"), + ("FS", "FS"), + ("platform", "posix"), + ] { + interp.elem_store(arr, k.into(), interp::Elem::Val(Value::Str(v.into()))); + } + let set = |interp: &mut Interp, slot: u32, v: Value| { + interp + .set_var(VarRef::Global(slot), v) + .map_err(|_| fail(interp)) + }; + set(interp, special::ARGC, Value::Num(argc))?; + if opts.csv { + interp.csv = true; + set(interp, special::FS, Value::Str(",".into()))?; + } + if let Some(fs) = &opts.fs { + set(interp, special::FS, Value::Str(lexer::unescape(fs)))?; + } + for a in &opts.assigns { + let Some((name, val)) = io::cmdline_assignment(a) else { + let msg = format!( + "`{}' argument to `-v' not in `var=value' form", + interp::truncate(a, 200) + ); + interp.fatal(&msg); + return Err(2); + }; + let v = Value::from_input(lexer::unescape(val)); + interp.assign_by_name(name, v).map_err(|_| fail(interp))?; + } + Ok(()) + } -impl Awk { - /// Process C-style escape sequences in a string (e.g., \t → tab, \n → newline) - fn process_escape_sequences(s: &str) -> String { - let mut result = String::new(); - let mut chars = s.chars(); - while let Some(c) = chars.next() { - if c == '\\' { - match chars.next() { - Some('t') => result.push('\t'), - Some('n') => result.push('\n'), - Some('r') => result.push('\r'), - Some('\\') => result.push('\\'), - Some('a') => result.push('\x07'), - Some('b') => result.push('\x08'), - Some('f') => result.push('\x0C'), - Some(other) => { - result.push('\\'); - result.push(other); - } - None => result.push('\\'), - } - } else { - result.push(c); - } + fn result(interp: &mut Interp, code: i32) -> Result { + if let Some(e) = interp.budget_error.take() { + return Err(e.into()); } - result + let mut r = ExecResult::with_code(std::mem::take(&mut interp.io.stdout), code); + r.stderr = std::mem::take(&mut interp.io.stderr).into(); + Ok(r) + } + + /// Whether the program may run shell commands. + fn may_run_commands(args: &[String]) -> bool { + args.iter().any(|a| { + a == "-f" + || a.starts_with("--file") + || (a.starts_with("-f") && !a.starts_with("-F")) + || a.contains("system") + || a.replace("||", "").contains('|') + }) } } #[async_trait] impl Builtin for Awk { async fn execute(&self, ctx: Context<'_>) -> Result { - if let Some(r) = super::check_help_version( - ctx.args, - "Usage: awk [OPTION]... 'program' [FILE]...\nPattern scanning and processing language.\n\n -F SEP\t\tuse SEP as field separator\n -v var=val\tassign variable before execution\n -f progfile\tread program from file\n --csv, -k\tCSV mode (set field separator to comma)\n --help\tdisplay this help and exit\n --version\toutput version information and exit\n", - Some("awk (bashkit) 0.1"), - ) { - return Ok(r); - } - let mut program_str = String::new(); - let mut files: Vec = Vec::new(); - let mut field_sep = " ".to_string(); - let mut pre_vars: Vec<(String, String)> = Vec::new(); - let mut csv_mode = false; - let mut i = 0; - - while i < ctx.args.len() { - let arg = &ctx.args[i]; - if arg == "--csv" || arg == "-k" { - csv_mode = true; - field_sep = ",".to_string(); - } else if arg == "-F" { - i += 1; - if i < ctx.args.len() { - field_sep = ctx.args[i].clone(); - } - } else if let Some(sep) = arg.strip_prefix("-F") { - field_sep = sep.to_string(); - } else if arg == "-v" { - // Variable assignment: -v var=value - i += 1; - if i < ctx.args.len() - && let Some(eq_pos) = ctx.args[i].find('=') - { - let name = ctx.args[i][..eq_pos].to_string(); - let mut value = ctx.args[i][eq_pos + 1..].to_string(); - // Strip surrounding quotes if present (shell may pass them) - if (value.starts_with('"') && value.ends_with('"')) - || (value.starts_with('\'') && value.ends_with('\'')) - { - value = value[1..value.len() - 1].to_string(); - } - pre_vars.push((name, value)); - } - } else if arg == "-f" { - // Read program from file - i += 1; - if i < ctx.args.len() { - let path = if ctx.args[i].starts_with('/') { - std::path::PathBuf::from(&ctx.args[i]) - } else { - vfs_join(ctx.cwd, &ctx.args[i]) - }; - program_str = match read_text_file(&*ctx.fs, &path, "awk").await { - Ok(t) => t, - Err(e) => return Ok(e), - }; - ctx.consume_budget_input(program_str.len())?; - } - } else if arg.starts_with('-') { - // Unknown option - ignore - } else if program_str.is_empty() { - program_str = arg.clone(); - } else { - files.push(arg.clone()); + match self.setup(&ctx, None).await? { + Setup::Done(r) => Ok(*r), + Setup::Run(mut interp) => { + let code = interp.run().await; + Self::result(&mut interp, code) } - i += 1; } + } - if program_str.is_empty() { - return Err(Error::Execution("awk: no program given".to_string())); + async fn execution_plan(&self, ctx: &Context<'_>) -> Result> { + if !Self::may_run_commands(ctx.args) { + return Ok(None); } + let bridge: Bridge = Arc::new(Mutex::new(BridgeState::default())); + let mut interp = match self.setup(ctx, Some(bridge.clone())).await? { + Setup::Done(r) => { + return Ok(Some(ExecutionPlan::Driver(Box::new(AwkRun::done( + bridge, *r, + ))))); + } + Setup::Run(i) => i, + }; + let fut = async move { + let code = interp.run().await; + let emitted = interp.io.emitted; + Self::result(&mut interp, code).map(|r| (r, emitted)) + }; + Ok(Some(ExecutionPlan::Driver(Box::new(AwkRun { + fut: Some(Box::pin(fut)), + bridge, + queued: None, + })))) + } +} - let program_str = normalize_awk_newlines(&program_str); - let mut parser = AwkParser::new(&program_str); - let program = parser.parse()?; - - let mut interp = AwkInterpreter::new(); - interp.execution_budget = ctx - .execution_budget() - .and_then(|budget| budget.try_with(Clone::clone).ok()); - let (max_loop, max_total_loop, max_live) = ctx - .execution_extension::() - .and_then(|limits| { - limits - .try_with(|l| { - ( - l.max_loop_iterations, - l.max_total_loop_iterations, - l.max_live_intermediate_bytes, - ) - }) - .ok() - }) - .unwrap_or_else(|| { - let d = ExecutionLimits::default(); - ( - d.max_loop_iterations, - d.max_total_loop_iterations, - d.max_live_intermediate_bytes, - ) - }); - interp.max_loop_iterations = max_loop; - interp.max_total_loop_iterations = max_total_loop; - interp.max_state_bytes = usize::try_from(max_live).unwrap_or(usize::MAX); - interp.functions = program.functions.clone(); - interp - .state - .set_fs(Self::process_escape_sequences(&field_sep)); - interp.fs = Some(ctx.fs.clone()); - interp.cwd = ctx.cwd.clone(); - if csv_mode { - interp.state.csv_mode = true; - interp.state.ofs = ",".to_string(); - } +type RunFuture = Pin> + Send>>; - // Set pre-assigned variables (-v) - for (name, value) in &pre_vars { - let awk_val = if let Ok(n) = value.parse::() { - AwkValue::Number(n) - } else { - AwkValue::String(value.clone()) - }; - interp.state.set_variable(name, awk_val); - } +/// Plan driver: runs the awk program and fulfils its command requests. +struct AwkRun { + fut: Option, + bridge: Bridge, + queued: Option, +} - // Run BEGIN actions - let mut exit_code: Option = None; - for action in &program.begin_actions { - if let AwkFlow::Exit(code) = interp.exec_action(action) { - exit_code = code; - // Run END actions even after exit - for end_action in &program.end_actions { - if let AwkFlow::Exit(_) = interp.exec_action(end_action) { - break; - } - } - Self::flush_file_outputs(&interp, &ctx).await?; - return Ok(Self::finish(interp, exit_code)); - } - } - if interp.is_fatal() { - // A limit hit mid-action in BEGIN: no input, no END. - return Ok(Self::finish(interp, exit_code)); +impl AwkRun { + fn done(bridge: Bridge, r: ExecResult) -> Self { + AwkRun { + fut: None, + bridge, + queued: Some(PlanStep::Done(r)), } + } +} - // Process input - let inputs: Vec = if files.is_empty() { - vec![ctx.stdin.map(ToString::to_string).unwrap_or_default()] - } else { - let mut inputs = Vec::new(); - for file in &files { - let path = if file.starts_with('/') { - std::path::PathBuf::from(file) - } else { - vfs_join(ctx.cwd, file) - }; - - let text = match read_text_file(&*ctx.fs, &path, "awk").await { - Ok(t) => t, - Err(e) => return Ok(e), - }; - ctx.consume_budget_input(text.len())?; - inputs.push(text); +#[async_trait] +impl PlanDriver for AwkRun { + async fn next(&mut self, last: Option) -> Result { + if let Some(r) = last { + if let Ok(mut s) = self.bridge.lock() { + s.response = Some(r); } - inputs + } else if let Some(step) = self.queued.take() { + return Ok(step); + } + let Some(fut) = self.fut.as_mut() else { + return Ok(PlanStep::Done(ExecResult::default())); }; - - 'files: for (file_idx, input) in inputs.iter().enumerate() { - interp.state.fnr = 0; - // Set FILENAME to current file path, or empty for stdin - if !files.is_empty() { - interp.state.insert_var( - "FILENAME".to_string(), - AwkValue::String(files[file_idx].clone()), - ); - } else { - interp - .state - .insert_var("FILENAME".to_string(), AwkValue::String(String::new())); + let bridge = self.bridge.clone(); + // Ready(Some) when awk finished, Ready(None) when it waits on a + // command; real I/O pending inside awk keeps us pending. + let outcome = std::future::poll_fn(|cx| match fut.as_mut().poll(cx) { + Poll::Ready(r) => Poll::Ready(Some(r)), + Poll::Pending => { + if bridge.lock().is_ok_and(|s| s.request.is_some()) { + Poll::Ready(None) + } else { + Poll::Pending + } } - // Index-based iteration so getline can advance the index - interp.input_lines = input.lines().map(|l| l.to_string()).collect(); - interp.line_index = 0; - - while interp.line_index < interp.input_lines.len() { - ctx.consume_budget_work(1)?; - let line = interp.input_lines[interp.line_index].clone(); - interp.state.set_line(&line); - - for (rule_idx, rule) in program.main_rules.iter().enumerate() { - // Check pattern (with range state tracking) - let matches = match &rule.pattern { - Some(pattern) => interp.matches_pattern_with_index(pattern, rule_idx), - None => true, - }; - - if matches { - let mut next_record = false; - for action in &rule.actions { - match interp.exec_action(action) { - AwkFlow::Continue => {} - AwkFlow::Next => { - next_record = true; - break; - } - AwkFlow::Exit(code) => { - exit_code = code; - break 'files; - } - _ => {} - } - } - if next_record { - break; - } - } + }) + .await; + match outcome { + Some(res) => { + self.fut = None; + let (r, emitted) = res?; + // Stream what awk printed since the last command. + let out = r.stdout.as_bytes().get(emitted.0..).unwrap_or_default(); + let err = r.stderr.as_bytes().get(emitted.1..).unwrap_or_default(); + if emitted == (0, 0) || (out.is_empty() && err.is_empty()) { + return Ok(PlanStep::Done(r)); } - interp.line_index += 1; + let step = PlanStep::Emit { + stdout: StreamData::from(out.to_vec()), + stderr: StreamData::from(err.to_vec()), + }; + self.queued = Some(PlanStep::Done(r)); + Ok(step) } - } - - // Run END actions (awk runs END even after exit in main body) - for action in &program.end_actions { - if let AwkFlow::Exit(code) = interp.exec_action(action) { - if exit_code.is_none() { - exit_code = code; + None => { + let Some(req) = self.bridge.lock().ok().and_then(|mut s| s.request.take()) else { + return Ok(PlanStep::Done(ExecResult::default())); + }; + let command = SubCommand { + name: "sh".to_string(), + args: vec!["-c".to_string(), req.command], + stdin: req.stdin.map(StreamData::from), + assignments: Vec::new(), + }; + let run = if req.capture { + PlanStep::Capture { command, cwd: None } + } else { + PlanStep::Run { command, cwd: None } + }; + if req.emit_stdout.is_empty() && req.emit_stderr.is_empty() { + return Ok(run); } - break; + self.queued = Some(run); + Ok(PlanStep::Emit { + stdout: StreamData::from(req.emit_stdout), + stderr: StreamData::from(req.emit_stderr), + }) } } - - Self::flush_file_outputs(&interp, &ctx).await?; - Ok(Self::finish(interp, exit_code)) - } -} - -impl Awk { - /// A fatal limit error exits 2 whatever `exit` code the program chose, - /// even when it fired inside an expression and the action ran on. - fn finish(interp: AwkInterpreter, exit_code: Option) -> ExecResult { - let code = if interp.is_fatal() { - 2 - } else { - exit_code.unwrap_or(0) - }; - let mut result = ExecResult::with_code(interp.output, code); - result.stderr = interp.stderr_output.into(); - result - } - - /// AWK redirection streams through VFS as output is produced. - async fn flush_file_outputs(_interp: &AwkInterpreter, _ctx: &Context<'_>) -> Result<()> { - Ok(()) } } diff --git a/crates/bashkit/src/builtins/awk/parser.rs b/crates/bashkit/src/builtins/awk/parser.rs index 66c7949ad..d3099e8f0 100644 --- a/crates/bashkit/src/builtins/awk/parser.rs +++ b/crates/bashkit/src/builtins/awk/parser.rs @@ -1,1675 +1,1716 @@ -//! AWK parser - turns a program string into an `AwkProgram` AST. +//! awk parser: recursive descent over the lazy lexer. +//! +//! Decisions: +//! - Precedence follows POSIX (lowest first): assignment, `?:`, `||`, `&&`, +//! `in`, `~ !~`, relational, `cmd | getline`, concatenation, additive, +//! multiplicative, unary `! + -`, `^` (right associative), `++ --`, `$`, +//! grouping. +//! - Inside `print`/`printf` arguments an unparenthesized `>` is output +//! redirection, not a comparison. +//! - THREAT[TM-DOS-027]: syntactic nesting (parentheses, unary operators, +//! blocks) is capped at `AWK_MAX_PARSER_DEPTH`; operator chains +//! (`a+b+c...`, long concatenations) also deepen the tree, so they count +//! against a larger total-depth cap. The evaluator recurses over this +//! tree, so the caps bound its stack use. use std::collections::HashMap; -use super::{AwkAction, AwkExpr, AwkFunctionDef, AwkOutputTarget, AwkPattern, AwkProgram, AwkRule}; +use super::ast::*; +use super::lexer::{LexError, Lexer, Tok}; use crate::builtins::limits::{ - AWK_MAX_MULTI_SUBSCRIPTS as MAX_AWK_MULTI_SUBSCRIPTS, - AWK_MAX_PARSER_DEPTH as MAX_AWK_PARSER_DEPTH, + AWK_MAX_MULTI_SUBSCRIPTS as MAX_SUBSCRIPTS, AWK_MAX_PARSER_DEPTH as MAX_DEPTH, }; -use crate::builtins::search_common::build_regex; -use crate::error::{Error, Result}; - -pub(super) struct AwkParser<'a> { - input: &'a str, - pos: usize, - /// Current recursion depth for expression parsing - depth: usize, - /// When true, `>` and `>>` are output redirection, not comparison ops. - /// Set during print/printf argument parsing per POSIX awk semantics. - in_print_context: bool, + +/// Total tree depth cap, operator chains included. +const MAX_TREE_DEPTH: usize = 1000; + +#[derive(Debug)] +pub(super) struct ParseError { + pub(super) msg: String, + pub(super) pos: usize, + pub(super) kind: ParseErrorKind, } -impl<'a> AwkParser<'a> { - fn is_identifier_start(c: char) -> bool { - c.is_alphabetic() || c == '_' - } +#[derive(Debug, PartialEq)] +pub(super) enum ParseErrorKind { + /// gawk "syntax error" style, printed with the source line and a caret. + Syntax, + /// A gawk `error:` without source echo (constant division by zero). + Error, + /// A gawk `fatal:` found while parsing (undefined function). + Fatal, + /// A bashkit resource limit. + Limit, +} - fn is_identifier_continue(c: char) -> bool { - c.is_alphanumeric() || c == '_' - } +type PResult = Result; - pub(super) fn new(input: &'a str) -> Self { - Self { - input, - pos: 0, - depth: 0, - in_print_context: false, +impl From for ParseError { + fn from(e: LexError) -> Self { + ParseError { + msg: e.msg, + pos: e.pos, + kind: ParseErrorKind::Syntax, } } +} - /// Get the character at the current byte position (char-boundary safe). - /// Returns None if pos is at or past end of input. - fn current_char(&self) -> Option { - self.input[self.pos..].chars().next() - } +pub(super) struct Parser<'a> { + lex: Lexer<'a>, + src: &'a str, + tok: Tok, + tok_pos: usize, + prog: Program, + global_index: HashMap, + func_index: HashMap, + /// Parameters of the function being parsed. + locals: Option>, + rec_depth: usize, + tree_depth: usize, + /// Parsing print arguments: `>` is redirection. + in_print: bool, + /// Parenthesis nesting inside print arguments. + paren_level: usize, + /// Inside a loop / switch (for break/continue checks). + loop_depth: usize, + /// Source labels for statement locations. + sources: &'a [Source], + loc_index: HashMap<(usize, u32), u32>, + /// Calls seen before their function's definition: (func id, offset). + pending_calls: Vec<(u32, usize)>, +} - /// Advance pos past the current character (handles multi-byte UTF-8). - fn advance(&mut self) { - if let Some(c) = self.current_char() { - self.pos += c.len_utf8(); - } - } +#[derive(Clone)] +struct Saved { + lex_pos: usize, + tok: Tok, + tok_pos: usize, +} - fn consume_while(&mut self, predicate: fn(char) -> bool) { - while self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if predicate(c) { - self.advance(); - } else { - break; - } +impl<'a> Parser<'a> { + pub(super) fn new(src: &'a str, sources: &'a [Source]) -> Self { + let mut prog = Program::default(); + let mut global_index = HashMap::new(); + for (i, name) in special::NAMES.iter().enumerate() { + prog.globals.push((*name).to_string()); + global_index.insert((*name).to_string(), i as u32); + } + Parser { + lex: Lexer::new(src), + src, + tok: Tok::Eof, + tok_pos: 0, + prog, + global_index, + func_index: HashMap::new(), + locals: None, + rec_depth: 0, + tree_depth: 0, + in_print: false, + paren_level: 0, + loop_depth: 0, + sources, + loc_index: HashMap::new(), + pending_calls: Vec::new(), } } - /// THREAT[TM-DOS-027]: Increment depth, error if limit exceeded - fn push_depth(&mut self) -> Result<()> { - self.depth += 1; - if self.depth > MAX_AWK_PARSER_DEPTH { - return Err(Error::Execution(format!( - "awk: expression nesting too deep ({} levels, max {})", - self.depth, MAX_AWK_PARSER_DEPTH - ))); - } + // ----- token plumbing ----- + + fn advance(&mut self) -> PResult<()> { + let (t, p) = self.lex.next()?; + self.tok = t; + self.tok_pos = p; Ok(()) } - /// Decrement depth after leaving a recursive parse - fn pop_depth(&mut self) { - if self.depth > 0 { - self.depth -= 1; + fn save(&self) -> Saved { + Saved { + lex_pos: self.lex.pos(), + tok: self.tok.clone(), + tok_pos: self.tok_pos, } } - pub(super) fn parse(&mut self) -> Result { - let mut program = AwkProgram { - begin_actions: Vec::new(), - main_rules: Vec::new(), - end_actions: Vec::new(), - functions: HashMap::new(), - }; - - self.skip_whitespace(); - - while self.pos < self.input.len() { - self.skip_whitespace(); - if self.pos >= self.input.len() { - break; - } - - // Check for function/BEGIN/END - if self.matches_keyword("function") { - self.skip_whitespace(); - let (name, func_def) = self.parse_function_def()?; - program.functions.insert(name, func_def); - } else if self.matches_keyword("BEGIN") { - self.skip_whitespace(); - let actions = self.parse_action_block()?; - program.begin_actions.extend(actions); - } else if self.matches_keyword("END") { - self.skip_whitespace(); - let actions = self.parse_action_block()?; - program.end_actions.extend(actions); - } else { - // Pattern-action rule - let rule = self.parse_rule()?; - program.main_rules.push(rule); - } - - self.skip_whitespace(); - } - - // If no rules, add default print rule - if program.main_rules.is_empty() - && program.begin_actions.is_empty() - && program.end_actions.is_empty() - { - program.main_rules.push(AwkRule { - pattern: None, - actions: vec![AwkAction::Print( - vec![AwkExpr::Field(Box::new(AwkExpr::Number(0.0)))], - None, - )], - }); - } - - Ok(program) + fn restore(&mut self, s: Saved) { + self.lex.reset(s.lex_pos); + self.tok = s.tok; + self.tok_pos = s.tok_pos; } - /// Parse a user-defined function: function name(params) { body } - fn parse_function_def(&mut self) -> Result<(String, AwkFunctionDef)> { - // Parse function name - let name = self.read_identifier()?; - self.skip_whitespace(); - - // Expect '(' - if self.pos >= self.input.len() || self.current_char().unwrap() != '(' { - return Err(Error::Execution( - "awk: expected '(' after function name".to_string(), - )); - } - self.pos += 1; - - // Parse parameter list - let mut params = Vec::new(); - self.skip_whitespace(); - while self.pos < self.input.len() && self.current_char().unwrap() != ')' { - if !params.is_empty() { - if self.current_char().unwrap() == ',' { - self.pos += 1; - } - self.skip_whitespace(); - } - if self.pos < self.input.len() && self.current_char().unwrap() != ')' { - params.push(self.read_identifier()?); - self.skip_whitespace(); - } - } - - // Expect ')' - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - return Err(Error::Execution( - "awk: expected ')' after function parameters".to_string(), - )); - } - self.pos += 1; - self.skip_whitespace(); - - // Parse function body as action block - let body = self.parse_action_block()?; - - Ok((name, AwkFunctionDef { params, body })) + fn err(&self, msg: &str) -> PResult { + Err(ParseError { + msg: msg.to_string(), + pos: self.tok_pos, + kind: ParseErrorKind::Syntax, + }) } - /// Read an identifier (alphanumeric + underscore) - fn read_identifier(&mut self) -> Result { - let start = self.pos; - self.consume_while(Self::is_identifier_continue); - if self.pos == start { - return Err(Error::Execution("awk: expected identifier".to_string())); + fn syntax(&self) -> PResult { + if matches!(self.tok, Tok::Eof | Tok::Newline) { + self.err("unexpected newline or end of string") + } else { + self.err("syntax error") } - Ok(self.input[start..self.pos].to_string()) } - fn matches_keyword(&mut self, keyword: &str) -> bool { - if self.input[self.pos..].starts_with(keyword) { - let after = self.pos + keyword.len(); - if after >= self.input.len() || { - let c = self.input[after..].chars().next().unwrap(); - !c.is_alphanumeric() && c != '_' - } { - self.pos = after; - return true; - } + fn expect(&mut self, t: Tok) -> PResult<()> { + if self.tok == t { + self.advance() + } else { + self.syntax() } - false } - fn skip_whitespace(&mut self) { - while self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if c.is_whitespace() { - self.advance(); - } else if c == '#' { - // Comment - skip to end of line (may contain multi-byte chars) - while self.pos < self.input.len() && self.current_char().unwrap() != '\n' { - self.advance(); - } - } else { - break; - } + fn opt_nls(&mut self) -> PResult<()> { + while self.tok == Tok::Newline { + self.advance()?; } + Ok(()) } - fn parse_rule(&mut self) -> Result { - let pattern = self.parse_pattern()?; - self.skip_whitespace(); - - let actions = if self.pos < self.input.len() && self.current_char().unwrap() == '{' { - self.parse_action_block()? - } else if pattern.is_some() { - // Default action is print - vec![AwkAction::Print( - vec![AwkExpr::Field(Box::new(AwkExpr::Number(0.0)))], - None, - )] - } else { - Vec::new() - }; - - Ok(AwkRule { pattern, actions }) - } - - /// Parse a `/regex/` literal into an `AwkPattern::Regex`. - /// Assumes the leading `/` is at `self.pos`. - fn parse_regex_pattern(&mut self) -> Result { - self.pos += 1; // skip opening '/' - let start = self.pos; - while self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if c == '/' { - let pattern = &self.input[start..self.pos]; - self.pos += 1; - let regex = build_regex(pattern) - .map_err(|e| Error::Execution(format!("awk: invalid regex: {}", e)))?; - return Ok(AwkPattern::Regex(regex)); - } else if c == '\\' { - self.pos += 1; // skip '\\' (ASCII) - self.advance(); // skip next char (may be multi-byte) - } else { - self.advance(); // regex content may be multi-byte - } + fn enter(&mut self) -> PResult<(usize, usize)> { + let saved = (self.rec_depth, self.tree_depth); + self.rec_depth += 1; + self.tree_depth += 1; + if self.rec_depth > MAX_DEPTH || self.tree_depth > MAX_TREE_DEPTH { + return Err(ParseError { + msg: format!( + "expression nesting too deep ({} levels, max {MAX_DEPTH})", + self.rec_depth.max(self.tree_depth) + ), + pos: self.tok_pos, + kind: ParseErrorKind::Limit, + }); } - Err(Error::Execution("awk: unterminated regex".to_string())) + Ok(saved) } - fn parse_pattern(&mut self) -> Result> { - let Some(first_pat) = self.parse_single_pattern()? else { - return Ok(None); - }; - - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ',' { - return Ok(Some(first_pat)); - } - - // THREAT[TM-DOS-027]: awk ranges contain exactly two operands. Do not - // recursively parse comma chains (`1,1,1,...`) because attacker-sized - // chains can exhaust the host stack before runtime limits apply. - self.pos += 1; // consume ',' - let second_pat = self - .parse_single_pattern()? - .ok_or_else(|| Error::Execution("awk: expected second pattern in range".to_string()))?; + fn leave(&mut self, saved: (usize, usize)) { + self.rec_depth = saved.0; + self.tree_depth = saved.1; + } - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == ',' { - return Err(Error::Execution( - "awk: unexpected ',' after range pattern".to_string(), - )); + /// One more link in an operator chain. + fn chain(&mut self) -> PResult<()> { + self.tree_depth += 1; + if self.tree_depth > MAX_TREE_DEPTH { + return Err(ParseError { + msg: format!("expression nesting too deep (max {MAX_TREE_DEPTH} operators)"), + pos: self.tok_pos, + kind: ParseErrorKind::Limit, + }); } - - Ok(Some(AwkPattern::Range( - Box::new(first_pat), - Box::new(second_pat), - ))) + Ok(()) } - fn parse_single_pattern(&mut self) -> Result> { - self.skip_whitespace(); + fn loc(&mut self, pos: usize) -> u32 { + let si = self + .sources + .iter() + .rposition(|s| s.start <= pos) + .unwrap_or(0); + let start = self.sources.get(si).map_or(0, |s| s.start); + let line = self.src[start..pos.min(self.src.len())] + .bytes() + .filter(|&b| b == b'\n') + .count() as u32 + + 1; + if let Some(&i) = self.loc_index.get(&(si, line)) { + return i; + } + let name = self + .sources + .get(si) + .map_or_else(|| "cmd. line".to_string(), |s| s.name.clone()); + let i = self.prog.locs.len() as u32; + self.prog.locs.push((name, line)); + self.loc_index.insert((si, line), i); + i + } - if self.pos >= self.input.len() { - return Ok(None); + fn global(&mut self, name: &str) -> u32 { + if let Some(&i) = self.global_index.get(name) { + return i; } + let i = self.prog.globals.len() as u32; + self.prog.globals.push(name.to_string()); + self.global_index.insert(name.to_string(), i); + i + } - let c = self.current_char().unwrap(); - - if c == '/' { - let start = self.pos; - let pattern = self.parse_regex_pattern()?; - let save = self.pos; - self.skip_whitespace(); - // `/re/ && cond`, `/a/ || /b/`: the regex is the left operand - // of a larger pattern expression. - if matches!(self.current_char(), Some('&' | '|' | '?')) { - self.pos = start; - let expr = self.parse_expression()?; - return Ok(Some(AwkPattern::Expression(expr))); - } - self.pos = save; - Ok(Some(pattern)) - } else if c == '{' { - Ok(None) - } else { - let expr = self.parse_expression()?; - Ok(Some(AwkPattern::Expression(expr))) + fn var(&mut self, name: &str) -> VarRef { + if let Some(l) = &self.locals + && let Some(&i) = l.get(name) + { + return VarRef::Local(i); } + VarRef::Global(self.global(name)) } - fn parse_action_block(&mut self) -> Result> { - self.skip_whitespace(); + fn func_id(&mut self, name: &str) -> u32 { + if let Some(&i) = self.func_index.get(name) { + return i; + } + let i = self.prog.funcs.len() as u32; + self.prog.funcs.push(Func { + name: name.to_string(), + ..Func::default() + }); + self.func_index.insert(name.to_string(), i); + i + } - if self.pos >= self.input.len() || self.current_char().unwrap() != '{' { - return Err(Error::Execution("awk: expected '{'".to_string())); - } - self.pos += 1; + fn regex_id(&mut self, re: String) -> u32 { + let i = self.prog.regexes.len() as u32; + self.prog.regexes.push(re); + i + } - let mut actions = Vec::new(); + // ----- program ----- + pub(super) fn parse_program(mut self) -> PResult { + self.advance()?; loop { - self.skip_whitespace(); - if self.pos >= self.input.len() { - return Err(Error::Execution( - "awk: unterminated action block".to_string(), - )); - } - - // Skip empty statements (consecutive semicolons from newline normalization) - while self.pos < self.input.len() && self.current_char().unwrap() == ';' { - self.pos += 1; - self.skip_whitespace(); - } - if self.pos >= self.input.len() { - return Err(Error::Execution( - "awk: unterminated action block".to_string(), - )); + while matches!(self.tok, Tok::Newline | Tok::Semi) { + self.advance()?; } - - let c = self.current_char().unwrap(); - if c == '}' { - self.pos += 1; + if self.tok == Tok::Eof { break; } - - let action = self.parse_action()?; - actions.push(action); - - self.skip_whitespace(); - // Allow semicolon separator - if self.pos < self.input.len() && self.current_char().unwrap() == ';' { - self.pos += 1; + self.parse_item()?; + } + for (f, pos) in std::mem::take(&mut self.pending_calls) { + if !self.prog.funcs[f as usize].defined { + return Err(ParseError { + msg: format!( + "function `{}' not defined", + self.prog.funcs[f as usize].name + ), + pos, + kind: ParseErrorKind::Fatal, + }); } } - - Ok(actions) + analyze_array_params(&mut self.prog); + Ok(self.prog) } - fn parse_action(&mut self) -> Result { - self.skip_whitespace(); - - // Check for keywords - if self.matches_keyword("print") { - return self.parse_print(); - } - if self.matches_keyword("printf") { - return self.parse_printf(); - } - if self.matches_keyword("next") { - return Ok(AwkAction::Next); - } - if self.matches_keyword("break") { - return Ok(AwkAction::Break); - } - if self.matches_keyword("continue") { - return Ok(AwkAction::Continue); - } - if self.matches_keyword("delete") { - return self.parse_delete(); - } - if self.matches_keyword("getline") { - return self.parse_getline(); - } - if self.matches_keyword("exit") { - self.skip_whitespace(); - if self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if c != '}' && c != ';' { - let expr = self.parse_expression()?; - return Ok(AwkAction::Exit(Some(expr))); + fn parse_item(&mut self) -> PResult<()> { + match self.tok.clone() { + Tok::Function => self.parse_function(), + Tok::Begin | Tok::End | Tok::BeginFile | Tok::EndFile => { + let which = self.tok.clone(); + self.advance()?; + self.opt_nls()?; + if self.tok != Tok::LBrace { + return self.syntax(); } - } - return Ok(AwkAction::Exit(None)); - } - if self.matches_keyword("return") { - self.skip_whitespace(); - if self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if c != '}' && c != ';' { - let expr = self.parse_expression()?; - return Ok(AwkAction::Return(Some(expr))); + let block = self.parse_block()?; + match which { + Tok::Begin => self.prog.begin.push(block), + Tok::End => self.prog.end.push(block), + Tok::BeginFile => self.prog.beginfile.push(block), + _ => self.prog.endfile.push(block), } + Ok(()) + } + Tok::LBrace => { + let body = self.parse_block()?; + self.prog.rules.push(Rule { + pattern: Pattern::All, + body: Some(body), + }); + Ok(()) + } + _ => { + let first = self.parse_expr()?; + let pattern = if self.tok == Tok::Comma { + self.advance()?; + self.opt_nls()?; + let second = self.parse_expr()?; + let id = self.prog.ranges; + self.prog.ranges += 1; + Pattern::Range(first, second, id) + } else { + Pattern::Expr(first) + }; + let body = if self.tok == Tok::LBrace { + Some(self.parse_block()?) + } else { + match self.tok { + Tok::Newline | Tok::Semi | Tok::Eof => {} + _ => return self.syntax(), + } + None + }; + self.prog.rules.push(Rule { pattern, body }); + Ok(()) } - return Ok(AwkAction::Return(None)); - } - if self.matches_keyword("if") { - return self.parse_if(); - } - if self.matches_keyword("for") { - return self.parse_for(); } - if self.matches_keyword("while") { - return self.parse_while(); + } + + fn parse_function(&mut self) -> PResult<()> { + self.advance()?; + let name = match self.tok.clone() { + Tok::Name(n) | Tok::FuncName(n) => n, + _ => return self.syntax(), + }; + let fpos = self.tok_pos; + self.advance()?; + if self.global_index.contains_key(&name) && special::NAMES.contains(&name.as_str()) { + return Err(ParseError { + msg: format!("function name `{name}' previously defined"), + pos: fpos, + kind: ParseErrorKind::Syntax, + }); } - if self.matches_keyword("do") { - return self.parse_do_while(); + self.expect(Tok::LParen)?; + let mut params = Vec::new(); + let mut map = HashMap::new(); + self.opt_nls()?; + while let Tok::Name(p) = self.tok.clone() { + if map.contains_key(&p) { + return self.err(&format!( + "function `{name}': parameter #{} `{p}' duplicates parameter #{}", + params.len() + 1, + map[&p] + 1 + )); + } + map.insert(p.clone(), params.len() as u32); + params.push(p); + self.advance()?; + self.opt_nls()?; + if self.tok == Tok::Comma { + self.advance()?; + self.opt_nls()?; + } else { + break; + } } - - // Otherwise it's an expression (including assignment) - let expr = self.parse_expression()?; - - // Check if it's an assignment - match expr { - AwkExpr::Assign(name, val) => Ok(AwkAction::Assign(name, *val)), - AwkExpr::ArrayAssign(name, key, val) => Ok(AwkAction::ArrayAssign(name, *key, *val)), - _ => Ok(AwkAction::Expression(expr)), + self.expect(Tok::RParen)?; + self.opt_nls()?; + let id = self.func_id(&name); + if self.prog.funcs[id as usize].defined { + return Err(ParseError { + msg: format!("function `{name}' previously defined"), + pos: fpos, + kind: ParseErrorKind::Syntax, + }); } + if self.tok != Tok::LBrace { + return self.syntax(); + } + self.locals = Some(map); + let body = self.parse_block()?; + self.locals = None; + let n = params.len(); + let f = &mut self.prog.funcs[id as usize]; + f.params = params; + f.body = body; + f.array_params = vec![false; n]; + f.defined = true; + Ok(()) } - fn parse_print(&mut self) -> Result { - self.skip_whitespace(); - let mut args = Vec::new(); - - // Enable print context so `>` and `>>` are not parsed as comparison - self.in_print_context = true; + // ----- statements ----- + fn parse_block(&mut self) -> PResult { + let saved = self.enter()?; + self.expect(Tok::LBrace)?; + let mut stmts = Vec::new(); loop { - if self.pos >= self.input.len() { - break; + while matches!(self.tok, Tok::Newline | Tok::Semi) { + self.advance()?; } - let c = self.current_char().unwrap(); - if c == '}' || c == ';' { + if self.tok == Tok::RBrace { + self.advance()?; break; } - // Stop at output redirection operators - if c == '>' || c == '|' { - break; - } - - let expr = self.parse_expression()?; - args.push(expr); - - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == ',' { - self.pos += 1; - self.skip_whitespace(); - } else { - break; + if self.tok == Tok::Eof { + return self.syntax(); } + stmts.push(self.parse_stmt()?); } - - if args.is_empty() { - args.push(AwkExpr::Field(Box::new(AwkExpr::Number(0.0)))); - } - - let target = self.parse_output_target()?; - self.in_print_context = false; - - Ok(AwkAction::Print(args, target)) + self.leave(saved); + Ok(stmts) } - fn parse_printf(&mut self) -> Result { - self.skip_whitespace(); - - // Enable print context so `>` and `>>` are not parsed as comparison - self.in_print_context = true; - - // Handle optional parenthesized form: printf("format", args) - let has_parens = self.pos < self.input.len() && self.current_char().unwrap() == '('; - if has_parens { - self.pos += 1; - self.skip_whitespace(); - } - - // Parse format string — accepts string literals or expressions - if self.pos >= self.input.len() { - self.in_print_context = false; - return Err(Error::Execution( - "awk: printf requires format string".to_string(), - )); - } - - let format_expr = if self.current_char().unwrap() == '"' { - // String literal format — parse directly - let s = self.parse_string()?; - AwkExpr::String(s) - } else { - // Expression as format string (e.g., printf substr($1,1,1)) - self.parse_expression()? - }; - let mut args = Vec::new(); - - self.skip_whitespace(); - while self.pos < self.input.len() && self.current_char().unwrap() == ',' { - self.pos += 1; - self.skip_whitespace(); - let expr = self.parse_expression()?; - args.push(expr); - self.skip_whitespace(); + /// Statement terminator: `;`, newline, or lookahead `}` / EOF. + fn end_simple(&mut self) -> PResult<()> { + match self.tok { + Tok::Semi | Tok::Newline => self.advance(), + Tok::RBrace | Tok::Eof => Ok(()), + _ => self.syntax(), } - - if has_parens && self.pos < self.input.len() && self.current_char().unwrap() == ')' { - self.pos += 1; - } - - let target = self.parse_output_target()?; - self.in_print_context = false; - - Ok(AwkAction::Printf(format_expr, args, target)) } - /// Parse optional output target after print/printf arguments: `> file`, `>> file`, `| cmd`. - /// Pipe is unsupported and returns a clear error. - fn parse_output_target(&mut self) -> Result> { - self.skip_whitespace(); - if self.pos >= self.input.len() { - return Ok(None); - } + /// Body of if/while/for: a statement, possibly after newlines; a bare + /// `;` is an empty body. + fn parse_body(&mut self) -> PResult { + self.opt_nls()?; + if self.tok == Tok::Semi { + self.advance()?; + return Ok(Vec::new()); + } + let s = self.parse_stmt()?; + Ok(match s.kind { + StmtKind::Block(b) => b, + _ => vec![s], + }) + } - let c = self.current_char().unwrap(); - if c == '>' { - self.pos += 1; - // Check for >> - let append = self.pos < self.input.len() && self.current_char().unwrap() == '>'; - if append { - self.pos += 1; + fn parse_stmt(&mut self) -> PResult { + let saved = self.enter()?; + let pos = self.tok_pos; + let loc = self.loc(pos); + let kind = match self.tok.clone() { + Tok::LBrace => StmtKind::Block(self.parse_block()?), + Tok::If => { + self.advance()?; + self.expect(Tok::LParen)?; + let cond = self.parse_expr()?; + self.expect(Tok::RParen)?; + let then = self.parse_body()?; + // Look past terminators for `else`. + let s = self.save(); + while matches!(self.tok, Tok::Newline | Tok::Semi) { + self.advance()?; + } + let els = if self.tok == Tok::Else { + self.advance()?; + Some(self.parse_body()?) + } else { + self.restore(s); + None + }; + StmtKind::If(cond, then, els) } - self.skip_whitespace(); - // Temporarily disable print context to parse the target as a normal expression - self.in_print_context = false; - let target = self.parse_expression()?; - self.in_print_context = true; - if append { - Ok(Some(AwkOutputTarget::Append(target))) - } else { - Ok(Some(AwkOutputTarget::Truncate(target))) + Tok::While => { + self.advance()?; + self.expect(Tok::LParen)?; + let cond = self.parse_expr()?; + self.expect(Tok::RParen)?; + if self.tok == Tok::Semi { + self.advance()?; + StmtKind::While(cond, Vec::new()) + } else { + self.loop_depth += 1; + let body = self.parse_body()?; + self.loop_depth -= 1; + StmtKind::While(cond, body) + } } - } else if c == '|' { - // Pipe output (e.g., `print ... | "cmd"`) not supported in virtual mode - Err(Error::Execution( - "awk: pipe output redirection (|) is not supported".to_string(), - )) - } else { - Ok(None) - } + Tok::Do => { + self.advance()?; + self.loop_depth += 1; + let body = self.parse_body()?; + self.loop_depth -= 1; + while matches!(self.tok, Tok::Newline | Tok::Semi) { + self.advance()?; + } + self.expect(Tok::While)?; + self.expect(Tok::LParen)?; + let cond = self.parse_expr()?; + self.expect(Tok::RParen)?; + self.end_simple()?; + StmtKind::Do(body, cond) + } + Tok::For => self.parse_for()?, + Tok::Semi => { + self.advance()?; + StmtKind::Block(Vec::new()) + } + Tok::Switch => self.parse_switch()?, + _ => { + let k = self.parse_simple_or_jump()?; + self.end_simple()?; + k + } + }; + self.leave(saved); + Ok(Stmt { kind, loc }) } - /// THREAT[TM-DOS-027]: Track depth for nested if/action blocks - fn parse_if(&mut self) -> Result { - self.push_depth()?; - - self.skip_whitespace(); - - if self.pos >= self.input.len() || self.current_char().unwrap() != '(' { - self.pop_depth(); - return Err(Error::Execution("awk: expected '(' after if".to_string())); - } - self.pos += 1; - - let condition = self.parse_expression()?; - - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - self.pop_depth(); - return Err(Error::Execution( - "awk: expected ')' after condition".to_string(), - )); + fn parse_for(&mut self) -> PResult { + self.advance()?; + self.expect(Tok::LParen)?; + // for (k in a) + if let Tok::Name(k) = self.tok.clone() { + let s = self.save(); + self.advance()?; + if self.tok == Tok::In { + self.advance()?; + if let Tok::Name(a) = self.tok.clone() { + self.advance()?; + if self.tok == Tok::RParen { + self.advance()?; + let key = Expr::Var(self.var(&k)); + let arr = Expr::Var(self.var(&a)); + self.loop_depth += 1; + let body = self.parse_body()?; + self.loop_depth -= 1; + return Ok(StmtKind::ForIn(key, arr, body)); + } + } + } + self.restore(s); } - self.pos += 1; - - self.skip_whitespace(); - let then_actions = if self.current_char().unwrap() == '{' { - self.parse_action_block()? + let init = if self.tok == Tok::Semi { + None } else { - vec![self.parse_action()?] + let loc = self.loc(self.tok_pos); + Some(Box::new(Stmt { + kind: self.parse_simple_or_jump()?, + loc, + })) }; - - self.skip_whitespace(); - // Consume optional ';' before else - if self.pos < self.input.len() && self.current_char().unwrap() == ';' { - self.pos += 1; - self.skip_whitespace(); - } - let else_actions = if self.matches_keyword("else") { - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == '{' { - self.parse_action_block()? - } else { - vec![self.parse_action()?] - } + self.expect(Tok::Semi)?; + self.opt_nls()?; + let cond = if self.tok == Tok::Semi { + None } else { - Vec::new() + Some(self.parse_expr()?) }; - - self.pop_depth(); - Ok(AwkAction::If(condition, then_actions, else_actions)) + self.expect(Tok::Semi)?; + self.opt_nls()?; + let incr = if self.tok == Tok::RParen { + None + } else { + let loc = self.loc(self.tok_pos); + Some(Box::new(Stmt { + kind: self.parse_simple_or_jump()?, + loc, + })) + }; + self.expect(Tok::RParen)?; + if self.tok == Tok::Semi { + self.advance()?; + return Ok(StmtKind::For(init, cond, incr, Vec::new())); + } + self.loop_depth += 1; + let body = self.parse_body()?; + self.loop_depth -= 1; + Ok(StmtKind::For(init, cond, incr, body)) } - fn parse_for(&mut self) -> Result { - self.skip_whitespace(); - - if self.pos >= self.input.len() || self.current_char().unwrap() != '(' { - return Err(Error::Execution("awk: expected '(' after for".to_string())); + fn parse_switch(&mut self) -> PResult { + self.advance()?; + self.expect(Tok::LParen)?; + let subject = self.parse_expr()?; + self.expect(Tok::RParen)?; + self.opt_nls()?; + self.expect(Tok::LBrace)?; + let mut cases = Vec::new(); + self.loop_depth += 1; + loop { + while matches!(self.tok, Tok::Newline | Tok::Semi) { + self.advance()?; + } + let label = match self.tok.clone() { + Tok::RBrace => { + self.advance()?; + break; + } + Tok::Case => { + self.advance()?; + if matches!(self.tok, Tok::Slash | Tok::DivAssign) { + let Tok::Regex(re) = self.lex.read_regex(self.tok_pos)? else { + return self.syntax(); + }; + self.advance()?; + CaseLabel::Regex(self.regex_id(re)) + } else { + let neg = if self.tok == Tok::Minus { + self.advance()?; + true + } else { + false + }; + let v = match self.tok.clone() { + Tok::Num(n) => Expr::Num(if neg { -n } else { n }), + Tok::Str(s) if !neg => Expr::Str(s), + _ => return self.syntax(), + }; + self.advance()?; + CaseLabel::Value(v) + } + } + Tok::Default => { + self.advance()?; + CaseLabel::Default + } + _ => return self.syntax(), + }; + self.expect(Tok::Colon)?; + let mut body = Vec::new(); + loop { + while matches!(self.tok, Tok::Newline | Tok::Semi) { + self.advance()?; + } + if matches!(self.tok, Tok::Case | Tok::Default | Tok::RBrace) { + break; + } + if self.tok == Tok::Eof { + return self.syntax(); + } + body.push(self.parse_stmt()?); + } + cases.push((label, body)); } - self.pos += 1; - self.skip_whitespace(); - - // Check for `for (key in arr)` syntax - let saved_pos = self.pos; - if let Ok(AwkExpr::Variable(var_name)) = self.parse_primary() { - self.skip_whitespace(); - if self.matches_keyword("in") { - self.skip_whitespace(); - // Parse array name - let start = self.pos; - self.consume_while(Self::is_identifier_continue); - let arr_name = self.input[start..self.pos].to_string(); + self.loop_depth -= 1; + Ok(StmtKind::Switch(subject, cases)) + } - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - return Err(Error::Execution("awk: expected ')' in for-in".to_string())); + fn parse_simple_or_jump(&mut self) -> PResult { + Ok(match self.tok.clone() { + Tok::Print | Tok::Printf => self.parse_print()?, + Tok::Next => { + self.advance()?; + StmtKind::Next + } + Tok::NextFile => { + self.advance()?; + StmtKind::NextFile + } + Tok::Break => { + self.advance()?; + StmtKind::Break + } + Tok::Continue => { + self.advance()?; + StmtKind::Continue + } + Tok::Exit => { + self.advance()?; + if self.at_simple_end() { + StmtKind::Exit(None) + } else { + StmtKind::Exit(Some(self.parse_expr()?)) } - self.pos += 1; - - self.skip_whitespace(); - let body = if self.pos < self.input.len() && self.current_char().unwrap() == '{' { - self.parse_action_block()? + } + Tok::Return => { + if self.locals.is_none() { + return self.err("`return' used outside function context"); + } + self.advance()?; + if self.at_simple_end() { + StmtKind::Return(None) } else { - vec![self.parse_action()?] + StmtKind::Return(Some(self.parse_expr()?)) + } + } + Tok::Delete => { + self.advance()?; + let paren = self.tok == Tok::LParen; + if paren { + self.advance()?; + } + let Tok::Name(n) = self.tok.clone() else { + return self.syntax(); }; - - return Ok(AwkAction::ForIn(var_name, arr_name, body)); + self.advance()?; + let v = self.var(&n); + let target = if self.tok == Tok::LBracket { + let subs = self.parse_subscripts()?; + Expr::Elem(v, subs) + } else { + Expr::Var(v) + }; + if paren { + self.expect(Tok::RParen)?; + } + StmtKind::Delete(target) } - } - - // Not for-in, backtrack and parse C-style for - self.pos = saved_pos; - - // Parse init - let init_expr = self.parse_expression()?; - let init = match init_expr { - AwkExpr::Assign(name, val) => AwkAction::Assign(name, *val), - AwkExpr::ArrayAssign(name, key, val) => AwkAction::ArrayAssign(name, *key, *val), - _ => AwkAction::Expression(init_expr), - }; + _ => StmtKind::Expr(self.parse_expr()?), + }) + } - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ';' { - return Err(Error::Execution( - "awk: expected ';' in for statement".to_string(), - )); - } - self.pos += 1; - - // Parse condition - self.skip_whitespace(); - let condition = self.parse_expression()?; - - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ';' { - return Err(Error::Execution( - "awk: expected ';' in for statement".to_string(), - )); - } - self.pos += 1; - - // Parse update - self.skip_whitespace(); - let update_expr = self.parse_expression()?; - let update = match update_expr { - AwkExpr::Assign(name, val) => AwkAction::Assign(name, *val), - AwkExpr::ArrayAssign(name, key, val) => AwkAction::ArrayAssign(name, *key, *val), - _ => AwkAction::Expression(update_expr), - }; + fn at_simple_end(&self) -> bool { + matches!(self.tok, Tok::Semi | Tok::Newline | Tok::RBrace | Tok::Eof) + } - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - return Err(Error::Execution( - "awk: expected ')' in for statement".to_string(), - )); + fn parse_print(&mut self) -> PResult { + let is_printf = self.tok == Tok::Printf; + self.advance()?; + let mut args = Vec::new(); + let old = (self.in_print, self.paren_level); + self.in_print = true; + self.paren_level = 0; + if !self.at_print_end() { + // `print (a, b) > f`: a parenthesized list. + let mut done = false; + if self.tok == Tok::LParen { + let s = self.save(); + self.advance()?; + self.paren_level += 1; + let mut list = vec![self.parse_expr()?]; + while self.tok == Tok::Comma { + self.advance()?; + self.opt_nls()?; + list.push(self.parse_expr()?); + } + self.paren_level -= 1; + if self.tok == Tok::RParen { + self.advance()?; + if self.at_print_end() { + args = list; + done = true; + } + } + if !done { + self.restore(s); + self.paren_level = 0; + } + } + if !done { + args.push(self.parse_expr()?); + while self.tok == Tok::Comma { + self.advance()?; + self.opt_nls()?; + args.push(self.parse_expr()?); + } + } } - self.pos += 1; - - self.skip_whitespace(); - let body = if self.pos < self.input.len() && self.current_char().unwrap() == '{' { - self.parse_action_block()? - } else { - vec![self.parse_action()?] + let redirect = match self.tok { + Tok::Gt | Tok::Append | Tok::Pipe => { + let kind = self.tok.clone(); + self.advance()?; + // The target is a concatenation-level expression without + // comparisons (`print > "a" "b"` writes to file "ab"). + let target = self.parse_concat()?; + Some(match kind { + Tok::Gt => Redirect::File(target), + Tok::Append => Redirect::Append(target), + _ => Redirect::Pipe(target), + }) + } + Tok::PipeAmp => return self.err("coprocesses (|&) are not supported"), + _ => None, }; - - Ok(AwkAction::For( - Box::new(init), - condition, - Box::new(update), - body, - )) - } - - fn parse_while(&mut self) -> Result { - self.skip_whitespace(); - - if self.pos >= self.input.len() || self.current_char().unwrap() != '(' { - return Err(Error::Execution( - "awk: expected '(' after while".to_string(), - )); + (self.in_print, self.paren_level) = old; + if is_printf { + if args.is_empty() { + return self.syntax(); + } + Ok(StmtKind::Printf(args, redirect)) + } else { + Ok(StmtKind::Print(args, redirect)) } - self.pos += 1; - - let condition = self.parse_expression()?; + } - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - return Err(Error::Execution( - "awk: expected ')' after while condition".to_string(), - )); - } - self.pos += 1; + fn at_print_end(&self) -> bool { + matches!( + self.tok, + Tok::Semi + | Tok::Newline + | Tok::RBrace + | Tok::Eof + | Tok::Gt + | Tok::Append + | Tok::Pipe + | Tok::PipeAmp + ) + } - self.skip_whitespace(); - let body = if self.pos < self.input.len() && self.current_char().unwrap() == '{' { - self.parse_action_block()? + // ----- expressions ----- + + pub(super) fn parse_expr(&mut self) -> PResult { + let saved = self.enter()?; + let e = self.parse_ternary()?; + let e = if e.is_lvalue() { + let op = match self.tok { + Tok::Assign => Some(None), + Tok::AddAssign => Some(Some(BinOp::Add)), + Tok::SubAssign => Some(Some(BinOp::Sub)), + Tok::MulAssign => Some(Some(BinOp::Mul)), + Tok::DivAssign => Some(Some(BinOp::Div)), + Tok::ModAssign => Some(Some(BinOp::Mod)), + Tok::PowAssign => Some(Some(BinOp::Pow)), + _ => None, + }; + match op { + Some(op) => { + self.advance()?; + self.opt_nls()?; + let rhs = self.parse_expr()?; + Expr::Assign(Box::new(e), op, Box::new(rhs)) + } + None => e, + } } else { - vec![self.parse_action()?] + e }; - - Ok(AwkAction::While(condition, body)) + self.leave(saved); + Ok(e) } - fn parse_do_while(&mut self) -> Result { - self.skip_whitespace(); - - let body = if self.pos < self.input.len() && self.current_char().unwrap() == '{' { - self.parse_action_block()? - } else { - vec![self.parse_action()?] - }; + fn parse_ternary(&mut self) -> PResult { + let cond = self.parse_or()?; + if self.tok != Tok::Question { + return Ok(cond); + } + self.advance()?; + self.opt_nls()?; + let a = self.parse_expr()?; + self.opt_nls()?; + self.expect(Tok::Colon)?; + self.opt_nls()?; + let b = self.parse_expr()?; + Ok(Expr::Cond(Box::new(cond), Box::new(a), Box::new(b))) + } - self.skip_whitespace(); - // A simple body ends with a terminator before `while` (`do x--; while (x)`). - while self.pos < self.input.len() && self.current_char().unwrap() == ';' { - self.pos += 1; - self.skip_whitespace(); - } - if !self.matches_keyword("while") { - return Err(Error::Execution( - "awk: expected 'while' after do body".to_string(), - )); + fn parse_or(&mut self) -> PResult { + let mut l = self.parse_and()?; + while self.tok == Tok::Or { + self.chain()?; + self.advance()?; + self.opt_nls()?; + let r = self.parse_and()?; + l = Expr::Or(Box::new(l), Box::new(r)); } + Ok(l) + } - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != '(' { - return Err(Error::Execution( - "awk: expected '(' after do-while".to_string(), - )); + fn parse_and(&mut self) -> PResult { + let mut l = self.parse_in()?; + while self.tok == Tok::And { + self.chain()?; + self.advance()?; + self.opt_nls()?; + let r = self.parse_in()?; + l = Expr::And(Box::new(l), Box::new(r)); } - self.pos += 1; + Ok(l) + } - let condition = self.parse_expression()?; + fn parse_in(&mut self) -> PResult { + let mut l = self.parse_match()?; + while self.tok == Tok::In { + self.chain()?; + self.advance()?; + let arr = self.parse_array_name()?; + let subs = match l { + Expr::List(v) => v, + Expr::Group(g) => vec![*g], + other => vec![other], + }; + l = Expr::In(subs, Box::new(arr)); + } + Ok(l) + } - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - return Err(Error::Execution( - "awk: expected ')' in do-while".to_string(), - )); + /// Array operand of `in`: a name, or `name[i]` (a subarray). + fn parse_array_name(&mut self) -> PResult { + let Tok::Name(n) = self.tok.clone() else { + return self.syntax(); + }; + self.advance()?; + let v = self.var(&n); + if self.tok == Tok::LBracket { + let subs = self.parse_subscripts()?; + return Ok(Expr::Elem(v, subs)); } - self.pos += 1; - - Ok(AwkAction::DoWhile(condition, body)) + Ok(Expr::Var(v)) } - fn parse_delete(&mut self) -> Result { - self.skip_whitespace(); - - // Parse array name - let start = self.pos; - self.consume_while(Self::is_identifier_continue); - let arr_name = self.input[start..self.pos].to_string(); - - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == '[' { - self.pos += 1; - let index = self.parse_expression()?; - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ']' { - return Err(Error::Execution("awk: expected ']'".to_string())); - } - self.pos += 1; - Ok(AwkAction::Delete(arr_name, index)) - } else { - // delete entire array - Ok(AwkAction::Delete( - arr_name, - AwkExpr::String("*".to_string()), - )) - } - } - - /// Parse `getline [var] [< file]`. - /// - /// Forms: - /// - `getline` — read next input record into $0 - /// - `getline var` — read next input record into var - /// - `getline var < file` — read next line from file into var - /// - `getline < file` — read next line from file into $0 - fn parse_getline(&mut self) -> Result { - self.skip_whitespace(); - - // Check what follows: variable name, '<', or end of statement - let mut var: Option = None; - - if self.pos < self.input.len() { - let c = self.current_char().unwrap(); - // If next char is an identifier start (not '<', ';', '}', etc.), parse variable - if Self::is_identifier_start(c) { - let start = self.pos; - self.advance(); - self.consume_while(Self::is_identifier_continue); - var = Some(self.input[start..self.pos].to_string()); - self.skip_whitespace(); - } - } - - // Check for '< file' redirection - if self.pos < self.input.len() && self.current_char().unwrap() == '<' { - self.pos += 1; // consume '<' - self.skip_whitespace(); - let file = self.parse_primary()?; - return Ok(AwkAction::GetlineFile { var, file }); - } - - // Plain getline (with optional var — but plain `getline var` without - // file redirection just reads next input line into var; for now we - // treat that the same as plain getline which updates $0). - // TODO: `getline var` should store into var without updating $0 - Ok(AwkAction::Getline) - } - - /// Parse `getline [var] [< file]` as an expression returning 1/0/-1. - fn parse_getline_expr(&mut self) -> Result { - self.skip_whitespace(); - - let mut var: Option = None; - - if self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if Self::is_identifier_start(c) { - let start = self.pos; - self.advance(); - self.consume_while(Self::is_identifier_continue); - var = Some(self.input[start..self.pos].to_string()); - self.skip_whitespace(); - } - } - - // Check for '< file' redirection - if self.pos < self.input.len() && self.current_char().unwrap() == '<' { - self.pos += 1; // consume '<' - self.skip_whitespace(); - let file = self.parse_primary()?; - return Ok(AwkExpr::GetlineFile { - var, - file: Box::new(file), - }); + fn parse_match(&mut self) -> PResult { + let mut l = self.parse_comparison()?; + while matches!(self.tok, Tok::Tilde | Tok::NoMatch) { + self.chain()?; + let neg = self.tok == Tok::NoMatch; + self.advance()?; + let r = self.parse_comparison()?; + l = Expr::Match(neg, Box::new(l), Box::new(r)); } - - // Plain getline expression without file — use FuncCall to represent - // TODO: support plain getline as expression (advance input line) - Ok(AwkExpr::FuncCall("__getline".to_string(), vec![])) + Ok(l) } - /// THREAT[TM-DOS-027]: Track depth on every expression entry - fn parse_expression(&mut self) -> Result { - self.push_depth()?; - let result = self.parse_assignment(); - self.pop_depth(); - result + fn parse_comparison(&mut self) -> PResult { + let l = self.parse_pipe_getline()?; + let op = match self.tok { + Tok::Lt => CmpOp::Lt, + Tok::Le => CmpOp::Le, + Tok::Ne => CmpOp::Ne, + Tok::Eq => CmpOp::Eq, + Tok::Ge => CmpOp::Ge, + Tok::Gt if !(self.in_print && self.paren_level == 0) => CmpOp::Gt, + _ => return Ok(l), + }; + self.advance()?; + let r = self.parse_pipe_getline()?; + Ok(Expr::Cmp(op, Box::new(l), Box::new(r))) } - fn parse_assignment(&mut self) -> Result { - let expr = self.parse_ternary()?; - - self.skip_whitespace(); - if self.pos >= self.input.len() { - return Ok(expr); - } - - // Check for compound assignment operators (+=, -=, *=, /=, %=) - let compound_ops = ["+=", "-=", "*=", "/=", "%="]; - for op in compound_ops { - if self.input[self.pos..].starts_with(op) { - self.pos += op.len(); - self.skip_whitespace(); - let value = self.parse_assignment()?; - - match &expr { - AwkExpr::Variable(name) => { - let bin_op = &op[..1]; - let current = AwkExpr::Variable(name.clone()); - let combined = - AwkExpr::BinOp(Box::new(current), bin_op.to_string(), Box::new(value)); - return Ok(AwkExpr::Assign(name.clone(), Box::new(combined))); - } - AwkExpr::FuncCall(fname, args) - if fname == "__array_access" && args.len() == 2 => - { - if let AwkExpr::Variable(arr_name) = &args[0] { - let bin_op = &op[..1]; - return Ok(AwkExpr::CompoundArrayAssign( - arr_name.clone(), - Box::new(args[1].clone()), - bin_op.to_string(), - Box::new(value), - )); - } - return Err(Error::Execution( - "awk: invalid assignment target".to_string(), - )); - } - AwkExpr::Field(index) => { - let bin_op = &op[..1]; - let current = AwkExpr::Field(index.clone()); - let combined = - AwkExpr::BinOp(Box::new(current), bin_op.to_string(), Box::new(value)); - return Ok(AwkExpr::FieldAssign(index.clone(), Box::new(combined))); - } - _ => { - return Err(Error::Execution( - "awk: invalid assignment target".to_string(), - )); - } - } + /// `cmd | getline [var]` (left associative: `"a" | getline | getline` + /// is not meaningful, but `cmd | getline > 0` is common). + fn parse_pipe_getline(&mut self) -> PResult { + let mut l = self.parse_concat()?; + while self.tok == Tok::Pipe { + let s = self.save(); + self.advance()?; + if self.tok != Tok::Getline { + // Output pipe of a print statement. + self.restore(s); + break; } + self.chain()?; + self.advance()?; + let var = self.parse_getline_var()?; + l = Expr::Getline(GetlineSrc::Cmd(Box::new(l)), var); } - - // Simple assignment - if self.current_char().unwrap() == '=' { - let next = self.input[self.pos..].chars().nth(1); - if next != Some('=') && next != Some('~') { - self.pos += 1; - self.skip_whitespace(); - let value = self.parse_assignment()?; - - match expr { - AwkExpr::Variable(name) => { - return Ok(AwkExpr::Assign(name, Box::new(value))); - } - AwkExpr::FuncCall(ref fname, ref args) - if fname == "__array_access" && args.len() == 2 => - { - if let AwkExpr::Variable(arr_name) = &args[0] { - return Ok(AwkExpr::ArrayAssign( - arr_name.clone(), - Box::new(args[1].clone()), - Box::new(value), - )); - } - return Err(Error::Execution( - "awk: invalid assignment target".to_string(), - )); - } - AwkExpr::Field(index) => { - return Ok(AwkExpr::FieldAssign(index, Box::new(value))); - } - _ => { - return Err(Error::Execution( - "awk: invalid assignment target".to_string(), - )); - } - } - } + if self.tok == Tok::PipeAmp { + return self.err("coprocesses (|&) are not supported"); } - - Ok(expr) + Ok(l) } - fn parse_ternary(&mut self) -> Result { - let expr = self.parse_or()?; - - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == '?' { - self.pos += 1; - self.skip_whitespace(); - let then_expr = self.parse_expression()?; - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == ':' { - self.pos += 1; - self.skip_whitespace(); - let else_expr = self.parse_expression()?; - // Encode ternary as a function call for evaluation - return Ok(AwkExpr::FuncCall( - "__ternary".to_string(), - vec![expr, then_expr, else_expr], - )); - } - } - - Ok(expr) + fn parse_getline_var(&mut self) -> PResult>> { + Ok(match self.tok.clone() { + Tok::Name(_) | Tok::Dollar => Some(Box::new(self.parse_incdec_operand()?)), + _ => None, + }) } - fn parse_or(&mut self) -> Result { - let mut left = self.parse_and()?; + /// Can the current token start a concatenated operand? + fn starts_concat_operand(&self) -> bool { + matches!( + self.tok, + Tok::Num(_) + | Tok::Str(_) + | Tok::Name(_) + | Tok::FuncName(_) + | Tok::Builtin(_) + | Tok::Dollar + | Tok::Not + | Tok::LParen + | Tok::Minus + | Tok::Plus + | Tok::Incr + | Tok::Decr + | Tok::At + ) + } + /// Concatenation (also the form of a redirection target). + fn parse_concat(&mut self) -> PResult { + let mut l = self.parse_additive()?; loop { - self.skip_whitespace(); - if self.input[self.pos..].starts_with("||") { - self.pos += 2; - self.skip_whitespace(); - let right = self.parse_and()?; - left = AwkExpr::BinOp(Box::new(left), "||".to_string(), Box::new(right)); - } else { + // `-`/`+` after an operand are binary (handled in additive), so + // they never start a concatenated operand here. + let starts = self.starts_concat_operand() + && !matches!(self.tok, Tok::Minus | Tok::Plus | Tok::Not); + if !starts { break; } + self.chain()?; + let r = self.parse_additive()?; + l = Expr::Concat(Box::new(l), Box::new(r)); } - - Ok(left) + Ok(l) } - fn parse_and(&mut self) -> Result { - let mut left = self.parse_comparison()?; + fn parse_additive(&mut self) -> PResult { + let mut l = self.parse_mul()?; + loop { + let op = match self.tok { + Tok::Plus => BinOp::Add, + Tok::Minus => BinOp::Sub, + _ => break, + }; + self.chain()?; + self.advance()?; + let r = self.parse_mul()?; + l = Expr::Binary(op, Box::new(l), Box::new(r)); + } + Ok(l) + } + fn parse_mul(&mut self) -> PResult { + let mut l = self.parse_unary()?; loop { - self.skip_whitespace(); - if self.input[self.pos..].starts_with("&&") { - self.pos += 2; - self.skip_whitespace(); - let right = self.parse_comparison()?; - left = AwkExpr::BinOp(Box::new(left), "&&".to_string(), Box::new(right)); - } else { - break; + let op = match self.tok { + Tok::Star => BinOp::Mul, + Tok::Slash => BinOp::Div, + Tok::Percent => BinOp::Mod, + _ => break, + }; + let pos = self.tok_pos; + self.chain()?; + self.advance()?; + let r = self.parse_unary()?; + if matches!(op, BinOp::Div | BinOp::Mod) + && matches!(r, Expr::Num(z) if z == 0.0) + && matches!(l, Expr::Num(_)) + { + return Err(ParseError { + msg: "division by zero attempted".to_string(), + pos, + kind: ParseErrorKind::Error, + }); } + l = Expr::Binary(op, Box::new(l), Box::new(r)); } - - Ok(left) + Ok(l) } - fn parse_comparison(&mut self) -> Result { - let left = self.parse_concat()?; - - self.skip_whitespace(); - - // Check for `in` operator: (key in arr) - if self.matches_keyword("in") { - self.skip_whitespace(); - let start = self.pos; - self.consume_while(Self::is_identifier_continue); - let arr_name = self.input[start..self.pos].to_string(); - return Ok(AwkExpr::InArray(Box::new(left), arr_name)); - } - - // In print context, `>` and `>>` are output redirection, not comparison. - // `>=` remains a comparison operator even in print context. - let ops: &[&str] = if self.in_print_context { - &["==", "!=", "<=", ">=", "<", "~", "!~"] - } else { - &["==", "!=", "<=", ">=", "<", ">", "~", "!~"] - }; - - for op in ops { - if self.input[self.pos..].starts_with(op) { - self.pos += op.len(); - self.skip_whitespace(); - let right = self.parse_concat()?; - return Ok(AwkExpr::BinOp( - Box::new(left), - op.to_string(), - Box::new(right), - )); + fn parse_unary(&mut self) -> PResult { + match self.tok { + Tok::Not => { + let saved = self.enter()?; + self.advance()?; + let e = self.parse_unary()?; + self.leave(saved); + Ok(Expr::Not(Box::new(e))) } + Tok::Minus => { + let saved = self.enter()?; + self.advance()?; + let e = self.parse_unary()?; + self.leave(saved); + Ok(match e { + Expr::Num(n) => Expr::Num(-n), + e => Expr::Neg(Box::new(e)), + }) + } + Tok::Plus => { + let saved = self.enter()?; + self.advance()?; + let e = self.parse_unary()?; + self.leave(saved); + Ok(Expr::Plus(Box::new(e))) + } + _ => self.parse_pow(), } + } - Ok(left) + fn parse_pow(&mut self) -> PResult { + let base = self.parse_postfix()?; + if self.tok == Tok::Caret { + let saved = self.enter()?; + self.advance()?; + // Right associative; the exponent may carry a unary sign. + let exp = match self.tok { + Tok::Minus | Tok::Plus | Tok::Not => self.parse_unary()?, + _ => self.parse_pow()?, + }; + self.leave(saved); + return Ok(Expr::Binary(BinOp::Pow, Box::new(base), Box::new(exp))); + } + Ok(base) } - fn is_keyword_at_pos(&self) -> bool { - let remaining = &self.input[self.pos..]; - let keywords = [ - "in", "if", "else", "while", "for", "do", "break", "continue", "next", "exit", - "return", "delete", "getline", "print", "printf", "function", - ]; - for kw in keywords { - if remaining.starts_with(kw) { - let after = self.pos + kw.len(); - if after >= self.input.len() || { - let c = self.input[after..].chars().next().unwrap(); - !c.is_alphanumeric() && c != '_' - } { - return true; - } + fn parse_postfix(&mut self) -> PResult { + match self.tok { + Tok::Incr | Tok::Decr => { + let delta = if self.tok == Tok::Incr { 1.0 } else { -1.0 }; + self.advance()?; + let lv = self.parse_incdec_operand()?; + return Ok(Expr::IncDec(Box::new(lv), delta, true)); } + _ => {} + } + let e = self.parse_primary()?; + if e.is_lvalue() && matches!(self.tok, Tok::Incr | Tok::Decr) { + let delta = if self.tok == Tok::Incr { 1.0 } else { -1.0 }; + self.advance()?; + return Ok(Expr::IncDec(Box::new(e), delta, false)); } - false + Ok(e) } - fn parse_concat(&mut self) -> Result { - let mut parts = vec![self.parse_additive()?]; - - loop { - self.skip_whitespace(); - if self.pos >= self.input.len() { - break; + /// Operand of prefix `++`/`--` and of `getline var`: an lvalue. + fn parse_incdec_operand(&mut self) -> PResult { + match self.tok.clone() { + Tok::Dollar => { + self.advance()?; + let saved = self.enter()?; + let e = self.parse_field_operand()?; + self.leave(saved); + Ok(Expr::Field(Box::new(e))) } - - let c = self.current_char().unwrap(); - // Check if this could be the start of another value for concatenation - if c == '"' || c == '$' || Self::is_identifier_start(c) || c == '(' { - // But not if it's a keyword or operator - let remaining = &self.input[self.pos..]; - if !remaining.starts_with("||") - && !remaining.starts_with("&&") - && !remaining.starts_with("==") - && !remaining.starts_with("!=") - && !self.is_keyword_at_pos() - && let Ok(next) = self.parse_additive() - { - parts.push(next); - continue; + Tok::Name(n) => { + self.advance()?; + let v = self.var(&n); + if self.tok == Tok::LBracket { + let subs = self.parse_subscripts()?; + Ok(Expr::Elem(v, subs)) + } else { + Ok(Expr::Var(v)) } } - break; - } - - if parts.len() == 1 { - Ok(parts.remove(0)) - } else { - Ok(AwkExpr::Concat(parts)) + _ => self.syntax(), } } - fn parse_additive(&mut self) -> Result { - let mut left = self.parse_multiplicative()?; - - loop { - self.skip_whitespace(); - if self.pos >= self.input.len() { - break; + /// Operand of `$`: a high-precedence expression (`$NF-1` is `($NF)-1`, + /// `$i++` is `($i)++`, `$++i` increments `i`). + fn parse_field_operand(&mut self) -> PResult { + match self.tok { + Tok::Incr | Tok::Decr => { + let delta = if self.tok == Tok::Incr { 1.0 } else { -1.0 }; + self.advance()?; + let lv = self.parse_incdec_operand()?; + Ok(Expr::IncDec(Box::new(lv), delta, true)) } - - let c = self.current_char().unwrap(); - if c == '+' || c == '-' { - // Don't consume if it's a compound assignment operator (+=, -=) - let next = self.input[self.pos..].chars().nth(1); - if next == Some('=') { - break; - } - self.pos += 1; - self.skip_whitespace(); - let right = self.parse_multiplicative()?; - left = AwkExpr::BinOp(Box::new(left), c.to_string(), Box::new(right)); - } else { - break; + Tok::Minus => { + self.advance()?; + let e = self.parse_field_operand()?; + Ok(Expr::Neg(Box::new(e))) + } + Tok::Plus => { + self.advance()?; + let e = self.parse_field_operand()?; + Ok(Expr::Plus(Box::new(e))) } + Tok::Not => { + self.advance()?; + let e = self.parse_field_operand()?; + Ok(Expr::Not(Box::new(e))) + } + _ => self.parse_primary(), } - - Ok(left) } - fn parse_multiplicative(&mut self) -> Result { - let mut left = self.parse_power()?; - - loop { - self.skip_whitespace(); - if self.pos >= self.input.len() { - break; + fn parse_subscripts(&mut self) -> PResult>> { + let mut groups = Vec::new(); + while self.tok == Tok::LBracket { + self.advance()?; + let old = (self.in_print, self.paren_level); + self.in_print = false; + let mut g = vec![self.parse_expr()?]; + while self.tok == Tok::Comma { + if g.len() >= MAX_SUBSCRIPTS { + return Err(ParseError { + msg: format!("too many array subscripts (max {MAX_SUBSCRIPTS})"), + pos: self.tok_pos, + kind: ParseErrorKind::Limit, + }); + } + self.advance()?; + self.opt_nls()?; + g.push(self.parse_expr()?); + } + (self.in_print, self.paren_level) = old; + self.expect(Tok::RBracket)?; + groups.push(g); + if groups.len() > MAX_SUBSCRIPTS { + return Err(ParseError { + msg: format!("too many array subscripts (max {MAX_SUBSCRIPTS})"), + pos: self.tok_pos, + kind: ParseErrorKind::Limit, + }); } + } + Ok(groups) + } - let c = self.current_char().unwrap(); - if c == '*' || c == '/' || c == '%' { - // Don't consume ** (power operator) - if c == '*' && self.input[self.pos..].chars().nth(1) == Some('*') { - break; - } - // Don't consume if it's a compound assignment operator (*=, /=, %=) - let next = self.input[self.pos..].chars().nth(1); - if next == Some('=') { + fn parse_args(&mut self) -> PResult> { + // Current token is `(`. + self.advance()?; + let old = (self.in_print, self.paren_level); + self.in_print = false; + self.opt_nls()?; + let mut args = Vec::new(); + if self.tok != Tok::RParen { + loop { + args.push(self.parse_expr()?); + self.opt_nls()?; + if self.tok == Tok::Comma { + self.advance()?; + self.opt_nls()?; + } else { break; } - self.pos += 1; - self.skip_whitespace(); - let right = self.parse_power()?; - left = AwkExpr::BinOp(Box::new(left), c.to_string(), Box::new(right)); - } else { - break; } } - - Ok(left) + (self.in_print, self.paren_level) = old; + self.expect(Tok::RParen)?; + Ok(args) } - fn parse_power(&mut self) -> Result { - let base = self.parse_unary()?; - - self.skip_whitespace(); - if self.pos >= self.input.len() { - return Ok(base); - } - - // Check for ^ or ** - if self.current_char().unwrap() == '^' { - self.pos += 1; - self.skip_whitespace(); - let exp = self.parse_unary()?; - return Ok(AwkExpr::BinOp( - Box::new(base), - "^".to_string(), - Box::new(exp), - )); - } - if self.input[self.pos..].starts_with("**") { - self.pos += 2; - self.skip_whitespace(); - let exp = self.parse_unary()?; - return Ok(AwkExpr::BinOp( - Box::new(base), - "^".to_string(), - Box::new(exp), - )); - } - - Ok(base) - } - - /// THREAT[TM-DOS-027]: Track depth on unary self-recursion - fn parse_unary(&mut self) -> Result { - self.skip_whitespace(); - - if self.pos >= self.input.len() { - return Err(Error::Execution( - "awk: unexpected end of expression".to_string(), - )); - } - - // Pre-increment: ++var or ++arr[key] - if self.input[self.pos..].starts_with("++") { - self.pos += 2; - self.skip_whitespace(); - match self.parse_primary()? { - AwkExpr::Variable(name) => return Ok(AwkExpr::PreIncrement(name)), - AwkExpr::FuncCall(ref fname, ref args) - if fname == "__array_access" && args.len() == 2 => - { - if let AwkExpr::Variable(arr_name) = &args[0] { - return Ok(AwkExpr::CompoundArrayAssign( - arr_name.clone(), - Box::new(args[1].clone()), - "+".to_string(), - Box::new(AwkExpr::Number(1.0)), - )); + fn parse_primary(&mut self) -> PResult { + let pos = self.tok_pos; + match self.tok.clone() { + Tok::Num(n) => { + self.advance()?; + Ok(Expr::Num(n)) + } + Tok::Str(s) => { + self.advance()?; + Ok(Expr::Str(s)) + } + Tok::Slash | Tok::DivAssign => { + let Tok::Regex(re) = self.lex.read_regex(self.tok_pos)? else { + return self.syntax(); + }; + self.advance()?; + Ok(Expr::Regex(self.regex_id(re))) + } + Tok::Dollar => { + self.advance()?; + let saved = self.enter()?; + let e = self.parse_field_operand()?; + self.leave(saved); + Ok(Expr::Field(Box::new(e))) + } + Tok::LParen => { + let saved = self.enter()?; + self.advance()?; + self.paren_level += 1; + let old_print = self.in_print; + self.in_print = false; + let first = self.parse_expr()?; + let mut list = Vec::new(); + if self.tok == Tok::Comma { + list.push(first); + while self.tok == Tok::Comma { + if list.len() >= MAX_SUBSCRIPTS { + return Err(ParseError { + msg: format!("too many array subscripts (max {MAX_SUBSCRIPTS})"), + pos: self.tok_pos, + kind: ParseErrorKind::Limit, + }); + } + self.advance()?; + self.opt_nls()?; + list.push(self.parse_expr()?); + } + self.in_print = old_print; + self.paren_level -= 1; + self.expect(Tok::RParen)?; + self.leave(saved); + if self.tok != Tok::In { + return self.syntax(); } - return Err(Error::Execution( - "awk: expected variable after ++".to_string(), - )); + return Ok(Expr::List(list)); } - _ => { - return Err(Error::Execution( - "awk: expected variable after ++".to_string(), - )); + self.in_print = old_print; + self.paren_level -= 1; + self.expect(Tok::RParen)?; + self.leave(saved); + Ok(Expr::Group(Box::new(first))) + } + Tok::Getline => { + self.advance()?; + let var = self.parse_getline_var()?; + if self.tok == Tok::Lt { + self.advance()?; + // The file operand: a primary (`getline < "a" "b"` + // reads file "a"). + let saved = self.enter()?; + let f = self.parse_postfix_no_concat()?; + self.leave(saved); + return Ok(Expr::Getline(GetlineSrc::File(Box::new(f)), var)); } + Ok(Expr::Getline(GetlineSrc::Main, var)) } - } - - // Pre-decrement: --var or --arr[key] - if self.input[self.pos..].starts_with("--") { - self.pos += 2; - self.skip_whitespace(); - match self.parse_primary()? { - AwkExpr::Variable(name) => return Ok(AwkExpr::PreDecrement(name)), - AwkExpr::FuncCall(ref fname, ref args) - if fname == "__array_access" && args.len() == 2 => - { - if let AwkExpr::Variable(arr_name) = &args[0] { - return Ok(AwkExpr::CompoundArrayAssign( - arr_name.clone(), - Box::new(args[1].clone()), - "-".to_string(), - Box::new(AwkExpr::Number(1.0)), - )); - } - return Err(Error::Execution( - "awk: expected variable after --".to_string(), - )); + Tok::Name(n) => { + self.advance()?; + let v = self.var(&n); + if self.tok == Tok::LBracket { + let subs = self.parse_subscripts()?; + return Ok(Expr::Elem(v, subs)); } - _ => { - return Err(Error::Execution( - "awk: expected variable after --".to_string(), - )); + Ok(Expr::Var(v)) + } + Tok::FuncName(n) => { + self.advance()?; + let id = self.func_id(&n); + let args = self.parse_args()?; + if !self.prog.funcs[id as usize].defined { + self.pending_calls.push((id, pos)); } + Ok(Expr::Call(id, args)) } + Tok::At => { + self.advance()?; + let name = match self.tok.clone() { + Tok::FuncName(n) | Tok::Name(n) => n, + _ => return self.syntax(), + }; + self.advance()?; + if self.tok != Tok::LParen { + return self.syntax(); + } + let v = Expr::Var(self.var(&name)); + let args = self.parse_args()?; + Ok(Expr::Indirect(Box::new(v), args)) + } + Tok::Builtin(name) => { + self.advance()?; + let Some(bi) = Bi::from_name(&name) else { + return self.syntax(); + }; + let args = if self.tok == Tok::LParen { + self.parse_args()? + } else if bi == Bi::Length { + Vec::new() + } else { + return self.syntax(); + }; + self.check_builtin_arity(bi, args.len(), pos)?; + Ok(Expr::Builtin(bi, args)) + } + Tok::Minus | Tok::Plus | Tok::Not => self.parse_unary(), + Tok::Incr | Tok::Decr => self.parse_postfix(), + _ => self.syntax(), } + } - let c = self.current_char().unwrap(); + /// File operand of `getline <`: `$x`, a name, a literal or a group. + fn parse_postfix_no_concat(&mut self) -> PResult { + self.parse_primary() + } - if c == '-' { - self.pos += 1; - self.push_depth()?; - let expr = self.parse_unary(); - self.pop_depth(); - return Ok(AwkExpr::UnaryOp("-".to_string(), Box::new(expr?))); + fn check_builtin_arity(&self, bi: Bi, n: usize, pos: usize) -> PResult<()> { + let (min, max) = match bi { + Bi::Length => (0, 1), + Bi::Substr => (2, 3), + Bi::Index => (2, 2), + Bi::Split => (2, 4), + Bi::Patsplit => (2, 4), + Bi::Sub | Bi::Gsub => (2, 3), + Bi::Gensub => (3, 4), + Bi::Match => (2, 3), + Bi::Sprintf => (1, usize::MAX), + Bi::Sin | Bi::Cos | Bi::Exp | Bi::Log | Bi::Sqrt | Bi::Int => (1, 1), + Bi::Atan2 => (2, 2), + Bi::Rand | Bi::Systime => (0, 0), + Bi::Srand => (0, 1), + Bi::Tolower | Bi::Toupper | Bi::System | Bi::Strtonum | Bi::Typeof | Bi::Isarray => { + (1, 1) + } + Bi::Close => (1, 2), + Bi::Fflush => (0, 1), + Bi::Strftime => (0, 3), + Bi::Mktime => (1, 2), + Bi::Asort | Bi::Asorti => (1, 3), + Bi::And | Bi::Or | Bi::Xor => (2, usize::MAX), + Bi::Lshift | Bi::Rshift => (2, 2), + Bi::Compl => (1, 1), + }; + if n < min || n > max { + return Err(ParseError { + msg: format!( + "{n} is invalid as number of arguments for {}", + super::lexer::BUILTIN_FUNCS + .iter() + .find(|name| Bi::from_name(name) == Some(bi)) + .unwrap_or(&"function") + ), + pos, + kind: ParseErrorKind::Syntax, + }); } + Ok(()) + } +} - if c == '!' { - self.pos += 1; - self.push_depth()?; - let expr = self.parse_unary(); - self.pop_depth(); - return Ok(AwkExpr::UnaryOp("!".to_string(), Box::new(expr?))); +/// Mark function parameters used as arrays (fixed point over calls). +fn analyze_array_params(prog: &mut Program) { + loop { + let mut changed = false; + for fi in 0..prog.funcs.len() { + let mut marks = prog.funcs[fi].array_params.clone(); + let body = std::mem::take(&mut prog.funcs[fi].body); + for s in &body { + walk_stmt(s, &mut |e| mark_expr(e, &mut marks, &prog.funcs)); + mark_stmt(s, &mut marks); + } + prog.funcs[fi].body = body; + if marks != prog.funcs[fi].array_params { + prog.funcs[fi].array_params = marks; + changed = true; + } } - - if c == '+' { - self.pos += 1; - self.push_depth()?; - let result = self.parse_unary(); - self.pop_depth(); - return result; + if !changed { + break; } + } +} - self.parse_postfix() +fn local_of(e: &Expr) -> Option { + match e { + Expr::Var(VarRef::Local(i)) => Some(*i as usize), + _ => None, } +} - fn parse_postfix(&mut self) -> Result { - let expr = self.parse_primary()?; +fn set_mark(marks: &mut [bool], e: &Expr) { + if let Some(i) = local_of(e) + && let Some(m) = marks.get_mut(i) + { + *m = true; + } +} - // Check for postfix ++ / -- - if self.pos + 1 < self.input.len() { - if self.input[self.pos..].starts_with("++") { - match &expr { - AwkExpr::Variable(name) => { - self.pos += 2; - return Ok(AwkExpr::PostIncrement(name.clone())); - } - AwkExpr::FuncCall(fname, args) - if fname == "__array_access" && args.len() == 2 => - { - // arr[key]++ → compound array assign with +1 - if let AwkExpr::Variable(arr_name) = &args[0] { - self.pos += 2; - return Ok(AwkExpr::CompoundArrayAssign( - arr_name.clone(), - Box::new(args[1].clone()), - "+".to_string(), - Box::new(AwkExpr::Number(1.0)), - )); - } - } - _ => {} +fn mark_expr(e: &Expr, marks: &mut [bool], funcs: &[Func]) { + match e { + Expr::Elem(VarRef::Local(i), _) => { + if let Some(m) = marks.get_mut(*i as usize) { + *m = true; + } + } + Expr::In(_, arr) => set_mark(marks, arr), + Expr::Builtin(bi, args) => { + for &p in bi.array_args() { + if let Some(a) = args.get(p) { + set_mark(marks, a); } } - if self.input[self.pos..].starts_with("--") { - match &expr { - AwkExpr::Variable(name) => { - self.pos += 2; - return Ok(AwkExpr::PostDecrement(name.clone())); - } - AwkExpr::FuncCall(fname, args) - if fname == "__array_access" && args.len() == 2 => - { - if let AwkExpr::Variable(arr_name) = &args[0] { - self.pos += 2; - return Ok(AwkExpr::CompoundArrayAssign( - arr_name.clone(), - Box::new(args[1].clone()), - "-".to_string(), - Box::new(AwkExpr::Number(1.0)), - )); - } + } + Expr::Call(f, args) => { + if let Some(func) = funcs.get(*f as usize) { + for (j, a) in args.iter().enumerate() { + if func.array_params.get(j).copied().unwrap_or(false) { + set_mark(marks, a); } - _ => {} } } } - - Ok(expr) + _ => {} } +} - fn parse_primary(&mut self) -> Result { - self.skip_whitespace(); - - if self.pos >= self.input.len() { - return Err(Error::Execution( - "awk: unexpected end of expression".to_string(), - )); - } - - let c = self.current_char().unwrap(); - - // Field reference $ - if c == '$' { - self.pos += 1; - self.push_depth()?; - let index = self.parse_primary()?; - self.pop_depth(); - return Ok(AwkExpr::Field(Box::new(index))); +fn mark_stmt(s: &Stmt, marks: &mut Vec) { + match &s.kind { + StmtKind::ForIn(_, arr, _) => set_mark(marks, arr), + StmtKind::Delete(target) => set_mark(marks, target), + _ => {} + } + for b in child_blocks(s) { + for s in b { + mark_stmt(s, marks); } + } +} - // Number - if c.is_ascii_digit() || c == '.' { - return self.parse_number(); +fn child_blocks(s: &Stmt) -> Vec<&Block> { + match &s.kind { + StmtKind::If(_, a, b) => { + let mut v = vec![a]; + if let Some(b) = b { + v.push(b); + } + v } + StmtKind::While(_, b) | StmtKind::Do(b, _) | StmtKind::ForIn(_, _, b) => vec![b], + StmtKind::Block(b) => vec![b], + StmtKind::For(_, _, _, b) => vec![b], + StmtKind::Switch(_, cases) => cases.iter().map(|(_, b)| b).collect(), + _ => Vec::new(), + } +} - // String - if c == '"' { - let s = self.parse_string()?; - return Ok(AwkExpr::String(s)); +/// Visit every expression in a statement tree. +pub(super) fn walk_stmt(s: &Stmt, f: &mut dyn FnMut(&Expr)) { + match &s.kind { + StmtKind::Expr(e) => walk_expr(e, f), + StmtKind::Print(args, r) | StmtKind::Printf(args, r) => { + for a in args { + walk_expr(a, f); + } + if let Some(Redirect::File(e) | Redirect::Append(e) | Redirect::Pipe(e)) = r { + walk_expr(e, f); + } } - - // Regex literal /pattern/ - if c == '/' { - self.pos += 1; - let start = self.pos; - while self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if c == '/' { - let pattern = &self.input[start..self.pos]; - self.pos += 1; - return Ok(AwkExpr::Regex(pattern.to_string())); - } else if c == '\\' { - self.pos += 1; // skip '\\' (ASCII) - self.advance(); // skip next char (may be multi-byte) - } else { - self.advance(); // regex content may be multi-byte - } + StmtKind::If(c, a, b) => { + walk_expr(c, f); + for s in a { + walk_stmt(s, f); } - return Err(Error::Execution("awk: unterminated regex".to_string())); - } - - // Parenthesized expression: `>` inside parens is comparison, not redirection - if c == '(' { - self.pos += 1; - self.push_depth()?; - let saved_print_ctx = self.in_print_context; - self.in_print_context = false; - let expr = self.parse_expression()?; - self.in_print_context = saved_print_ctx; - self.pop_depth(); - self.skip_whitespace(); - if self.pos >= self.input.len() || self.current_char().unwrap() != ')' { - return Err(Error::Execution("awk: expected ')'".to_string())); - } - self.pos += 1; - return Ok(expr); - } - - // Variable or function call - if Self::is_identifier_start(c) { - let start = self.pos; - self.advance(); - self.consume_while(Self::is_identifier_continue); - let name = self.input[start..self.pos].to_string(); - - // getline [var] [< file] as expression (returns 1/0/-1) - if name == "getline" { - return self.parse_getline_expr(); - } - - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == '(' { - // Function call - self.pos += 1; - let mut args = Vec::new(); - loop { - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == ')' { - self.pos += 1; - break; - } - let arg = self.parse_expression()?; - args.push(arg); - self.skip_whitespace(); - if self.pos < self.input.len() && self.current_char().unwrap() == ',' { - self.pos += 1; - } + if let Some(b) = b { + for s in b { + walk_stmt(s, f); } - return Ok(AwkExpr::FuncCall(name, args)); - } - - // Array indexing: arr[index] or arr[e1,e2,...] (multi-subscript with SUBSEP) - if self.pos < self.input.len() && self.current_char().unwrap() == '[' { - self.pos += 1; // consume '[' - let mut subscripts = vec![self.parse_expression()?]; - self.skip_whitespace(); - // THREAT[TM-DOS-027]: SUBSEP_CONCAT still evaluates recursively, so cap - // attacker-controlled comma lists before folding them into a left-deep AST. - while self.pos < self.input.len() && self.current_char().unwrap() == ',' { - if subscripts.len() >= MAX_AWK_MULTI_SUBSCRIPTS { - return Err(Error::Execution(format!( - "awk: too many array subscripts (max {})", - MAX_AWK_MULTI_SUBSCRIPTS - ))); - } - self.pos += 1; // consume ',' - self.skip_whitespace(); - subscripts.push(self.parse_expression()?); - self.skip_whitespace(); - } - if self.pos >= self.input.len() || self.current_char().unwrap() != ']' { - return Err(Error::Execution("awk: expected ']'".to_string())); + } + } + StmtKind::While(c, b) | StmtKind::Do(b, c) => { + walk_expr(c, f); + for s in b { + walk_stmt(s, f); + } + } + StmtKind::For(i, c, n, b) => { + if let Some(i) = i { + walk_stmt(i, f); + } + if let Some(c) = c { + walk_expr(c, f); + } + if let Some(n) = n { + walk_stmt(n, f); + } + for s in b { + walk_stmt(s, f); + } + } + StmtKind::ForIn(k, a, b) => { + walk_expr(k, f); + walk_expr(a, f); + for s in b { + walk_stmt(s, f); + } + } + StmtKind::Block(b) => { + for s in b { + walk_stmt(s, f); + } + } + StmtKind::Exit(Some(e)) | StmtKind::Return(Some(e)) | StmtKind::Delete(e) => { + walk_expr(e, f) + } + StmtKind::Switch(e, cases) => { + walk_expr(e, f); + for (_, b) in cases { + for s in b { + walk_stmt(s, f); } - self.pos += 1; // consume ']' - let index_expr = if subscripts.len() == 1 { - subscripts.remove(0) - } else { - // Join multiple subscripts with SUBSEP - let mut result = subscripts.remove(0); - for sub in subscripts { - result = AwkExpr::BinOp( - Box::new(result), - "SUBSEP_CONCAT".to_string(), - Box::new(sub), - ); - } - result - }; - return Ok(AwkExpr::FuncCall( - "__array_access".to_string(), - vec![AwkExpr::Variable(name), index_expr], - )); } - - return Ok(AwkExpr::Variable(name)); } - - Err(Error::Execution(format!( - "awk: unexpected character: {}", - c - ))) + _ => {} } +} - fn parse_number(&mut self) -> Result { - let start = self.pos; - // Integer part. - while self.pos < self.input.len() && self.current_char().unwrap().is_ascii_digit() { - self.pos += 1; +pub(super) fn walk_expr(e: &Expr, f: &mut dyn FnMut(&Expr)) { + f(e); + match e { + Expr::Elem(_, groups) => { + for g in groups { + for x in g { + walk_expr(x, f); + } + } } - // Fractional part: digits with an optional '.' (leading ".5" and - // trailing "1." both valid; lone '.' rejected below by f64 parse). - if self.pos < self.input.len() && self.current_char().unwrap() == '.' { - self.pos += 1; - while self.pos < self.input.len() && self.current_char().unwrap().is_ascii_digit() { - self.pos += 1; + Expr::Field(x) | Expr::Neg(x) | Expr::Plus(x) | Expr::Not(x) | Expr::Group(x) => { + walk_expr(x, f) + } + Expr::IncDec(x, _, _) => walk_expr(x, f), + Expr::Assign(a, _, b) + | Expr::Binary(_, a, b) + | Expr::Cmp(_, a, b) + | Expr::Match(_, a, b) + | Expr::And(a, b) + | Expr::Or(a, b) + | Expr::Concat(a, b) => { + walk_expr(a, f); + walk_expr(b, f); + } + Expr::Cond(a, b, c) => { + walk_expr(a, f); + walk_expr(b, f); + walk_expr(c, f); + } + Expr::In(subs, arr) => { + for s in subs { + walk_expr(s, f); } + walk_expr(arr, f); } - // Exponent: e/E followed by optional sign and at least one digit. - // Only consumed when the full exponent is present, so a binary - // +/- after the number stays a separate operator (GH-2389). - if self.pos < self.input.len() && matches!(self.current_char().unwrap(), 'e' | 'E') { - let mut end = self.pos + 1; - if end < self.input.len() - && (self.input.as_bytes()[end] == b'+' || self.input.as_bytes()[end] == b'-') - { - end += 1; + Expr::Call(_, args) | Expr::Builtin(_, args) | Expr::List(args) => { + for a in args { + walk_expr(a, f); } - if end < self.input.len() && self.input.as_bytes()[end].is_ascii_digit() { - while end < self.input.len() && self.input.as_bytes()[end].is_ascii_digit() { - end += 1; - } - self.pos = end; + } + Expr::Indirect(v, args) => { + walk_expr(v, f); + for a in args { + walk_expr(a, f); + } + } + Expr::Getline(src, var) => { + if let GetlineSrc::File(x) | GetlineSrc::Cmd(x) = src { + walk_expr(x, f); + } + if let Some(v) = var { + walk_expr(v, f); } } + _ => {} + } +} - let num_str = &self.input[start..self.pos]; - let num: f64 = num_str - .parse() - .map_err(|_| Error::Execution(format!("awk: invalid number: {}", num_str)))?; +#[cfg(test)] +mod tests { + use super::*; - Ok(AwkExpr::Number(num)) + fn parse(src: &str) -> PResult { + let sources = vec![Source { + name: "cmd. line".to_string(), + start: 0, + }]; + Parser::new(src, &sources).parse_program() } - fn parse_string(&mut self) -> Result { - if self.pos >= self.input.len() || self.current_char().unwrap() != '"' { - return Err(Error::Execution("awk: expected string".to_string())); - } - self.pos += 1; // skip opening '"' (ASCII) - - let mut result = String::new(); - while self.pos < self.input.len() { - let c = self.current_char().unwrap(); - if c == '"' { - self.pos += 1; // skip closing '"' (ASCII) - return Ok(result); - } else if c == '\\' { - self.pos += 1; // skip '\\' (ASCII) - if self.pos < self.input.len() { - let escaped = self.current_char().unwrap(); - match escaped { - 'n' => { - result.push('\n'); - self.advance(); - } - 't' => { - result.push('\t'); - self.advance(); - } - 'r' => { - result.push('\r'); - self.advance(); - } - '\\' => { - result.push('\\'); - self.advance(); - } - '"' => { - result.push('"'); - self.advance(); - } - 'u' => { - // gawk 5.3+ Unicode escape: \u followed by 1-8 hex digits - self.advance(); // skip 'u' - let mut hex = String::new(); - while hex.len() < 8 - && self.pos < self.input.len() - && self.current_char().is_some_and(|c| c.is_ascii_hexdigit()) - { - hex.push(self.current_char().unwrap()); - self.pos += 1; // hex digits are ASCII - } - if hex.is_empty() { - result.push('\\'); - result.push('u'); - } else if let Ok(cp) = u32::from_str_radix(&hex, 16) { - if let Some(ch) = char::from_u32(cp) { - result.push(ch); - } else { - // Invalid Unicode code point — keep literal - result.push('\\'); - result.push('u'); - result.push_str(&hex); - } - } else { - result.push('\\'); - result.push('u'); - result.push_str(&hex); - } - } - _ => { - result.push('\\'); - result.push(escaped); - self.advance(); - } - } - } - } else { - result.push(c); - self.advance(); // character may be multi-byte + #[test] + fn parses_common_programs() { + for src in [ + "{print $1}", + "BEGIN { x = 1; y = x ^= 2 } END { print x, y }", + "!seen[$0]++", + "NR==1, /end/ { print }", + "function f(a, b, c) { c = a + b; return c } { print f($1, $2) }", + "{ if ($1 > 2) print \"big\"; else print \"small\" }", + "{ for (k in a) print k, a[k] }", + "{ for (i = 1; i <= NF; i++) s += $i } END { print s }", + "BEGIN { while ((\"echo hi\" | getline line) > 0) print line }", + "BEGIN { print \"x\" > \"/dev/stderr\"; print 1, 2 > \"f\" }", + "BEGIN { printf(\"%d\\n\", 3) > \"f\" }", + "{ print (1,2) in a }", + "BEGIN { if ((1,2) in a) print \"y\" }", + "BEGIN { a[1][2] = 3; print length(a[1]) }", + "BEGIN { switch (x) { case 1: print 1; break; case /a/: print 2; default: print 3 } }", + "BEGIN { getline line < \"file\"; print line }", + "{ $3 = \"\"; print }", + "BEGIN { x = -2^2; print x }", + "BEGIN { print 1 \" \" 2 }", + "/a/ && /b/", + "BEGIN { f = \"g\"; print @f(1) } function g(x) { return x }", + "BEGIN { n = split(\"a b\", arr); print n }", + "BEGIN { print length }", + "{ print $NF-1, $(NF-1) }", + "BEGIN {\n if (x)\n print 1\n else\n print 2\n}", + "BEGIN { do x++; while (x < 3); print x }", + ] { + if let Err(e) = parse(src) { + panic!("{src}: {} at {}", e.msg, e.pos); } } + } + + #[test] + fn syntax_errors() { + let e = parse("{print $1").unwrap_err(); + assert_eq!(e.msg, "unexpected newline or end of string"); + let e = parse("BEGIN { x = = 1 }").unwrap_err(); + assert_eq!(e.msg, "syntax error"); + assert_eq!(e.pos, 12); + let e = parse("BEGIN { nosuch(1) }").unwrap_err(); + assert_eq!(e.kind, ParseErrorKind::Fatal); + let e = parse("BEGIN { print 1/0 }").unwrap_err(); + assert_eq!(e.kind, ParseErrorKind::Error); + } + + #[test] + fn depth_limits() { + let deep = format!("{{print {}1{}}}", "(".repeat(200), ")".repeat(200)); + assert_eq!(parse(&deep).unwrap_err().kind, ParseErrorKind::Limit); + let unary = format!("{{print {}1}}", "- ".repeat(200)); + assert_eq!(parse(&unary).unwrap_err().kind, ParseErrorKind::Limit); + let chain = format!("{{print 1{}}}", "+1".repeat(5000)); + assert_eq!(parse(&chain).unwrap_err().kind, ParseErrorKind::Limit); + let ok = format!("{{print 1{}}}", "+1".repeat(200)); + assert!(parse(&ok).is_ok()); + } - Err(Error::Execution("awk: unterminated string".to_string())) + #[test] + fn array_params_are_detected() { + let p = parse( + "function f(a, n) { a[1] = n } function g(x, y) { f(x, 1); return y } BEGIN { g(z) }", + ) + .unwrap(); + assert_eq!(p.funcs[0].array_params, vec![true, false]); + assert_eq!(p.funcs[1].array_params, vec![true, false]); } } diff --git a/crates/bashkit/src/builtins/awk/regex.rs b/crates/bashkit/src/builtins/awk/regex.rs new file mode 100644 index 000000000..210d129b9 --- /dev/null +++ b/crates/bashkit/src/builtins/awk/regex.rs @@ -0,0 +1,584 @@ +//! POSIX ERE (gawk dialect) on top of the `regex` crate, with POSIX +//! leftmost-longest match semantics. +//! +//! Decisions: +//! - Patterns are translated, not passed through: `.` matches newline, +//! `\y`/`\<`/`\>` are word boundaries, `{` that does not start a valid +//! interval is literal, a leading `*`/`+`/`?` is literal, bracket +//! expressions follow POSIX (`]` first is literal, backslash escapes, +//! `[:class:]`), unknown escapes like `\d` are the literal letter (gawk). +//! - Leftmost-longest: the `regex` crate is leftmost-first. Both agree on +//! where the leftmost match starts; from that start an anchored lazy DFA +//! with `MatchKind::All` finds the longest end. Patterns whose +//! leftmost-first and leftmost-longest results cannot differ (no +//! alternation, no quantifier followed by more pattern) skip the DFA. +//! - THREAT[TM-DOS-023]: both engines are built with the shared regex size +//! limits; compiled regexes (and failures) are cached per run with a +//! bounded entry count and pattern bytes. + +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex, OnceLock}; + +use regex::Regex; +use regex_automata::hybrid::dfa::{Cache as DfaCache, DFA}; +use regex_automata::nfa::thompson; +use regex_automata::{Anchored, Input, MatchKind}; + +use crate::builtins::limits::RUNTIME_REGEX_CACHE_ENTRIES; +use crate::builtins::search_common::{REGEX_DFA_SIZE_LIMIT, REGEX_SIZE_LIMIT}; + +pub(crate) struct AwkRegex { + re: Regex, + longest: Option<(DFA, Mutex)>, + /// Translated pattern, for the whole-span capture fallback. + rust: String, + full: OnceLock>, +} + +impl std::fmt::Debug for AwkRegex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.re.as_str()) + } +} + +/// Compile failure, as a short user-facing message. +pub(crate) struct RegexError(pub(crate) String); + +impl AwkRegex { + pub(crate) fn new(ere: &str, icase: bool) -> Result { + let t = translate(ere); + let mut pat = String::with_capacity(t.pattern.len() + 8); + pat.push_str(if icase { "(?si)" } else { "(?s)" }); + pat.push_str(&t.pattern); + let re = regex::RegexBuilder::new(&pat) + .size_limit(REGEX_SIZE_LIMIT) + .dfa_size_limit(REGEX_DFA_SIZE_LIMIT) + .build() + .map_err(|e| RegexError(short_error(&e)))?; + let longest = if t.needs_longest { + DFA::builder() + .configure( + DFA::config() + .match_kind(MatchKind::All) + .cache_capacity(REGEX_DFA_SIZE_LIMIT) + .unicode_word_boundary(true), + ) + .thompson(thompson::Config::new().nfa_size_limit(Some(REGEX_SIZE_LIMIT))) + .build(&pat) + .ok() + .map(|dfa| { + let cache = dfa.create_cache(); + (dfa, Mutex::new(cache)) + }) + } else { + None + }; + Ok(AwkRegex { + re, + longest, + rust: pat, + full: OnceLock::new(), + }) + } + + pub(crate) fn is_match(&self, s: &str) -> bool { + self.re.is_match(s) + } + + /// Leftmost-longest match at or after byte `start`. + pub(crate) fn find_at(&self, s: &str, start: usize) -> Option<(usize, usize)> { + let m = self.re.find_at(s, start)?; + Some((m.start(), self.longest_end(s, m.start(), m.end()))) + } + + fn longest_end(&self, s: &str, start: usize, end: usize) -> usize { + let Some((dfa, cache)) = &self.longest else { + return end; + }; + if end == s.len() { + return end; + } + let Ok(mut cache) = cache.lock() else { + return end; + }; + let input = Input::new(s).range(start..).anchored(Anchored::Yes); + match dfa.try_search_fwd(&mut cache, &input) { + Ok(Some(hm)) if hm.offset() > end && s.is_char_boundary(hm.offset()) => hm.offset(), + _ => end, + } + } + + /// Leftmost-longest match with capture group spans (group 0 first). + pub(crate) fn captures_at(&self, s: &str, start: usize) -> Option>> { + let caps = self.re.captures_at(s, start)?; + let m0 = caps.get(0)?; + let end = self.longest_end(s, m0.start(), m0.end()); + if end == m0.end() { + return Some( + caps.iter() + .map(|g| g.map(|m| (m.start(), m.end()))) + .collect(), + ); + } + // The longest match is not the leftmost-first one: take groups from + // a whole-span match of the same pattern on that span. + let begin = m0.start(); + let full = self.full.get_or_init(|| { + regex::RegexBuilder::new(&format!("^(?:{})$", self.rust)) + .size_limit(REGEX_SIZE_LIMIT) + .dfa_size_limit(REGEX_DFA_SIZE_LIMIT) + .build() + .ok() + }); + let sub = &s[begin..end]; + if let Some(full) = full + && let Some(c) = full.captures(sub) + { + return Some( + c.iter() + .map(|g| g.map(|m| (begin + m.start(), begin + m.end()))) + .collect(), + ); + } + let mut v: Vec> = vec![None; caps.len()]; + v[0] = Some((begin, end)); + Some(v) + } +} + +fn short_error(e: ®ex::Error) -> String { + match e { + regex::Error::CompiledTooBig(_) => "regular expression too big".to_string(), + _ => { + let s = e.to_string(); + // The last line of the regex crate's message is the summary. + let line = s + .lines() + .rev() + .find(|l| !l.trim().is_empty()) + .unwrap_or("invalid regexp"); + let line = line.trim().trim_start_matches("error: "); + line.chars().take(200).collect() + } + } +} + +struct Translated { + pattern: String, + needs_longest: bool, +} + +fn regex_syntax_is_meta(c: char) -> bool { + matches!( + c, + '\\' | '.' + | '+' + | '*' + | '?' + | '(' + | ')' + | '|' + | '[' + | ']' + | '{' + | '}' + | '^' + | '$' + | '#' + | '&' + | '-' + | '~' + ) +} + +/// Translate a POSIX/gawk ERE to `regex` crate syntax. +fn translate(ere: &str) -> Translated { + let chars: Vec = ere.chars().collect(); + let mut out = String::with_capacity(ere.len() + 8); + let mut i = 0; + // A quantifier here would have nothing to repeat (start, after `(`, + // `|`, `^`): POSIX/gawk treat it as literal. + let mut at_start = true; + let mut has_alt = false; + let mut quantified = false; + let mut needs_longest = false; + let mut depth = 0usize; + // Escape only regex metacharacters: `\<`, `\>` and friends are + // assertions in the regex crate, not literals. + let push_lit = |out: &mut String, c: char| { + if regex_syntax_is_meta(c) { + out.push('\\'); + } + out.push(c); + }; + while i < chars.len() { + let c = chars[i]; + // An atom after a quantifier means the match length can trade off. + let is_quant = matches!(c, '*' | '+' | '?' | '{'); + if quantified && !is_quant && c != ')' && c != '|' && c != '$' { + needs_longest = true; + } + match c { + '\\' => { + i += 1; + let Some(&e) = chars.get(i) else { + out.push_str("\\\\"); + break; + }; + match e { + 'y' => out.push_str("\\b"), + 'B' => out.push_str("\\B"), + '<' => out.push_str("\\b{start}"), + '>' => out.push_str("\\b{end}"), + '`' => out.push_str("\\A"), + '\'' => out.push_str("\\z"), + 's' | 'S' | 'w' | 'W' => { + out.push('\\'); + out.push(e); + } + 'n' => out.push_str("\\n"), + 't' => out.push_str("\\t"), + 'r' => out.push_str("\\r"), + 'f' => out.push_str("\\x0C"), + 'v' => out.push_str("\\x0B"), + 'a' => out.push_str("\\x07"), + 'b' => out.push_str("\\x08"), + '0'..='7' => { + let mut v = e as u32 - '0' as u32; + let mut n = 1; + while n < 3 { + match chars.get(i + 1) { + Some(d @ '0'..='7') => { + v = v * 8 + (*d as u32 - '0' as u32); + i += 1; + n += 1; + } + _ => break, + } + } + push_lit(&mut out, char::from_u32(v).unwrap_or('\0')); + } + other => push_lit(&mut out, other), + } + at_start = false; + i += 1; + } + '[' => { + match bracket(&chars, i) { + Some((class, next)) => { + out.push_str(&class); + i = next; + } + None => { + out.push_str("\\["); + i += 1; + } + } + at_start = false; + } + '(' => { + out.push('('); + depth += 1; + at_start = true; + i += 1; + } + ')' => { + if depth > 0 { + depth -= 1; + out.push(')'); + } else { + out.push_str("\\)"); + } + at_start = false; + i += 1; + } + '|' => { + out.push('|'); + has_alt = true; + at_start = true; + i += 1; + } + '^' => { + out.push('^'); + at_start = true; + i += 1; + } + '$' => { + out.push('$'); + at_start = false; + i += 1; + } + '*' | '+' | '?' => { + if at_start { + push_lit(&mut out, c); + at_start = false; + } else { + out.push(c); + quantified = true; + } + i += 1; + } + '{' => match interval(&chars, i) { + Some((text, next)) if !at_start => { + out.push_str(&text); + quantified = true; + i = next; + } + _ => { + out.push_str("\\{"); + at_start = false; + i += 1; + } + }, + '}' => { + out.push_str("\\}"); + at_start = false; + i += 1; + } + '.' => { + out.push('.'); + at_start = false; + i += 1; + } + c => { + push_lit(&mut out, c); + at_start = false; + i += 1; + } + } + } + // Unbalanced `(`: close them so the pattern compiles like gawk's + // (gawk reports an error; an unclosed group is rare in practice). + for _ in 0..depth { + out.push(')'); + } + Translated { + pattern: out, + needs_longest: needs_longest || has_alt, + } +} + +/// `{n}`, `{n,}`, `{n,m}`, `{,m}` starting at `i`; `None` if not valid. +fn interval(chars: &[char], i: usize) -> Option<(String, usize)> { + let mut j = i + 1; + let mut lo = String::new(); + while let Some(c) = chars.get(j).filter(|c| c.is_ascii_digit()) { + lo.push(*c); + j += 1; + } + let mut hi = None; + if chars.get(j) == Some(&',') { + j += 1; + let mut h = String::new(); + while let Some(c) = chars.get(j).filter(|c| c.is_ascii_digit()) { + h.push(*c); + j += 1; + } + hi = Some(h); + } + if chars.get(j) != Some(&'}') { + return None; + } + if lo.is_empty() && hi.as_ref().is_none_or(|h| h.is_empty()) { + return None; + } + let lo = if lo.is_empty() { "0".to_string() } else { lo }; + let text = match hi { + None => format!("{{{lo}}}"), + Some(h) if h.is_empty() => format!("{{{lo},}}"), + Some(h) => format!("{{{lo},{h}}}"), + }; + Some((text, j + 1)) +} + +/// Translate a bracket expression starting at `chars[i] == '['`. +fn bracket(chars: &[char], i: usize) -> Option<(String, usize)> { + let mut j = i + 1; + let mut out = String::from("["); + if chars.get(j) == Some(&'^') { + out.push('^'); + j += 1; + } + let mut first = true; + let mut items = 0; + loop { + let c = *chars.get(j)?; + if c == ']' && !first { + j += 1; + break; + } + first = false; + // [:class:], [.c.], [=c=] + if c == '[' + && let Some(&d) = chars.get(j + 1) + && matches!(d, ':' | '.' | '=') + { + let mut k = j + 2; + let mut name = String::new(); + while k + 1 < chars.len() && !(chars[k] == d && chars[k + 1] == ']') { + name.push(chars[k]); + k += 1; + } + if k + 1 >= chars.len() { + return None; + } + j = k + 2; + if d == ':' { + out.push_str(match name.as_str() { + "alpha" => "\\p{Alphabetic}", + "upper" => "\\p{Uppercase}", + "lower" => "\\p{Lowercase}", + "alnum" => "\\p{Alphabetic}0-9", + "digit" => "0-9", + "space" => "\\s", + "blank" => " \\t", + "punct" => "[:punct:]", + "print" => "[:print:]", + "graph" => "[:graph:]", + "cntrl" => "[:cntrl:]", + "xdigit" => "[:xdigit:]", + "word" => "\\w", + _ => return None, + }); + } else { + for ch in name.chars() { + class_lit(&mut out, ch); + } + } + items += 1; + continue; + } + let (lit, next) = class_char(chars, j); + j = next; + // Range? + if chars.get(j) == Some(&'-') && chars.get(j + 1).is_some_and(|&n| n != ']') { + let (hi, after) = class_char(chars, j + 1); + if hi >= lit { + class_lit(&mut out, lit); + out.push('-'); + class_lit(&mut out, hi); + } + j = after; + } else { + class_lit(&mut out, lit); + } + items += 1; + } + if items == 0 { + return None; + } + out.push(']'); + Some((out, j)) +} + +/// One (possibly escaped) character inside a bracket expression. +fn class_char(chars: &[char], j: usize) -> (char, usize) { + if chars[j] == '\\' + && let Some(&e) = chars.get(j + 1) + { + let c = match e { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + 'f' => '\x0c', + 'v' => '\x0b', + 'a' => '\x07', + 'b' => '\x08', + other => other, + }; + return (c, j + 2); + } + (chars[j], j + 1) +} + +fn class_lit(out: &mut String, c: char) { + if regex_syntax_is_meta(c) { + out.push('\\'); + } + out.push(c); +} + +/// Per-run cache of compiled dynamic regexes (strings used as regexes). +#[derive(Default)] +pub(crate) struct RegexCache { + entries: HashMap<(Arc, bool), Result, String>>, + order: VecDeque<(Arc, bool)>, + bytes: usize, +} + +impl RegexCache { + pub(crate) fn get(&mut self, pattern: &str, icase: bool) -> Result, String> { + if let Some(r) = self.entries.get(&(Arc::from(pattern), icase)) { + return r.clone(); + } + if pattern.len() > REGEX_SIZE_LIMIT { + return Err("regular expression too big".to_string()); + } + let compiled = AwkRegex::new(pattern, icase).map(Arc::new).map_err(|e| e.0); + while self.entries.len() >= RUNTIME_REGEX_CACHE_ENTRIES + || self.bytes + pattern.len() > REGEX_SIZE_LIMIT + { + let Some(old) = self.order.pop_front() else { + break; + }; + self.bytes -= old.0.len(); + self.entries.remove(&old); + } + let key: (Arc, bool) = (Arc::from(pattern), icase); + self.bytes += pattern.len(); + self.order.push_back(key.clone()); + self.entries.insert(key, compiled.clone()); + compiled + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn re(p: &str) -> AwkRegex { + AwkRegex::new(p, false).ok().unwrap() + } + + #[test] + fn leftmost_longest() { + assert_eq!(re("ab|abcd").find_at("xabcd", 0), Some((1, 5))); + assert_eq!(re("a*(ab)?b?").find_at("aab", 0), Some((0, 3))); + assert_eq!(re("x*").find_at("abc", 0), Some((0, 0))); + assert_eq!(re("(a|ab)(c|bcd)").find_at("abcd", 0), Some((0, 4))); + } + + #[test] + fn translation() { + assert!(re("a.b").is_match("a\nb")); + assert!(re("[/]").is_match("/")); + assert!(re("[]a]").is_match("]")); + assert!(re("[^]a]").is_match("b")); + assert!(re("a{2}").is_match("aa")); + assert!(re("{").is_match("{")); + assert!(re("a{").is_match("a{")); + assert!(re("^+").is_match("+")); + assert!(!re("^+").is_match("a")); + assert!(re("\\yfoo\\y").is_match("a foo b")); + assert!(re("[[:alpha:]]+").is_match("olá")); + assert!(re("\\.").is_match(".")); + assert!(!re("\\.").is_match("a")); + assert!(re("\\d").is_match("d")); + assert!(re("[a\\]]").is_match("]")); + assert!(re("a|").is_match("b")); + assert!(re("\\").is_match("in the end")); + assert!(!re("\\").is_match("other")); + } + + #[test] + fn captures_follow_longest() { + let r = re("(a|ab)(c|bcd)"); + let c = r.captures_at("abcd", 0).unwrap(); + assert_eq!(c[0], Some((0, 4))); + } + + #[test] + fn huge_pattern_fails_cleanly() { + assert!(AwkRegex::new("((a|b){1000}){1000}", false).is_err()); + } +} diff --git a/crates/bashkit/src/builtins/awk/tests.rs b/crates/bashkit/src/builtins/awk/tests.rs index 3283cf35d..f685c2d95 100644 --- a/crates/bashkit/src/builtins/awk/tests.rs +++ b/crates/bashkit/src/builtins/awk/tests.rs @@ -1,6 +1,7 @@ //! Tests for awk: parser limits, interpreter limits, end-to-end runs. -use super::parser::AwkParser; +use super::ast::{Program, Source}; +use super::parser::{ParseError, Parser}; use super::{Awk, csv_split_fields}; #[rustfmt::skip] use crate::builtins::limits::{ @@ -20,8 +21,16 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; +fn parse(program: &str) -> std::result::Result { + let sources = [Source { + name: "cmd. line".to_string(), + start: 0, + }]; + Parser::new(program, &sources).parse_program() +} + async fn run_awk(args: &[&str], stdin: Option<&str>) -> Result { - let awk = Awk; + let awk = Awk::default(); let fs = Arc::new(InMemoryFs::new()); let mut vars = HashMap::new(); let mut cwd = PathBuf::from("/"); @@ -133,8 +142,10 @@ async fn test_awk_variables() { #[tokio::test] async fn test_awk_unicode_identifier_no_panic() { + // gawk identifiers are ASCII: a syntax error, not a panic. let result = run_awk(&["BEGIN{café=7; print café}"], None).await.unwrap(); - assert_eq!(result.stdout, "7\n"); + assert_eq!(result.exit_code, 1); + assert!(result.stderr.starts_with("awk: cmd. line:1: ")); } #[tokio::test] @@ -201,10 +212,9 @@ fn test_awk_parser_depth_limit_parens() { let close = ")".repeat(depth); let program = format!("{{print {open}1{close}}}"); - let mut parser = AwkParser::new(&program); - let result = parser.parse(); + let result = parse(&program); assert!(result.is_err(), "deeply nested parens must be rejected"); - let err = result.unwrap_err().to_string(); + let err = result.unwrap_err().msg; assert!( err.contains("nesting too deep"), "error should mention nesting: {err}" @@ -219,10 +229,9 @@ fn test_awk_parser_depth_limit_unary() { let prefix = "- ".repeat(depth); let program = format!("{{print {prefix}1}}"); - let mut parser = AwkParser::new(&program); - let result = parser.parse(); + let result = parse(&program); assert!(result.is_err(), "deeply chained unary ops must be rejected"); - let err = result.unwrap_err().to_string(); + let err = result.unwrap_err().msg; assert!( err.contains("nesting too deep"), "error should mention nesting: {err}" @@ -235,13 +244,12 @@ fn test_awk_parser_rejects_chained_range_pattern() { let operands = std::iter::repeat_n("1", 150).collect::>().join(","); let program = format!("{operands}{{print}}"); - let mut parser = AwkParser::new(&program); - let result = parser.parse(); + let result = parse(&program); assert!(result.is_err(), "comma-chained ranges must be rejected"); - let err = result.unwrap_err().to_string(); + let err = result.unwrap_err().msg; assert!( - err.contains("unexpected ',' after range pattern"), - "error should mention unexpected comma after range: {err}" + err.contains("syntax error"), + "comma after a range is a syntax error: {err}" ); } @@ -254,8 +262,7 @@ fn test_awk_parser_moderate_nesting_ok() { let close = ")".repeat(depth); let program = format!("{{print {open}1{close}}}"); - let mut parser = AwkParser::new(&program); - let result = parser.parse(); + let result = parse(&program); assert!( result.is_ok(), "moderate nesting should succeed: {:?}", // debug-ok: assert-failure message @@ -431,10 +438,12 @@ async fn test_awk_unspaced_plus_minus_after_number() { assert_eq!(result.stdout, expected, "program: {program}"); } // Negative: a lone '.' is still not a number. - let err = run_awk(&["BEGIN{print .}"], None).await.unwrap_err(); + let r = run_awk(&["BEGIN{print .}"], None).await.unwrap(); + assert_eq!(r.exit_code, 1); assert!( - err.to_string().contains("invalid number"), - "unexpected: {err}" + r.stderr.contains("syntax error"), + "unexpected: {}", + r.stderr ); } @@ -635,9 +644,10 @@ async fn test_awk_rejects_too_many_multi_subscripts() { .join(","); let program = format!("BEGIN {{ a[{subscripts}] = 1 }}"); - let err = run_awk(&[&program], Some("")).await.unwrap_err(); + let r = run_awk(&[&program], Some("")).await.unwrap(); - assert!(err.to_string().contains("too many array subscripts")); + assert_eq!(r.exit_code, 2); + assert!(r.stderr.contains("too many array subscripts")); } #[tokio::test] @@ -669,7 +679,7 @@ async fn run_awk_with_fs( args: &[&str], stdin: Option<&str>, ) -> (Result, Arc) { - let awk = Awk; + let awk = Awk::default(); let fs = Arc::new(InMemoryFs::new()); let mut vars = HashMap::new(); let mut cwd = PathBuf::from("/"); @@ -759,14 +769,21 @@ async fn test_awk_print_redirect_multiple_to_same_file() { } #[tokio::test] -async fn test_awk_print_redirect_pipe_unsupported() { - // Pipe output should return clear error - let (result, _fs) = run_awk_with_fs(&[r#"BEGIN{print "hello" | "cat"}"#], None).await; - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); +async fn test_awk_print_redirect_pipe_needs_plan_driver() { + // Direct `execute` has no shell to run commands: clear message, and + // the output pipe's command reports 127. The shell path goes through + // the plan driver (see the spec tests). + let (result, _fs) = run_awk_with_fs( + &[r#"BEGIN{print "hello" | "cat"; print close("cat")}"#], + None, + ) + .await; + let result = result.unwrap(); + assert_eq!(result.stdout, "127\n"); assert!( - err_msg.contains("pipe"), - "expected pipe error, got: {err_msg}" + result.stderr.contains("running commands is not supported"), + "got: {}", + result.stderr ); } @@ -810,10 +827,14 @@ async fn test_awk_csv_nf() { #[tokio::test] async fn test_awk_csv_ofs() { - // In CSV mode, OFS defaults to comma + // Like gawk 5.3, --csv changes input splitting only; OFS stays " ". let result = run_awk(&["--csv", "{print $1, $3}"], Some("a,b,c")) .await .unwrap(); + assert_eq!(result.stdout, "a c\n"); + let result = run_awk(&["--csv", "-v", "OFS=,", "{print $1, $3}"], Some("a,b,c")) + .await + .unwrap(); assert_eq!(result.stdout, "a,c\n"); } @@ -895,7 +916,7 @@ async fn run_awk_with_file( file_path: &str, file_content: &str, ) -> Result { - let awk = Awk; + let awk = Awk::default(); let fs = Arc::new(InMemoryFs::new()); fs.write_file(std::path::Path::new(file_path), file_content.as_bytes()) .await @@ -1142,7 +1163,7 @@ async fn run_awk_with_custom_fs( stdin: Option<&str>, fs: Arc, ) -> Result { - let awk = Awk; + let awk = Awk::default(); let mut vars = HashMap::new(); let mut cwd = PathBuf::from("/"); let args: Vec = args.iter().map(|s| s.to_string()).collect(); @@ -1278,7 +1299,7 @@ async fn test_awk_getline_file_size_limit() { } #[tokio::test] -async fn test_awk_getline_file_normalizes_cache_key() { +async fn test_awk_getline_streams_are_keyed_by_name() { let fs = Arc::new(InMemoryFs::new()); fs.write_file(std::path::Path::new("/tmp/data.txt"), b"one\ntwo\n") .await @@ -1297,7 +1318,9 @@ async fn test_awk_getline_file_normalizes_cache_key() { .await .unwrap(); - assert_eq!(result.stdout, "1 one\n1 two\n0\n"); + // gawk keys redirections by the name as written: each spelling is its + // own stream (each still counts toward the open-file and byte caps). + assert_eq!(result.stdout, "1 one\n1 one\n1\n"); } #[tokio::test] diff --git a/crates/bashkit/src/builtins/awk/value.rs b/crates/bashkit/src/builtins/awk/value.rs new file mode 100644 index 000000000..dca1ef819 --- /dev/null +++ b/crates/bashkit/src/builtins/awk/value.rs @@ -0,0 +1,223 @@ +//! awk values: numbers, strings, strnums and the uninitialized value. +//! +//! Decisions (POSIX awk "Expressions in awk", gawk 5 behavior): +//! - Input-derived text (fields, `getline` vars, `split` elements, `ARGV`, +//! `ENVIRON`, `-v`/operand assignments) is a *strnum*: it compares +//! numerically when it looks like a number, as a string otherwise. +//! - String to number uses the C `strtod` prefix rule (`"12abc"` is 12, +//! `"0x1A"` is 0: hex is only recognized in program source). +//! - Number to string: integral values print as integers, others through +//! `CONVFMT` (or `OFMT` for `print`). +//! - String comparison is bytewise (C locale collation). + +use std::cmp::Ordering; + +use super::format::format_number; + +#[derive(Debug, Clone, PartialEq)] +pub(super) enum Value { + Num(f64), + Str(String), + /// Input text that looks numeric: compares as a number, prints as text. + StrNum(String, f64), + Uninit, +} + +impl Value { + /// Classify input-derived text as strnum or plain string. + pub(super) fn from_input(s: String) -> Value { + match looks_numeric(&s) { + Some(n) => Value::StrNum(s, n), + None => Value::Str(s), + } + } + + pub(super) fn to_num(&self) -> f64 { + match self { + Value::Num(n) | Value::StrNum(_, n) => *n, + Value::Str(s) => str_to_num(s), + Value::Uninit => 0.0, + } + } + + /// String form using `CONVFMT` for non-integral numbers. + pub(super) fn to_str_fmt(&self, convfmt: &str) -> String { + match self { + Value::Num(n) => format_number(*n, convfmt), + Value::Str(s) | Value::StrNum(s, _) => s.clone(), + Value::Uninit => String::new(), + } + } + + pub(super) fn to_bool(&self) -> bool { + match self { + Value::Num(n) => *n != 0.0, + Value::StrNum(_, n) => *n != 0.0, + Value::Str(s) => !s.is_empty(), + Value::Uninit => false, + } + } + + /// Bytes this value keeps on the heap (TM-DOS-110 accounting). + pub(super) fn heap_bytes(&self) -> usize { + match self { + Value::Str(s) | Value::StrNum(s, _) => s.len(), + _ => 0, + } + } + + pub(super) fn is_numeric(&self) -> bool { + matches!(self, Value::Num(_) | Value::StrNum(..) | Value::Uninit) + } + + /// `typeof()` name. + pub(super) fn type_name(&self) -> &'static str { + match self { + Value::Num(_) => "number", + Value::Str(_) => "string", + Value::StrNum(..) => "strnum", + Value::Uninit => "untyped", + } + } +} + +/// Compare two scalars by the POSIX rules: numeric when both are numeric +/// (numbers, numeric-looking input, uninitialized), string otherwise. +pub(super) fn compare(a: &Value, b: &Value, convfmt: &str) -> Ordering { + if a.is_numeric() && b.is_numeric() { + let (x, y) = (a.to_num(), b.to_num()); + // NaN compares unequal to everything; treat as equal-to-nothing. + return x.partial_cmp(&y).unwrap_or(if x.is_nan() && y.is_nan() { + Ordering::Equal + } else if x.is_nan() { + Ordering::Greater + } else { + Ordering::Less + }); + } + let (x, y) = (a.to_str_fmt(convfmt), b.to_str_fmt(convfmt)); + x.as_bytes().cmp(y.as_bytes()) +} + +/// Length of the longest prefix of `s` that `strtod` would consume, with +/// its value. Leading blanks are skipped. Returns `(0, 0.0)` for none. +pub(super) fn strtod_prefix(s: &str) -> (usize, f64) { + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() && matches!(b[i], b' ' | b'\t' | b'\n' | b'\r' | b'\x0b' | b'\x0c') { + i += 1; + } + let start = i; + if i < b.len() && (b[i] == b'+' || b[i] == b'-') { + i += 1; + } + // gawk accepts +inf/-inf/+nan/-nan only with an explicit sign. + if i > start { + let rest = &s[i..]; + for (word, val) in [("inf", f64::INFINITY), ("nan", f64::NAN)] { + if rest.len() >= 3 && rest[..3].eq_ignore_ascii_case(word) { + let neg = b[start] == b'-'; + let mut len = i + 3; + if word == "inf" && rest.len() >= 8 && rest[..8].eq_ignore_ascii_case("infinity") { + len = i + 8; + } + let v = if neg { -val } else { val }; + return (len, v); + } + } + } + let digits_start = i; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + let mut mantissa_digits = i - digits_start; + if i < b.len() && b[i] == b'.' { + let dot = i; + i += 1; + let frac_start = i; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + mantissa_digits += i - frac_start; + if mantissa_digits == 0 { + i = dot; + } + } + if mantissa_digits == 0 { + return (0, 0.0); + } + if i < b.len() && (b[i] == b'e' || b[i] == b'E') { + let mut j = i + 1; + if j < b.len() && (b[j] == b'+' || b[j] == b'-') { + j += 1; + } + let exp_start = j; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + if j > exp_start { + i = j; + } + } + let v = s[start..i].parse::().unwrap_or(0.0); + (i, v) +} + +/// awk string-to-number: the `strtod` prefix, 0 when there is none. +pub(super) fn str_to_num(s: &str) -> f64 { + strtod_prefix(s).1 +} + +/// `Some(n)` when the whole of `s` (blanks around allowed) is a number. +pub(super) fn looks_numeric(s: &str) -> Option { + let (len, v) = strtod_prefix(s); + if len == 0 { + return None; + } + if s[len..] + .bytes() + .all(|c| matches!(c, b' ' | b'\t' | b'\n' | b'\r' | b'\x0b' | b'\x0c')) + { + Some(v) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strtod_prefix_rules() { + assert_eq!(str_to_num("12abc"), 12.0); + assert_eq!(str_to_num(" -3.5e2x"), -350.0); + assert_eq!(str_to_num("0x1A"), 0.0); + assert_eq!(str_to_num(".5"), 0.5); + assert_eq!(str_to_num("."), 0.0); + assert_eq!(str_to_num("1e"), 1.0); + assert_eq!(str_to_num("inf"), 0.0); + assert_eq!(str_to_num("-inf"), f64::NEG_INFINITY); + assert!(str_to_num("+nan").is_nan()); + } + + #[test] + fn strnum_classification() { + assert!(looks_numeric(" 10 ").is_some()); + assert!(looks_numeric("10a").is_none()); + assert!(looks_numeric("").is_none()); + assert!(looks_numeric("007").is_some()); + } + + #[test] + fn comparison_rules() { + let n = |x| Value::Num(x); + let s = |x: &str| Value::Str(x.into()); + let i = |x: &str| Value::from_input(x.into()); + assert_eq!(compare(&i("10"), &i("9"), "%.6g"), Ordering::Greater); + assert_eq!(compare(&s("10"), &s("9"), "%.6g"), Ordering::Less); + assert_eq!(compare(&i("10"), &n(9.0), "%.6g"), Ordering::Greater); + assert_eq!(compare(&Value::Uninit, &n(0.0), "%.6g"), Ordering::Equal); + assert_eq!(compare(&Value::Uninit, &s(""), "%.6g"), Ordering::Equal); + } +} diff --git a/crates/bashkit/src/builtins/date.rs b/crates/bashkit/src/builtins/date.rs index 7dfeab51d..dbf556f40 100644 --- a/crates/bashkit/src/builtins/date.rs +++ b/crates/bashkit/src/builtins/date.rs @@ -206,6 +206,24 @@ impl Date { parse_date_string(s, self.now(), timezone) } + /// C `mktime`: local broken-down time `[year, month, day, hour, min, + /// sec]` in the sandbox `TZ` to epoch seconds. Out-of-range fields + /// carry over (month 13 is January of the next year). Used by awk. + pub(super) fn mktime(&self, tz: Option<&String>, t: [i64; 6]) -> Option { + let timezone = SandboxTimezone::from_env(tz).ok()?; + let months = t[0].checked_mul(12)?.checked_add(t[1] - 1)?; + let year = i32::try_from(months.div_euclid(12)).ok()?; + let month = u32::try_from(months.rem_euclid(12) + 1).ok()?; + let base = NaiveDate::from_ymd_opt(year, month, 1)?.and_hms_opt(0, 0, 0)?; + let secs = (t[2] - 1) + .checked_mul(86_400)? + .checked_add(t[3].checked_mul(3_600)?)? + .checked_add(t[4].checked_mul(60)?)? + .checked_add(t[5])?; + let dt = base.checked_add_signed(Duration::try_seconds(secs)?)?; + timezone.local_to_utc(dt, "").ok().map(|d| d.timestamp()) + } + /// Current virtual time as (epoch seconds, nanoseconds). Shared with /// `find`'s age tests so they see the same clock as `date`. pub(super) fn now_epoch(&self) -> (i64, u32) { diff --git a/crates/bashkit/src/interpreter/mod.rs b/crates/bashkit/src/interpreter/mod.rs index 279ee296a..f0bffc73b 100644 --- a/crates/bashkit/src/interpreter/mod.rs +++ b/crates/bashkit/src/interpreter/mod.rs @@ -1637,10 +1637,6 @@ impl Interpreter { // Text processing "grep" => Grep, "sed" => Sed, - "awk" => Awk, - "gawk" => Awk, - "mawk" => Awk, - "nawk" => Awk, "head" => Head, "tail" => Tail, "sort" => Sort, @@ -1854,6 +1850,9 @@ impl Interpreter { Arc::new(builtins::Touch::with_clock(clock)), ); builtins.insert("pr".to_string(), Arc::new(builtins::Pr::with_clock(clock))); + for name in ["awk", "gawk", "mawk", "nawk"] { + builtins.insert(name.to_string(), Arc::new(builtins::Awk::with_clock(clock))); + } // System info builtins (configurable virtual values) let hostname_val = hostname.unwrap_or_else(|| builtins::DEFAULT_HOSTNAME.to_string()); diff --git a/crates/bashkit/tests/integration/builtin_fs_error_tests.rs b/crates/bashkit/tests/integration/builtin_fs_error_tests.rs index 2062ae5b8..aaf9a7979 100644 --- a/crates/bashkit/tests/integration/builtin_fs_error_tests.rs +++ b/crates/bashkit/tests/integration/builtin_fs_error_tests.rs @@ -26,7 +26,7 @@ const READERS: &[(&str, &str, i32)] = &[ ("uniq", "uniq /nope", 1), ("cut", "cut -c1 /nope", 1), ("nl", "nl /nope", 1), - ("awk", "awk '{print}' /nope", 1), + ("awk", "awk '{print}' /nope", 2), ("paste", "paste /nope", 1), ("rev", "rev /nope", 1), ("tac", "tac /nope", 1), diff --git a/crates/bashkit/tests/spec_cases/awk/awk.test.sh b/crates/bashkit/tests/spec_cases/awk/awk.test.sh index 19cd23c05..7e92bb296 100644 --- a/crates/bashkit/tests/spec_cases/awk/awk.test.sh +++ b/crates/bashkit/tests/spec_cases/awk/awk.test.sh @@ -745,3 +745,53 @@ echo '5' | awk '{ ### expect yes ### end + +### awk_system_runs_in_shell +# system() runs the command in the sandbox shell, output stays in order +awk 'BEGIN { print "before"; r = system("echo inside; exit 3"); print "after", r }' +### expect +before +inside +after 3 +### end + +### awk_print_to_pipe +# print | cmd feeds the command; close() runs it and returns its status +printf 'b\na\nc\n' | awk '{ print | "sort" } END { r = close("sort"); print "done", r }' +### expect +a +b +c +done 0 +### end + +### awk_cmd_getline_loop +# cmd | getline reads the command's output line by line +awk 'BEGIN { while (("printf \"1\\n2\\n3\\n\"" | getline n) > 0) s += n; close("x"); print s, NR }' +### expect +6 0 +### end + +### awk_gawk_syntax_error +# Syntax errors use gawk's message with a caret, exit 1 +awk 'BEGIN { print ( }' 2>&1; echo "rc=$?" +### expect +awk: cmd. line:1: BEGIN { print ( } +awk: cmd. line:1: ^ syntax error +rc=1 +### end + +### awk_substr_gawk_edges +# gawk: start below 1 keeps the length, fractions truncate +awk 'BEGIN { s = "hello world"; print substr(s, 0, 3) "|" substr(s, -1, 3) "|" substr(s, 1.5, 2.3) }' +### expect +hel|hel|he +### end + +### awk_missing_file_is_fatal +# A missing input file is a gawk fatal error (exit 2) +awk '{ print }' /nope 2>&1; echo "rc=$?" +### expect +awk: fatal: cannot open file `/nope' for reading: No such file or directory +rc=2 +### end diff --git a/knowledge/foundations/builtins.md b/knowledge/foundations/builtins.md index 463dc3bd8..e11c1bc30 100644 --- a/knowledge/foundations/builtins.md +++ b/knowledge/foundations/builtins.md @@ -384,6 +384,38 @@ Deliberate gaps are L-MAKE-001..003 in [Known Limitations](../operations/limitations.md); amplification caps are TM-DOS-126 in the [Threat Model](../security/threat-model.md). +### awk + +`awk` (also `gawk`, `mawk`, `nawk`) targets gawk 5.2, Debian's `awk` +(`builtins/awk/`): its options (`-F`, `-v`, `-f`, `-e`, `--csv`, ignored gawk +flags, usage on stderr with exit 1), its messages (`awk: cmd. line:1:` with a +caret for syntax errors, located runtime `fatal:` errors), its number output +and the extensions agents use (`gensub`, `asort`/`asorti`, `patsplit`, +`strftime`/`mktime`/`systime`, `BEGINFILE`/`ENDFILE`, `switch`, arrays of +arrays, `PROCINFO["sorted_in"]`, `FIELDWIDTHS`/`FPAT`, `IGNORECASE`, `RS` as a +regex, `RT`). `rand()` reproduces gawk's random() sequence. Debian-oracle +differential score: 94% of 260 awk cases (51% before the rewrite); the +remaining gawk gap is unsorted `for (k in a)` order. + +Pipeline: `lexer` -> `parser` (names resolved to slots, AST in `ast`) -> +`interp` (async evaluator) with `io` (records, `getline`, redirections, +commands) and `funcs` (builtin functions). `regex` translates EREs to the +`regex` crate and adds POSIX leftmost-longest matching with a hybrid DFA +when an alternation or quantifier could prefer a shorter match. Arrays live +in an arena referenced by id, which gives by-reference array arguments. +Each expression or statement kind is its own boxed future, so one awk call +level costs a few KiB of stack (the 64-level call cap fits a 2 MiB debug +thread with room to spare). + +Commands (`system()`, `print | cmd`, `cmd | getline`) run as `sh -c` through +an `ExecutionPlan::Driver`, used only when the arguments contain `|` or +`system` or a `-f` program; awk output is streamed with `PlanStep::Emit` +before each command so order is kept. Without a driver (direct `execute`), +commands report 127. Gaps are L-AWK-001..004 in +[Known Limitations](../operations/limitations.md); caps are TM-DOS-027, +-028, -033, -109, -110, -116 and -128 in the +[Threat Model](../security/threat-model.md). + ### Network Builtins `curl`, `wget`, `http` require the `http_client` feature + URL allowlist. diff --git a/knowledge/log.md b/knowledge/log.md index 196d2b0cc..44b2da4bf 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Feature**: awk rewritten to target gawk 5.2 (Debian's `awk`). New lexer, parser, async evaluator, ERE translation with POSIX leftmost-longest matching, gawk number formatting and `printf`, gawk's `rand()` sequence, `substr`/`getline`/`typeof` edge cases, `-f`/`-e`/`--` options and gawk's error messages. `system()`, `print | cmd` and `cmd | getline` now run through the sandbox shell (execution-plan driver). Debian-oracle score for awk went from 51% to 94% of 260 cases; the remaining gawk gap is unsorted `for (k in a)` order (L-AWK-001). All resource caps kept; new TM-DOS-128 for awk commands. See foundations/builtins "awk". * **Feature**: `sudo` and `busybox`, the top missing commands in the gap replay (`sudo tee /etc/x <>` clamp shift amount | **MITIGATED** | | TM-DOS-043 | Arithmetic side-effect overflow/panic | `((x+=1))` or `$((x++))` at the `i64` boundaries | Compound assignment and prefix/postfix increment/decrement use wrapping arithmetic in both evaluator paths | **MITIGATED** | @@ -295,6 +295,7 @@ runaway scripts without permanently breaking the session. | TM-DOS-125 | Child-shell recursion stack overflow | A script that runs itself via `sh`/`bash` (`echo 'sh s.sh' > s.sh; sh s.sh`) nested interpreters until the process stack overflowed, aborting every tenant in the process | Each child shell counts against `max_function_depth` and a separate cap of `MAX_CHILD_SHELL_DEPTH` (8) nested shells, sized so the deepest mix fits a 2 MiB debug stack; over the cap the shell fails `maximum nesting depth exceeded` (exit 2). Forked jobs inherit the depth because they are polled on the caller's stack. Regression: `recursive_child_shell_is_bounded` | **MITIGATED** | | TM-DOS-126 | make amplification | Recursive variables (`X = $(X)$(X)`), self-including makefiles, `$(MAKE)` recursion, huge dependency graphs or `$(wildcard **)` fan-out turning one `make` call into unbounded CPU, memory or stack | Expansion depth 200 and 4 MiB per expansion, recursive self-reference is fatal as in GNU; include depth 16; at most 100,000 targets, 10,000 `$(shell)`/`$(wildcard)`/`include` answers and 100,000 paths per `$(wildcard)` per run; `$(MAKE)` stops at MAKELEVEL 4, sized so the deepest recursion fits a 2 MiB debug thread stack (each level is roughly 200 KiB of make, `sh -c` and interpreter frames). Recipes run as ordinary `sh -c` commands under the session's budget, timeout and cancel token. Regressions: `builtins::make::expand::tests`, `l_make_003_sequential_sandbox_shell` | **MITIGATED** | | TM-DOS-127 | Pattern substitution amplification | `${x//pattern/rep}` with a glob pattern over a long value, or a huge pattern, turning one expansion into quadratic matching | Glob patterns compile to a size-capped (1 MiB) linear-time regex; extglob patterns fall back to at most 10,000 `glob_match` attempts and leave the value unchanged past the budget; results stay under `MAX_EXPANSION_RESULT_BYTES`. Regressions: `param-operand-quotes.test.sh`, `replace_pattern_extglob_budget` | **MITIGATED** | +| TM-DOS-128 | awk command amplification | `system()`, `print \| cmd` and `cmd \| getline` in a loop, or a command whose output floods awk | Commands run as ordinary `sh -c` commands through the awk execution-plan driver, under the session's command budget, timeout and cancel token; loops around them hit the awk loop caps (TM-DOS-033); `print \| cmd` input and `cmd \| getline` output count toward the awk output cap (TM-DOS-028 caps) and the getline open-file and retained-byte caps (TM-DOS-116), charged to the request budget. Per awk call level the evaluator keeps one small boxed future per expression or statement, so the 64-level call cap fits in about 512 KiB of debug-build stack. Regressions: `test_awk_print_redirect_pipe_needs_plan_driver`, `test_awk_recursive_function_depth_limit` | **MITIGATED** | | TM-DOS-101 | yq structured-data amplification | YAML aliases can expand exponentially during deserialization; deep YAML/JSON, document floods, jaq generators, and YAML re-serialization can consume stack, CPU, or memory beyond the source size | Reject YAML alias tokens with a bounded lexical pass before deserialization; VFS/stdin and aggregate input budgets; serde_yaml_ng recursion cap 128 plus Bashkit depth 100; 4096-document cap; shared jaq work/deadline/output limits; post-serialization stdout cap; YAML+JSON depth/document/input/output regressions plus `yq_fuzz` and arbitrary-input proptest | **MITIGATED** | | TM-DOS-107 | Static-analysis command validation amplification | A large comment before thousands of commands makes a whole-script ordered-subsequence scan per command | Build one source-character position index, then validate each analyzed command-name character with a logarithmic position lookup instead of rescanning source | **MITIGATED** | **TM-DOS-051** is historical. The custom parser was deleted with the `yaml` @@ -320,7 +321,7 @@ panicked. Resolved with `wrapping_*` ops, masked shift amounts, clamped exponent **TM-DOS-031** (mitigated): see table, linear backtracking for plain `*`, plus a recursion-depth cap for extglob nesting in `glob_match_impl` (`interpreter/glob.rs`). -**Implementation**: `timeout` 30s + `parser_timeout` 5s + `max_parser_operations` 100K in `limits.rs` (TM-DOS-023/024); `MAX_AWK_PARSER_DEPTH` 100 (`builtins/awk.rs`) and `MAX_JQ_JSON_DEPTH` 100 (`builtins/jq/`) for TM-DOS-027; `MAX_LCS_CELLS` 10M (`builtins/diff.rs`) for TM-DOS-028. +**Implementation**: `timeout` 30s + `parser_timeout` 5s + `max_parser_operations` 100K in `limits.rs` (TM-DOS-023/024); `AWK_MAX_PARSER_DEPTH` 100 (`builtins/awk/parser.rs`) and `MAX_JQ_JSON_DEPTH` 100 (`builtins/jq/`) for TM-DOS-027; `MAX_LCS_CELLS` 10M (`builtins/diff.rs`) for TM-DOS-028. --- @@ -1411,7 +1412,7 @@ This section maps former vulnerability IDs to the new threat ID scheme and track | AST depth limit (100) | TM-DOS-022 | `limits.rs` | Yes | | Child parser limit propagation | TM-DOS-021 | `parser/mod.rs` | Yes | | Arithmetic depth limit (50) | TM-DOS-026 | `interpreter/mod.rs` | Yes | -| Builtin parser depth limit (100) | TM-DOS-027 | `builtins/awk.rs`, `builtins/jq/` | Yes | +| Builtin parser depth limit (100) | TM-DOS-027 | `builtins/awk/parser.rs`, `builtins/jq/` | Yes | | Execution timeout (30s) | TM-DOS-023 | `limits.rs` | Yes | | Builtin output pre-allocation caps | TM-DOS-058, TM-DOS-090 | `limits.rs`, `builtins/shuf.rs` | Yes | | Shared request execution budget | TM-DOS-096 | `limits.rs`, `parser/mod.rs`, `interpreter/mod.rs`, high-risk builtins/runtimes | Yes | @@ -1522,7 +1523,7 @@ ExecutionLimits::new() .max_history_bytes(1_048_576) // TM-DOS-094 .max_history_output_bytes(1_048_576) // TM-DOS-094 // Note: MAX_ARITHMETIC_DEPTH (50) is a compile-time constant in interpreter (TM-DOS-026) -// Note: MAX_AWK_PARSER_DEPTH (100) is a compile-time constant in builtins/awk.rs (TM-DOS-027) +// Note: AWK_MAX_PARSER_DEPTH (100) is a compile-time constant in builtins/limits.rs (TM-DOS-027) // Note: MAX_JQ_JSON_DEPTH (100) is a compile-time constant in builtins/jq/ (TM-DOS-027) // Note: MAX_FILE_VAR_REQUESTS (128) and MAX_FILE_VAR_BYTES (16MiB) cap jq file bindings (TM-DOS-062)