diff --git a/.github/workflows/debian-oracle.yml b/.github/workflows/debian-oracle.yml new file mode 100644 index 000000000..73e73a381 --- /dev/null +++ b/.github/workflows/debian-oracle.yml @@ -0,0 +1,62 @@ +name: Debian Oracle Scoreboard + +# Runs the pseudo-linus Debian-oracle bench (fetched at a pinned commit, see +# scripts/debian-oracle/run.sh) and fails when a tool's score drops below +# scripts/debian-oracle/floors.tsv. Scores go to the job summary. See +# knowledge/operations/testing.md "Debian-Oracle Scoreboard". + +on: + pull_request: + paths: + - 'crates/bashkit/src/builtins/**' + - 'crates/bashkit/src/interpreter/**' + - 'crates/bashkit/src/parser/**' + - 'crates/bashkit/src/fs/**' + - 'crates/jaq-json/**' + - 'scripts/debian-oracle/**' + - '.github/workflows/debian-oracle.yml' + schedule: + # Weekly Mondays 06:00 UTC. + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: debian-oracle-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + scoreboard: + name: Debian oracle scoreboard + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + workspaces: target/debian-oracle/pseudo-linus/testbench -> ../../target + + - name: Run scoreboard + run: scripts/debian-oracle/run.sh + + - name: Upload failing cases + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: debian-oracle-fails + path: | + target/debian-oracle/fails.txt + target/debian-oracle/report.md + if-no-files-found: ignore diff --git a/crates/bashkit/src/builtins/sed/mod.rs b/crates/bashkit/src/builtins/sed/mod.rs index b6979998c..eaaf7ae3e 100644 --- a/crates/bashkit/src/builtins/sed/mod.rs +++ b/crates/bashkit/src/builtins/sed/mod.rs @@ -304,7 +304,15 @@ impl Builtin for Sed { let path = resolve(ctx.cwd, name); match ctx.fs.read_file(&path).await { Ok(bytes) => { - script.push_str(String::from_utf8_lossy(&bytes).trim_end_matches('\n')) + let text = String::from_utf8_lossy(&bytes); + let trimmed = text.trim_end_matches('\n'); + script.push_str(trimmed); + // An escaped final newline (`i\` + newline) is + // text, an empty line in GNU sed; keep it. + let escapes = trimmed.chars().rev().take_while(|&c| c == '\\').count(); + if escapes % 2 == 1 && trimmed.len() < text.len() { + script.push('\n'); + } } // GNU exits 4 when a script file cannot be opened. Err(e) => { diff --git a/crates/bashkit/src/builtins/sed/tests.rs b/crates/bashkit/src/builtins/sed/tests.rs index 6d97dcacf..be962a6ba 100644 --- a/crates/bashkit/src/builtins/sed/tests.rs +++ b/crates/bashkit/src/builtins/sed/tests.rs @@ -808,6 +808,18 @@ async fn escaped_delimiter_in_a_regex_is_the_plain_character() { assert_eq!(out(&["s.a\\.b.X."], "axb\n").await, "X\n"); } +#[tokio::test] +async fn script_file_ending_in_backslash_newline_inserts_an_empty_line() { + // `i\` + newline as the last line of a script file is an empty text + // line in GNU sed; only a bare `i\` at the very end adds nothing. + let fs = Arc::new(InMemoryFs::new()); + fs.write_file(std::path::Path::new("/in1"), b"/foo/i\\\n") + .await + .unwrap(); + let r = run_with(fs, &["-f", "/in1"], Some("bar\nfoo\n")).await; + assert_eq!(r.stdout, "bar\n\nfoo\n"); +} + #[tokio::test] async fn bare_append_backslash_adds_nothing() { assert_eq!(out(&["a\\"], "a\n").await, "a\n"); diff --git a/justfile b/justfile index c33d0fa51..3d53c4d81 100644 --- a/justfile +++ b/justfile @@ -217,6 +217,12 @@ run-script file *args: # === Benchmarks === +# Debian-oracle scoreboard (pseudo-linus bench, fetched at a pinned commit). +# Fails on a regression below scripts/debian-oracle/floors.tsv; pass +# --update-floors after a fix raises a score. Optional TOOL args narrow the run. +debian-oracle *ARGS: + scripts/debian-oracle/run.sh {{ARGS}} + # Run benchmarks comparing bashkit to bash and save site-indexed JSON/Markdown results bench: cargo run -p bashkit-bench --release -- --save diff --git a/knowledge/log.md b/knowledge/log.md index 9ab3f2091..845482608 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-07 +* **Test**: Debian-oracle scoreboard. `just debian-oracle` / `.github/workflows/debian-oracle.yml` run the pseudo-linus bench (about 9,600 cases recorded on real Debian, fetched at a pinned commit because parts of its corpus derive from GPL suites) and fail when a tool drops below `scripts/debian-oracle/floors.tsv`. See operations/testing "Debian-Oracle Scoreboard". * **Fix**: `grep` follows GNU grep 3.11. Prefixes print as `FILE:LINE:BYTE:` (context lines with `-`), group separators appear between non-adjacent groups and files whenever a context option is given (`-NUM`, `--group-separator`, `--no-group-separator`), trailing context after `-m` is printed in full, `-o` skips empty matches and is POSIX leftmost-longest (`grep -oE 'ab|abcd'` prints `abcd`), `-b -o` gives each match's offset. New `grep_pattern` translates GNU BRE/ERE (brackets, intervals, `\< \> \b \w`, literal leading `*` in BRE, dropped leading ERE operator with GNU's warning, glibc error messages) onto awk's leftmost-longest `AwkRegex`; back-references work in BRE and ERE via fancy-regex (TM-DOS-025 cap). Options parse like getopt_long (unique prefixes such as `--no-gr`, `-T`, `-d`, `--label`, `-` for stdin, `-R` follows symlinks), an empty `-f` file matches nothing, a directory operand without `-r` reports `Is a directory`, invalid UTF-8 output turns the file binary, `\r` is line data. Debian-oracle grep score 60.5% to 96.8% (185 cases); grep's probes in the regex set all pass except two raw-`\x01` patterns the shell drops. Remaining misses are readdir order (L-GREP-003); L-GREP-002 lists the other gaps. See foundations/builtins "grep". * **Fix**: jq `implode` maps invalid code points to U+FFFD, `strptime`/`fromdate` fail with jq's `date "X" does not match format "F"`, and `@base64d` accepts unpadded input and non-zero trailing bits with jq's error texts. diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index c9b7bc915..e49cc4feb 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -315,6 +315,32 @@ Do not re-import over existing files; fix cases in place. If an intentional divergence is found, switch the case to `### bash_diff` with a canonical `L-*` row in [Known Limitations](limitations.md). +## Debian-Oracle Scoreboard + +`scripts/debian-oracle/run.sh` (`just debian-oracle`) runs the +[pseudo-linus](https://github.com/JoaoHenriqueBarbosa/pseudo-linus) test bench +(MIT) against Bashkit: about 9,600 command cases across 45 tools (awk, jq, +sed, grep, coreutils, ...) whose stdout, stderr, exit status and final file +tree were recorded on real Debian. The adapter in +`scripts/debian-oracle/adapter/` loads each fixture into the in-memory +filesystem and compares with the bench's own `harness::compare_outcome`. + +- **Fetched, not vendored.** Parts of the bench corpus derive from GPL test + suites (GNU grep/sed), so the script clones it at a pinned commit + (`PSEUDO_LINUS_REV`) into `target/debian-oracle/` at run time. Bump the pin + deliberately and refresh floors in the same change. +- **Floors.** `scripts/debian-oracle/floors.tsv` holds each tool's minimum + lenient pass count. The run fails when a tool drops more than one case below + its floor (one case of slack for wall-clock timeouts). After a fix raises a + score, run `scripts/debian-oracle/run.sh --update-floors` and commit the + floors so the gain is locked in. +- **CI.** `.github/workflows/debian-oracle.yml` runs on PRs touching + builtins, the interpreter or the scoreboard, weekly, and on demand. The + table goes to the job summary; failing cases are uploaded as an artifact. +- Goldens come from Debian's tool versions (jq 1.7.1, GNU grep 3.11, gawk); + version-specific gaps (Bashkit targets jq 1.8) stay as failures rather than + being special-cased. + ## SQLite CLI Differential Tests CSV comparisons explicitly set the host CLI row separator to LF. SQLite diff --git a/scripts/debian-oracle/adapter/Cargo.toml b/scripts/debian-oracle/adapter/Cargo.toml new file mode 100644 index 000000000..e679fa5db --- /dev/null +++ b/scripts/debian-oracle/adapter/Cargo.toml @@ -0,0 +1,16 @@ +# Bashkit adapter for the pseudo-linus Debian-oracle test bench. +# +# Not a bashkit workspace member: `run.sh` copies this crate into a pinned +# pseudo-linus checkout (testbench/crates/bk-oracle) and points the bashkit +# dependency at this repository, because it links the bench's `harness` crate. +[package] +name = "bk-oracle" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +harness = { workspace = true } +anyhow = "1" +tokio = { version = "1", features = ["rt", "time"] } +bashkit = { path = "@BASHKIT@", features = ["jq", "git", "sqlite"] } diff --git a/scripts/debian-oracle/adapter/src/main.rs b/scripts/debian-oracle/adapter/src/main.rs new file mode 100644 index 000000000..0b2623d64 --- /dev/null +++ b/scripts/debian-oracle/adapter/src/main.rs @@ -0,0 +1,205 @@ +//! Runs the pseudo-linus Debian-oracle cases against bashkit. +//! +//! Each case's fixture goes into bashkit's in-memory filesystem at +//! `harness::CASE_DIR` (modes, symlinks, mtimes), the script runs with the +//! bench environment, stdin and `ExecutionLimits::cli()`, and the outcome +//! (stdout, stderr, exit, final tree) is compared with the Debian golden by +//! the bench's own `harness::compare_outcome`. +//! +//! Output: one `toolcasesstrictlenient` line per tool on +//! stdout; failing case ids with a short diff go to `$FAILS_OUT`. +//! Decision: each case runs on its own thread with a 10 s wall clock, so a +//! hang or panic costs that case only. + +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::time::{Duration, SystemTime}; + +use bashkit::{Bash, ExecOptions, ExecutionLimits, FileSystem, FileType, GitConfig}; +use harness::{Bytes, Entry, Invocation, MemTree, Outcome}; + +const CASE_TIMEOUT: Duration = Duration::from_secs(10); + +fn main() -> anyhow::Result<()> { + let args: Vec = std::env::args().skip(1).collect(); + let tools = if args.is_empty() { harness::paths::tools()? } else { args }; + let mut fails = Vec::new(); + for tool in tools { + let (cases, _) = harness::paths::load_tool(&tool)?; + let (mut strict, mut lenient) = (0, 0); + for (case, golden) in &cases { + let Ok(inv) = case.invocation() else { continue }; + let actual = run_isolated(inv); + let cmp = harness::compare_outcome(case, golden, &actual); + strict += usize::from(cmp.strict); + if cmp.lenient { + lenient += 1; + } else { + let detail: String = cmp.detail.join(" | ").chars().take(400).collect(); + fails.push(format!("{tool}/{}: {detail}", cmp.id)); + } + } + println!("{tool}\t{}\t{strict}\t{lenient}", cases.len()); + } + let out = std::env::var("FAILS_OUT").unwrap_or_else(|_| "fails.txt".into()); + std::fs::write(out, fails.join("\n") + "\n")?; + Ok(()) +} + +fn run_isolated(inv: Invocation) -> Outcome { + let (tx, rx) = mpsc::channel(); + std::thread::Builder::new() + .stack_size(64 << 20) + .spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime"); + let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + rt.block_on(run_case(&inv)) + })); + let _ = tx.send(res.unwrap_or_else(|_| Outcome::unsupported("panic"))); + }) + .expect("spawn case thread"); + rx.recv_timeout(CASE_TIMEOUT).unwrap_or_else(|_| Outcome { + timed_out: true, + ..Outcome::unsupported("timeout") + }) +} + +async fn run_case(inv: &Invocation) -> Outcome { + let mut builder = Bash::builder() + .cwd(harness::CASE_DIR) + .limits(ExecutionLimits::cli()) + .username("root") + .hostname("oracle") + .git(GitConfig::new()) + .sqlite(); + for (k, v) in inv.full_env() { + builder = builder.env(k, v); + } + if let Some(ft) = &inv.faketime { + match faketime_epoch(ft) { + Some(epoch) => builder = builder.fixed_epoch(epoch), + None => return Outcome::unsupported(format!("unsupported faketime: {ft}")), + } + } + let mut bash = builder.build(); + let fs = bash.fs(); + if let Err(e) = load_fixture(fs.as_ref(), &inv.files).await { + return Outcome::unsupported(format!("fixture: {e}")); + } + let script = match &inv.script { + Some(s) => s.clone(), + None => inv.argv.iter().map(|a| shell_quote(a)).collect::>().join(" "), + }; + let opts = ExecOptions::new().stdin(inv.stdin.clone()); + let (stdout, stderr, exit) = match bash.exec_with_options(&script, opts).await { + Ok(r) => (r.stdout.into_bytes(), r.stderr.into_bytes(), r.exit_code), + // An API error (parse, limit) is the observable result of the case. + Err(e) => { + let code = if matches!(e, bashkit::Error::Parse { .. }) { 2 } else { 1 }; + (Vec::new(), format!("bashkit: {e}\n").into_bytes(), code) + } + }; + let files = match snapshot(fs.as_ref()).await { + Ok(t) => t, + Err(e) => return Outcome::unsupported(format!("snapshot: {e}")), + }; + Outcome { + stdout: Bytes(stdout), + stderr: Bytes(stderr), + exit: Some(exit), + files, + ..Outcome::default() + } +} + +async fn load_fixture(fs: &dyn FileSystem, tree: &MemTree) -> bashkit::Result<()> { + let root = PathBuf::from(harness::CASE_DIR); + let mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(harness::FIXTURE_MTIME); + fs.mkdir(&root, true).await?; + // BTreeMap order: parents before children. + for (rel, entry) in &tree.entries { + let path = root.join(rel); + match entry { + Entry::Dir { .. } => fs.mkdir(&path, true).await?, + Entry::File { data, .. } => { + let bytes = data.as_ref().map(|d| d.0.clone()).unwrap_or_default(); + fs.write_file(&path, &bytes).await?; + } + Entry::Symlink { target } => fs.symlink(Path::new(target), &path).await?, + } + } + // Modes and mtimes deepest first, so writing a child does not bump its parent. + for (rel, entry) in tree.entries.iter().rev() { + if let Entry::File { mode, .. } | Entry::Dir { mode } = entry { + let path = root.join(rel); + fs.chmod(&path, *mode).await?; + fs.set_modified_time(&path, mtime).await?; + } + } + fs.set_modified_time(&root, mtime).await +} + +async fn snapshot(fs: &dyn FileSystem) -> bashkit::Result { + let root = PathBuf::from(harness::CASE_DIR); + let mut tree = MemTree::new(); + if !fs.exists(&root).await? { + return Ok(tree); + } + let mut stack = vec![root.clone()]; + while let Some(dir) = stack.pop() { + for item in fs.read_dir(&dir).await? { + let path = dir.join(&item.name); + let rel = path + .strip_prefix(&root) + .expect("inside case dir") + .to_string_lossy() + .into_owned(); + let mode = item.metadata.mode & 0o7777; + let entry = match item.metadata.file_type { + FileType::Symlink => { + Entry::symlink(fs.read_link(&path).await?.to_string_lossy().into_owned()) + } + FileType::Directory => { + stack.push(path); + Entry::dir(mode) + } + FileType::File => Entry::file(fs.read_file(&path).await?, mode), + FileType::Fifo => Entry::file(Vec::new(), mode | 0o010000), + }; + tree.entries.insert(rel, entry); + } + } + Ok(tree) +} + +/// `faketime` spec ("YYYY-MM-DD HH:MM:SS" UTC, or "@epoch") to epoch seconds. +fn faketime_epoch(spec: &str) -> Option { + let spec = spec.trim(); + if let Some(rest) = spec.strip_prefix('@') { + return rest.parse().ok(); + } + let (date, time) = spec.split_once(' ').unwrap_or((spec, "00:00:00")); + let d: Vec = date.split('-').map(|p| p.parse().ok()).collect::>()?; + let t: Vec = time.split(':').map(|p| p.parse().ok()).collect::>()?; + if d.len() != 3 || t.is_empty() || t.len() > 3 { + return None; + } + // Days since 1970-01-01, proleptic Gregorian (Howard Hinnant's algorithm). + let y = if d[1] <= 2 { d[0] - 1 } else { d[0] }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let doy = (153 * ((d[1] + 9) % 12) + 2) / 5 + d[2] - 1; + let days = era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468; + let secs = t[0] * 3600 + t.get(1).copied().unwrap_or(0) * 60 + t.get(2).copied().unwrap_or(0); + Some(days * 86_400 + secs) +} + +fn shell_quote(s: &str) -> String { + let plain = !s.is_empty() + && !s.starts_with('=') + && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"_./=:,+@%-".contains(&b)); + if plain { s.to_string() } else { format!("'{}'", s.replace('\'', r"'\''")) } +} diff --git a/scripts/debian-oracle/floors.tsv b/scripts/debian-oracle/floors.tsv new file mode 100644 index 000000000..1c46fd884 --- /dev/null +++ b/scripts/debian-oracle/floors.tsv @@ -0,0 +1,46 @@ +# toolminimum lenient passes (scripts/debian-oracle/run.sh --update-floors) +archive 62 +awk 245 +bc 9 +binutils 0 +column 41 +coreutils 665 +csv 12 +curl 0 +date 69 +diff 52 +dpkg 29 +envsubst 11 +faketime 0 +file 7 +find 27 +git 2 +glibc 0 +grep 179 +hexdump 55 +iconv 6 +initscripts 0 +jq 172 +less 0 +libc 35 +ncurses 1 +patch 15 +procps 30 +python 0 +regex 992 +scripts 0 +sed 171 +shadow 0 +shell 79 +smoke 5 +sqlite 3 +strings 50 +tree 10 +utillinux 43 +wget 0 +which 10 +xargs 24 +xxd 24 +yq 21 +zdump 0 +zic 0 diff --git a/scripts/debian-oracle/run.sh b/scripts/debian-oracle/run.sh new file mode 100755 index 000000000..92880e225 --- /dev/null +++ b/scripts/debian-oracle/run.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Debian-oracle scoreboard: run the pseudo-linus test bench against bashkit. +# +# The bench (https://github.com/JoaoHenriqueBarbosa/pseudo-linus, MIT) holds +# command cases with outputs recorded on real Debian. Some suites in its +# corpus are derived from GPL test suites (GNU grep/sed), so it is fetched at +# run time at a pinned commit, never vendored into this repository. +# +# Usage: run.sh [--update-floors] [TOOL...] +# Prints a markdown table (also appended to $GITHUB_STEP_SUMMARY when set) +# and fails when a tool's lenient pass count drops more than TOLERANCE cases +# below scripts/debian-oracle/floors.tsv. --update-floors rewrites the floors +# from this run. Failing cases land in $ORACLE_DIR/fails.txt. +set -euo pipefail + +PSEUDO_LINUS_REPO="https://github.com/JoaoHenriqueBarbosa/pseudo-linus" +PSEUDO_LINUS_REV="039036aa8148a6802afe10663c9253e059241b45" +# One case of slack per tool for wall-clock timeouts on slow runners. +TOLERANCE=1 + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +ORACLE_DIR="${ORACLE_DIR:-$ROOT/target/debian-oracle}" +SRC="$ORACLE_DIR/pseudo-linus" +FLOORS="$HERE/floors.tsv" + +update=0 +if [[ "${1:-}" == "--update-floors" ]]; then + update=1 + shift +fi + +mkdir -p "$ORACLE_DIR" +if [[ "$(git -C "$SRC" rev-parse HEAD 2>/dev/null)" != "$PSEUDO_LINUS_REV" ]]; then + rm -rf "$SRC" + git init -q "$SRC" + git -C "$SRC" fetch -q --depth 1 "$PSEUDO_LINUS_REPO" "$PSEUDO_LINUS_REV" + git -C "$SRC" checkout -q FETCH_HEAD +fi + +TB="$SRC/testbench" +rm -rf "$TB/crates/bk-oracle" +cp -r "$HERE/adapter" "$TB/crates/bk-oracle" +sed -i "s|@BASHKIT@|$ROOT/crates/bashkit|" "$TB/crates/bk-oracle/Cargo.toml" + +# The bench's .cargo/config.toml links with mold, which runners lack. +rm -f "$SRC/.cargo/config.toml" +(cd "$TB" && CARGO_TARGET_DIR="$ORACLE_DIR/target" \ + cargo build -q --release -p bk-oracle --ignore-rust-version) + +scores="$ORACLE_DIR/scores.tsv" +(cd "$TB" && FAILS_OUT="$ORACLE_DIR/fails.txt" "$ORACLE_DIR/target/release/bk-oracle" "$@") > "$scores" + +report="$ORACLE_DIR/report.md" +awk -F'\t' -v floors="$FLOORS" -v tol="$TOLERANCE" ' + BEGIN { + while ((getline line < floors) > 0) { + if (line ~ /^#/ || line == "") continue + split(line, f, "\t"); floor[f[1]] = f[2] + } + print "| Tool | Cases | Strict | Lenient | Floor |" + print "|------|------:|-------:|--------:|------:|" + } + { + n += $2; s += $3; l += $4 + fl = ($1 in floor) ? floor[$1] : "-" + mark = "" + if (fl != "-" && $4 < fl - tol) { mark = " **regressed**"; bad = 1 } + else if (fl != "-" && $4 > fl) { mark = " (above floor)" } + printf "| %s | %d | %.1f%% | %.1f%% (%d) | %s%s |\n", $1, $2, 100*$3/($2?$2:1), 100*$4/($2?$2:1), $4, fl, mark + } + END { + printf "| **Total** | %d | %.1f%% | %.1f%% | |\n", n, 100*s/(n?n:1), 100*l/(n?n:1) + exit bad + } +' "$scores" > "$report" && status=0 || status=$? + +cat "$report" +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "## Debian-oracle scoreboard" + echo + echo "pseudo-linus \`${PSEUDO_LINUS_REV:0:12}\`, lenient = output match ignoring known noise." + echo + cat "$report" + } >> "$GITHUB_STEP_SUMMARY" +fi + +if (( update )); then + { + echo "# toolminimum lenient passes (scripts/debian-oracle/run.sh --update-floors)" + cut -f1,4 "$scores" + } > "$FLOORS" + echo "floors updated: $FLOORS" + exit 0 +fi + +if (( status != 0 )); then + echo "::error::Debian-oracle score regressed below scripts/debian-oracle/floors.tsv; failing cases in $ORACLE_DIR/fails.txt" >&2 + exit 1 +fi