test: Debian-oracle scoreboard in CI with per-tool floors (+ sed -f regression it caught)

.github/workflows/debian-oracle.yml

@@ -0,0 +1,62 @@
+name: Debian Oracle Scoreboard
+
+# Runs the pseudo-linus Debian-oracle bench (fetched at a pinned commit, see
+# scripts/debian-oracle/run.sh) and fails when a tool's score drops below
+# scripts/debian-oracle/floors.tsv. Scores go to the job summary. See
+# knowledge/operations/testing.md "Debian-Oracle Scoreboard".
+
+on:
+  pull_request:
+    paths:
+      - 'crates/bashkit/src/builtins/**'
+      - 'crates/bashkit/src/interpreter/**'
+      - 'crates/bashkit/src/parser/**'
+      - 'crates/bashkit/src/fs/**'
+      - 'crates/jaq-json/**'
+      - 'scripts/debian-oracle/**'
+      - '.github/workflows/debian-oracle.yml'
+  schedule:
+    # Weekly Mondays 06:00 UTC.
+    - cron: '0 6 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: debian-oracle-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  scoreboard:
+    name: Debian oracle scoreboard
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0
+
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
+        with:
+          workspaces: target/debian-oracle/pseudo-linus/testbench -> ../../target
+
+      - name: Run scoreboard
+        run: scripts/debian-oracle/run.sh
+
+      - name: Upload failing cases
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: debian-oracle-fails
+          path: |
+            target/debian-oracle/fails.txt
+            target/debian-oracle/report.md
+          if-no-files-found: ignore

crates/bashkit/src/builtins/sed/mod.rs

@@ -304,7 +304,15 @@ impl Builtin for Sed {
                     let path = resolve(ctx.cwd, name);
                     match ctx.fs.read_file(&path).await {
                         Ok(bytes) => {
-                            script.push_str(String::from_utf8_lossy(&bytes).trim_end_matches('\n'))
+                            let text = String::from_utf8_lossy(&bytes);
+                            let trimmed = text.trim_end_matches('\n');
+                            script.push_str(trimmed);
+                            // An escaped final newline (`i\` + newline) is
+                            // text, an empty line in GNU sed; keep it.
+                            let escapes = trimmed.chars().rev().take_while(|&c| c == '\\').count();
+                            if escapes % 2 == 1 && trimmed.len() < text.len() {
+                                script.push('\n');
+                            }
                         }
                         // GNU exits 4 when a script file cannot be opened.
                         Err(e) => {

crates/bashkit/src/builtins/sed/tests.rs

@@ -808,6 +808,18 @@ async fn escaped_delimiter_in_a_regex_is_the_plain_character() {
     assert_eq!(out(&["s.a\\.b.X."], "axb\n").await, "X\n");
 }
 
+#[tokio::test]
+async fn script_file_ending_in_backslash_newline_inserts_an_empty_line() {
+    // `i\` + newline as the last line of a script file is an empty text
+    // line in GNU sed; only a bare `i\` at the very end adds nothing.
+    let fs = Arc::new(InMemoryFs::new());
+    fs.write_file(std::path::Path::new("/in1"), b"/foo/i\\\n")
+        .await
+        .unwrap();
+    let r = run_with(fs, &["-f", "/in1"], Some("bar\nfoo\n")).await;
+    assert_eq!(r.stdout, "bar\n\nfoo\n");
+}
+
 #[tokio::test]
 async fn bare_append_backslash_adds_nothing() {
     assert_eq!(out(&["a\\"], "a\n").await, "a\n");

justfile

@@ -217,6 +217,12 @@ run-script file *args:
 
 # === Benchmarks ===
 
+# Debian-oracle scoreboard (pseudo-linus bench, fetched at a pinned commit).
+# Fails on a regression below scripts/debian-oracle/floors.tsv; pass
+# --update-floors after a fix raises a score. Optional TOOL args narrow the run.
+debian-oracle *ARGS:
+    scripts/debian-oracle/run.sh {{ARGS}}
+
 # Run benchmarks comparing bashkit to bash and save site-indexed JSON/Markdown results
 bench:
     cargo run -p bashkit-bench --release -- --save

knowledge/log.md

@@ -2,6 +2,7 @@
 
 ## 2026-10-07
 
+* **Test**: Debian-oracle scoreboard. `just debian-oracle` / `.github/workflows/debian-oracle.yml` run the pseudo-linus bench (about 9,600 cases recorded on real Debian, fetched at a pinned commit because parts of its corpus derive from GPL suites) and fail when a tool drops below `scripts/debian-oracle/floors.tsv`. See operations/testing "Debian-Oracle Scoreboard".
 * **Fix**: `grep` follows GNU grep 3.11. Prefixes print as `FILE:LINE:BYTE:` (context lines with `-`), group separators appear between non-adjacent groups and files whenever a context option is given (`-NUM`, `--group-separator`, `--no-group-separator`), trailing context after `-m` is printed in full, `-o` skips empty matches and is POSIX leftmost-longest (`grep -oE 'ab|abcd'` prints `abcd`), `-b -o` gives each match's offset. New `grep_pattern` translates GNU BRE/ERE (brackets, intervals, `\< \> \b \w`, literal leading `*` in BRE, dropped leading ERE operator with GNU's warning, glibc error messages) onto awk's leftmost-longest `AwkRegex`; back-references work in BRE and ERE via fancy-regex (TM-DOS-025 cap). Options parse like getopt_long (unique prefixes such as `--no-gr`, `-T`, `-d`, `--label`, `-` for stdin, `-R` follows symlinks), an empty `-f` file matches nothing, a directory operand without `-r` reports `Is a directory`, invalid UTF-8 output turns the file binary, `\r` is line data. Debian-oracle grep score 60.5% to 96.8% (185 cases); grep's probes in the regex set all pass except two raw-`\x01` patterns the shell drops. Remaining misses are readdir order (L-GREP-003); L-GREP-002 lists the other gaps. See foundations/builtins "grep".
 
 * **Fix**: jq `implode` maps invalid code points to U+FFFD, `strptime`/`fromdate` fail with jq's `date "X" does not match format "F"`, and `@base64d` accepts unpadded input and non-zero trailing bits with jq's error texts.

knowledge/operations/testing.md

@@ -315,6 +315,32 @@ Do not re-import over existing files; fix cases in place. If an intentional
 divergence is found, switch the case to `### bash_diff` with a canonical `L-*`
 row in [Known Limitations](limitations.md).
 
+## Debian-Oracle Scoreboard
+
+`scripts/debian-oracle/run.sh` (`just debian-oracle`) runs the
+[pseudo-linus](https://github.com/JoaoHenriqueBarbosa/pseudo-linus) test bench
+(MIT) against Bashkit: about 9,600 command cases across 45 tools (awk, jq,
+sed, grep, coreutils, ...) whose stdout, stderr, exit status and final file
+tree were recorded on real Debian. The adapter in
+`scripts/debian-oracle/adapter/` loads each fixture into the in-memory
+filesystem and compares with the bench's own `harness::compare_outcome`.
+
+- **Fetched, not vendored.** Parts of the bench corpus derive from GPL test
+  suites (GNU grep/sed), so the script clones it at a pinned commit
+  (`PSEUDO_LINUS_REV`) into `target/debian-oracle/` at run time. Bump the pin
+  deliberately and refresh floors in the same change.
+- **Floors.** `scripts/debian-oracle/floors.tsv` holds each tool's minimum
+  lenient pass count. The run fails when a tool drops more than one case below
+  its floor (one case of slack for wall-clock timeouts). After a fix raises a
+  score, run `scripts/debian-oracle/run.sh --update-floors` and commit the
+  floors so the gain is locked in.
+- **CI.** `.github/workflows/debian-oracle.yml` runs on PRs touching
+  builtins, the interpreter or the scoreboard, weekly, and on demand. The
+  table goes to the job summary; failing cases are uploaded as an artifact.
+- Goldens come from Debian's tool versions (jq 1.7.1, GNU grep 3.11, gawk);
+  version-specific gaps (Bashkit targets jq 1.8) stay as failures rather than
+  being special-cased.
+
 ## SQLite CLI Differential Tests
 
 CSV comparisons explicitly set the host CLI row separator to LF. SQLite

scripts/debian-oracle/adapter/Cargo.toml

@@ -0,0 +1,16 @@
+# Bashkit adapter for the pseudo-linus Debian-oracle test bench.
+#
+# Not a bashkit workspace member: `run.sh` copies this crate into a pinned
+# pseudo-linus checkout (testbench/crates/bk-oracle) and points the bashkit
+# dependency at this repository, because it links the bench's `harness` crate.
+[package]
+name = "bk-oracle"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[dependencies]
+harness = { workspace = true }
+anyhow = "1"
+tokio = { version = "1", features = ["rt", "time"] }
+bashkit = { path = "@BASHKIT@", features = ["jq", "git", "sqlite"] }

scripts/debian-oracle/adapter/src/main.rs

@@ -0,0 +1,205 @@
+//! Runs the pseudo-linus Debian-oracle cases against bashkit.
+//!
+//! Each case's fixture goes into bashkit's in-memory filesystem at
+//! `harness::CASE_DIR` (modes, symlinks, mtimes), the script runs with the
+//! bench environment, stdin and `ExecutionLimits::cli()`, and the outcome
+//! (stdout, stderr, exit, final tree) is compared with the Debian golden by
+//! the bench's own `harness::compare_outcome`.
+//!
+//! Output: one `tool<TAB>cases<TAB>strict<TAB>lenient` line per tool on
+//! stdout; failing case ids with a short diff go to `$FAILS_OUT`.
+//! Decision: each case runs on its own thread with a 10 s wall clock, so a
+//! hang or panic costs that case only.
+
+use std::path::{Path, PathBuf};
+use std::sync::mpsc;
+use std::time::{Duration, SystemTime};
+
+use bashkit::{Bash, ExecOptions, ExecutionLimits, FileSystem, FileType, GitConfig};
+use harness::{Bytes, Entry, Invocation, MemTree, Outcome};
+
+const CASE_TIMEOUT: Duration = Duration::from_secs(10);
+
+fn main() -> anyhow::Result<()> {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let tools = if args.is_empty() { harness::paths::tools()? } else { args };
+    let mut fails = Vec::new();
+    for tool in tools {
+        let (cases, _) = harness::paths::load_tool(&tool)?;
+        let (mut strict, mut lenient) = (0, 0);
+        for (case, golden) in &cases {
+            let Ok(inv) = case.invocation() else { continue };
+            let actual = run_isolated(inv);
+            let cmp = harness::compare_outcome(case, golden, &actual);
+            strict += usize::from(cmp.strict);
+            if cmp.lenient {
+                lenient += 1;
+            } else {
+                let detail: String = cmp.detail.join(" | ").chars().take(400).collect();
+                fails.push(format!("{tool}/{}: {detail}", cmp.id));
+            }
+        }
+        println!("{tool}\t{}\t{strict}\t{lenient}", cases.len());
+    }
+    let out = std::env::var("FAILS_OUT").unwrap_or_else(|_| "fails.txt".into());
+    std::fs::write(out, fails.join("\n") + "\n")?;
+    Ok(())
+}
+
+fn run_isolated(inv: Invocation) -> Outcome {
+    let (tx, rx) = mpsc::channel();
+    std::thread::Builder::new()
+        .stack_size(64 << 20)
+        .spawn(move || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("tokio runtime");
+            let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                rt.block_on(run_case(&inv))
+            }));
+            let _ = tx.send(res.unwrap_or_else(|_| Outcome::unsupported("panic")));
+        })
+        .expect("spawn case thread");
+    rx.recv_timeout(CASE_TIMEOUT).unwrap_or_else(|_| Outcome {
+        timed_out: true,
+        ..Outcome::unsupported("timeout")
+    })
+}
+
+async fn run_case(inv: &Invocation) -> Outcome {
+    let mut builder = Bash::builder()
+        .cwd(harness::CASE_DIR)
+        .limits(ExecutionLimits::cli())
+        .username("root")
+        .hostname("oracle")
+        .git(GitConfig::new())
+        .sqlite();
+    for (k, v) in inv.full_env() {
+        builder = builder.env(k, v);
+    }
+    if let Some(ft) = &inv.faketime {
+        match faketime_epoch(ft) {
+            Some(epoch) => builder = builder.fixed_epoch(epoch),
+            None => return Outcome::unsupported(format!("unsupported faketime: {ft}")),
+        }
+    }
+    let mut bash = builder.build();
+    let fs = bash.fs();
+    if let Err(e) = load_fixture(fs.as_ref(), &inv.files).await {
+        return Outcome::unsupported(format!("fixture: {e}"));
+    }
+    let script = match &inv.script {
+        Some(s) => s.clone(),
+        None => inv.argv.iter().map(|a| shell_quote(a)).collect::<Vec<_>>().join(" "),
+    };
+    let opts = ExecOptions::new().stdin(inv.stdin.clone());
+    let (stdout, stderr, exit) = match bash.exec_with_options(&script, opts).await {
+        Ok(r) => (r.stdout.into_bytes(), r.stderr.into_bytes(), r.exit_code),
+        // An API error (parse, limit) is the observable result of the case.
+        Err(e) => {
+            let code = if matches!(e, bashkit::Error::Parse { .. }) { 2 } else { 1 };
+            (Vec::new(), format!("bashkit: {e}\n").into_bytes(), code)
+        }
+    };
+    let files = match snapshot(fs.as_ref()).await {
+        Ok(t) => t,
+        Err(e) => return Outcome::unsupported(format!("snapshot: {e}")),
+    };
+    Outcome {
+        stdout: Bytes(stdout),
+        stderr: Bytes(stderr),
+        exit: Some(exit),
+        files,
+        ..Outcome::default()
+    }
+}
+
+async fn load_fixture(fs: &dyn FileSystem, tree: &MemTree) -> bashkit::Result<()> {
+    let root = PathBuf::from(harness::CASE_DIR);
+    let mtime = SystemTime::UNIX_EPOCH + Duration::from_secs(harness::FIXTURE_MTIME);
+    fs.mkdir(&root, true).await?;
+    // BTreeMap order: parents before children.
+    for (rel, entry) in &tree.entries {
+        let path = root.join(rel);
+        match entry {
+            Entry::Dir { .. } => fs.mkdir(&path, true).await?,
+            Entry::File { data, .. } => {
+                let bytes = data.as_ref().map(|d| d.0.clone()).unwrap_or_default();
+                fs.write_file(&path, &bytes).await?;
+            }
+            Entry::Symlink { target } => fs.symlink(Path::new(target), &path).await?,
+        }
+    }
+    // Modes and mtimes deepest first, so writing a child does not bump its parent.
+    for (rel, entry) in tree.entries.iter().rev() {
+        if let Entry::File { mode, .. } | Entry::Dir { mode } = entry {
+            let path = root.join(rel);
+            fs.chmod(&path, *mode).await?;
+            fs.set_modified_time(&path, mtime).await?;
+        }
+    }
+    fs.set_modified_time(&root, mtime).await
+}
+
+async fn snapshot(fs: &dyn FileSystem) -> bashkit::Result<MemTree> {
+    let root = PathBuf::from(harness::CASE_DIR);
+    let mut tree = MemTree::new();
+    if !fs.exists(&root).await? {
+        return Ok(tree);
+    }
+    let mut stack = vec![root.clone()];
+    while let Some(dir) = stack.pop() {
+        for item in fs.read_dir(&dir).await? {
+            let path = dir.join(&item.name);
+            let rel = path
+                .strip_prefix(&root)
+                .expect("inside case dir")
+                .to_string_lossy()
+                .into_owned();
+            let mode = item.metadata.mode & 0o7777;
+            let entry = match item.metadata.file_type {
+                FileType::Symlink => {
+                    Entry::symlink(fs.read_link(&path).await?.to_string_lossy().into_owned())
+                }
+                FileType::Directory => {
+                    stack.push(path);
+                    Entry::dir(mode)
+                }
+                FileType::File => Entry::file(fs.read_file(&path).await?, mode),
+                FileType::Fifo => Entry::file(Vec::new(), mode | 0o010000),
+            };
+            tree.entries.insert(rel, entry);
+        }
+    }
+    Ok(tree)
+}
+
+/// `faketime` spec ("YYYY-MM-DD HH:MM:SS" UTC, or "@epoch") to epoch seconds.
+fn faketime_epoch(spec: &str) -> Option<i64> {
+    let spec = spec.trim();
+    if let Some(rest) = spec.strip_prefix('@') {
+        return rest.parse().ok();
+    }
+    let (date, time) = spec.split_once(' ').unwrap_or((spec, "00:00:00"));
+    let d: Vec<i64> = date.split('-').map(|p| p.parse().ok()).collect::<Option<_>>()?;
+    let t: Vec<i64> = time.split(':').map(|p| p.parse().ok()).collect::<Option<_>>()?;
+    if d.len() != 3 || t.is_empty() || t.len() > 3 {
+        return None;
+    }
+    // Days since 1970-01-01, proleptic Gregorian (Howard Hinnant's algorithm).
+    let y = if d[1] <= 2 { d[0] - 1 } else { d[0] };
+    let era = if y >= 0 { y } else { y - 399 } / 400;
+    let yoe = y - era * 400;
+    let doy = (153 * ((d[1] + 9) % 12) + 2) / 5 + d[2] - 1;
+    let days = era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468;
+    let secs = t[0] * 3600 + t.get(1).copied().unwrap_or(0) * 60 + t.get(2).copied().unwrap_or(0);
+    Some(days * 86_400 + secs)
+}
+
+fn shell_quote(s: &str) -> String {
+    let plain = !s.is_empty()
+        && !s.starts_with('=')
+        && s.bytes().all(|b| b.is_ascii_alphanumeric() || b"_./=:,+@%-".contains(&b));
+    if plain { s.to_string() } else { format!("'{}'", s.replace('\'', r"'\''")) }
+}