diff --git a/crates/bashkit/tests/integration/awk_field_depth_tests.rs b/crates/bashkit/tests/integration/awk_field_depth_tests.rs index 7b31a9ec8..f731eb1ef 100644 --- a/crates/bashkit/tests/integration/awk_field_depth_tests.rs +++ b/crates/bashkit/tests/integration/awk_field_depth_tests.rs @@ -1,5 +1,18 @@ //! THREAT[TM-DOS-027]: field-operand recursion must share AWK depth accounting. //! Run potentially aborting programs only in a supervised child test process. +//! +//! The child is bounded by OS CPU/stack/core limits plus a parent-side wall +//! watchdog, output drain cap and peak-RSS watchdog. Memory containment is +//! deliberately RSS-based rather than an address-space cap (`ulimit -v`): +//! AddressSanitizer reserves ~14 TiB of *virtual* address space for its +//! shadow map at startup, so any `RLIMIT_AS` cap makes a sanitizer- +//! instrumented child die in `ReserveShadowMemoryRange` before it runs a +//! line of test code. That is what reddened the nightly ASAN job on run 257 +//! while required CI, which is not instrumented, stayed green on the same +//! commit. RSS is also the better proxy for the runaway *allocation* this +//! bound exists to catch, and it is the convention the other bounded-child +//! harnesses already use (`BOUNDED_CLI_LIMITS` in `threat_model_tests`, +//! the probe harness in `bashkit-cli/tests/cli_oneshot.rs`). use std::io::Read; use std::path::Path; @@ -32,15 +45,40 @@ fn drain( }) } +/// Peak-RSS ceiling for the bounded child, enforced by the parent watchdog. +/// +/// Generous enough for an AddressSanitizer build of the whole integration +/// binary (instrumentation plus redzones cost several times the native +/// footprint) and still orders of magnitude below the multi-gigabyte growth +/// an unbounded field-operand recursion would reach, so the bound stays +/// discriminating without being toolchain-specific. +#[cfg(unix)] +const RSS_CAP_KB: u64 = 2 * 1024 * 1024; + +/// Best-effort peak-RSS readback for a live child via unix `ps`. +/// +/// `None` only means this poll found no live PID; the CPU/stack/wall/output +/// bounds hold regardless of whether the numeric readback is available. +#[cfg(unix)] +fn sample_rss_kb(pid: u32) -> Option { + let out = Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output() + .ok()?; + String::from_utf8(out.stdout) + .ok()? + .trim() + .parse::() + .ok() +} + #[test] #[cfg(unix)] fn awk_field_depth_bounded() { - // OS CPU/stack/core limits, Linux address-space cap, continuous output - // drains and a wall watchdog also contain a regression that aborts/hangs. - #[cfg(target_os = "linux")] - let limits = - "ulimit -t 10 && ulimit -s 8192 && ulimit -c 0 && ulimit -v 2097152 && exec \"$@\""; - #[cfg(not(target_os = "linux"))] + // OS CPU/stack/core limits, continuous output drains, a wall watchdog and + // the RSS watchdog below also contain a regression that aborts/hangs. No + // address-space cap: see the module header -- it would kill an ASAN child + // at startup. Growth is bounded by RSS_CAP_KB instead. let limits = "ulimit -t 10 && ulimit -s 8192 && ulimit -c 0 && exec \"$@\""; let mut command = Command::new("/bin/sh"); command.args(["-c", limits, "awk-depth"]); @@ -58,15 +96,23 @@ fn awk_field_depth_bounded() { .spawn() .expect("spawn bounded child"); let exceeded = Arc::new(AtomicBool::new(false)); + let child_id = child.id(); let stdout = drain(child.stdout.take().unwrap(), exceeded.clone()); let stderr = drain(child.stderr.take().unwrap(), exceeded.clone()); let start = Instant::now(); let mut stopped = false; + let mut max_rss_kb: Option = None; let status = loop { if let Some(status) = child.try_wait().unwrap() { break status; } - if exceeded.load(Ordering::Relaxed) || start.elapsed() > Duration::from_secs(20) { + // The child has not been reaped yet, so it is alive for this readback. + if let Some(rss) = sample_rss_kb(child_id) { + max_rss_kb = Some(max_rss_kb.unwrap_or(0).max(rss)); + } + let over_rss = max_rss_kb.is_some_and(|rss| rss > RSS_CAP_KB); + if exceeded.load(Ordering::Relaxed) || over_rss || start.elapsed() > Duration::from_secs(20) + { stopped = true; child.kill().unwrap(); break child.wait().unwrap(); @@ -77,7 +123,16 @@ fn awk_field_depth_bounded() { let stderr = stderr.join().unwrap(); assert!( !stopped && !exceeded.load(Ordering::Relaxed), - "child exceeded bounds" + "child exceeded bounds (peak RSS {} KiB, cap {RSS_CAP_KB} KiB)", + max_rss_kb.map_or("unavailable".to_owned(), |rss| rss.to_string()) + ); + // The loop leaves via `try_wait` before it re-checks the cap, so a child + // that breached on its last poll and then exited on its own reaches here + // with `stopped` false. Assert the peak so that still fails closed. + assert!( + max_rss_kb.is_none_or(|rss| rss <= RSS_CAP_KB), + "peak child RSS {} KiB exceeds {RSS_CAP_KB} KiB bound", + max_rss_kb.unwrap_or_default() ); assert!( status.success(), diff --git a/crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs b/crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs new file mode 100644 index 000000000..a663904f0 --- /dev/null +++ b/crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs @@ -0,0 +1,89 @@ +// Regression guard for supervised-child test harnesses under sanitizers. +// +// A bounded-child launcher caps CPU (`ulimit -t`) and stack (`ulimit -s`) and +// leaves memory to the parent-side peak-RSS watchdog. An address-space cap +// (`ulimit -v`, RLIMIT_AS) must never appear: AddressSanitizer reserves ~14 TiB +// of virtual address space for its shadow map at startup, so an instrumented +// child dies in `ReserveShadowMemoryRange` before running a line of test code. +// The child then fails on the launcher's cap rather than the bound under test, +// which is how nightly run 257 went red on `awk_field_depth_bounded` while +// required CI, which is not instrumented, stayed green on the same commit. +// +// See knowledge/operations/testing.md, "Bounded-child launchers cap CPU and +// stack, never address space". +// +// Escape hatch: end the line with `// rlimit-as-ok: ` for a launcher +// that is provably never built with a sanitizer and has a real reason to cap +// address space. + +use std::path::{Path, PathBuf}; + +fn crates_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate must live under repo/crates") + .to_path_buf() +} + +fn rust_files(dir: &Path, out: &mut Vec) { + let entries = + std::fs::read_dir(dir).unwrap_or_else(|e| panic!("read_dir {}: {e}", dir.display())); + for entry in entries { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + // `target` directories hold build output, not sources to police. + if path.file_name().is_some_and(|n| n == "target") { + continue; + } + rust_files(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } +} + +/// No test launcher caps the child's address space. +#[test] +fn bounded_child_launchers_never_cap_address_space() { + let mut files = Vec::new(); + rust_files(&crates_dir(), &mut files); + // Fail closed: a scan that found nothing proves nothing. + assert!( + files + .iter() + .any(|f| f.ends_with("integration/awk_field_depth_tests.rs")), + "scan did not reach the known bounded-child harnesses" + ); + + let mut offenders = Vec::new(); + for file in &files { + if file.ends_with("integration/bounded_child_launcher_scan_tests.rs") { + continue; + } + let source = std::fs::read_to_string(file) + .unwrap_or_else(|e| panic!("read {}: {e}", file.display())); + for (number, line) in source.lines().enumerate() { + let code = line.trim(); + // Prose explaining the rule names the caps it bans, so comment + // lines are described, not enforced. + if code.starts_with("//") || code.contains("// rlimit-as-ok:") { + continue; + } + // Collapse runs of whitespace so `ulimit -v` cannot slip past. + let flat = code.split_whitespace().collect::>().join(" "); + // `-d`/`RLIMIT_DATA` cap the data segment and break ASAN the same + // way; the `RLIMIT_*` forms catch a direct `setrlimit` in Rust, + // which the shell spellings alone would miss. + let banned = ["ulimit -v", "ulimit -d", "RLIMIT_AS", "RLIMIT_DATA"]; + if banned.iter().any(|pattern| flat.contains(pattern)) { + offenders.push(format!("{}:{}: {code}", file.display(), number + 1)); + } + } + } + assert!( + offenders.is_empty(), + "address-space caps break sanitizer-instrumented children; cap RSS from \ + the parent instead (knowledge/operations/testing.md):\n{}", + offenders.join("\n") + ); +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 3223e404a..cc0bb3643 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -30,6 +30,7 @@ pub mod backtick_quote_tests; pub mod base64_binary_tests; pub mod bash_source_tests; pub mod blackbox_security_tests; +pub mod bounded_child_launcher_scan_tests; pub mod builtin_cap_security_tests; pub mod builtin_error_security_tests; pub mod builtin_fs_error_tests; diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 52cd23a9b..44ff6e173 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -272,6 +272,36 @@ Only a test that asserts a timeout sets a short deadline, and it sets it explicitly. Before adding a cap test, confirm the cap is the sole limit that can stop the script. +## Bounded-child launchers cap CPU and stack, never address space + +A supervised-child harness bounds its child with `ulimit -t` (CPU) and +`ulimit -s` (stack), plus parent-side wall, output-drain and peak-RSS +watchdogs. It must **not** set an address-space cap (`ulimit -v`, +`RLIMIT_AS`). + +AddressSanitizer reserves roughly 14 TiB of *virtual* address space for its +shadow map during startup, so any `RLIMIT_AS` makes an instrumented child die +in `ReserveShadowMemoryRange` before it executes a line of test code. The +child's failure is then the launcher's cap, not the bound under test, and the +test fails for a reason unrelated to what it asserts. This reddened the +nightly ASAN job on run 257 (`awk_field_depth_bounded`, the only harness that +still used `ulimit -v`) while required CI, which is not instrumented, stayed +green on the same commit -- the same divergence shape as the CLI-path bug in +run 252. + +Cap resident memory instead: sample the live child's RSS from the parent +(`ps -o rss=`) each poll, keep the maximum, kill on breach and assert the +peak afterwards so growth fails closed even when the child exits between +polls. RSS is also the better proxy for the runaway *allocation* such a bound +exists to catch, and it costs nothing in virtual space, so it holds on +instrumented and native builds alike. Size the ceiling from a measured native +peak with room for instrumentation: `awk_field_depth_bounded` peaks near +29 MiB natively against a 2 GiB ceiling, which an ASAN build fits comfortably +while multi-gigabyte growth still trips it. + +`BOUNDED_CLI_LIMITS` in `threat_model_tests.rs` and the probe harness in +`bashkit-cli/tests/cli_oneshot.rs` are the reference launchers. + The same file holds one test whose guard is **throughput**, not a cap (`ordinary_workloads_still_fit`), and it has the opposite constraint: its budget has to sit *between* the linear cost and the quadratic one on the