From dd6bea0826bf359f817109ba7ecd44c3c95aa5d5 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 12 Oct 2026 09:19:25 +0000 Subject: [PATCH] fix(tests): cap bounded-child RSS instead of address space The `awk_field_depth_bounded` launcher capped the child's address space with `ulimit -v 2097152`. AddressSanitizer reserves roughly 14 TiB of *virtual* address space for its shadow map during startup, so the instrumented child died in `ReserveShadowMemoryRange` before executing a line of test code. The test then failed on the launcher's own cap rather than on the depth bound it asserts, which is why nightly run 257 went red while required CI, which is not instrumented, stayed green on the same commit -- the same divergence shape as the CLI-path bug in run 252. `awk_field_depth_tests.rs` was the only harness still using such a cap. Memory containment moves to a parent-side peak-RSS watchdog that samples the live child each poll, kills it on breach and asserts the peak afterwards, so a child that breached on its final poll and then exited on its own still fails closed. CPU, stack, core, output-drain and wall bounds are unchanged. RSS is also the better proxy for the runaway allocation the bound exists to catch and costs nothing in virtual space, so one mechanism holds on instrumented and native builds alike; it is the convention the other bounded-child harnesses already use. The replacement bound is not vacuous: the child's measured peak is 29,216 KiB over roughly four polls in a 45 ms lifetime, against a 2 GiB ceiling (the same figure as the old cap, now measured on resident rather than virtual memory), which an ASAN build fits while multi-gigabyte growth still trips it. A static scan now fails any launcher that reintroduces an address-space cap, in either the shell spellings or a direct `setrlimit`, with a `// rlimit-as-ok: ` hatch for one that provably never builds with a sanitizer. Verified by running nightly's own invocation (`-Z build-std`, `-Z sanitizer=address`) on this change: 3 passed, 0 failed, where `awk_field_depth_bounded` previously failed. The mechanism was reproduced standalone with a 3-line ASAN binary, which fails under the old cap with the byte-identical `0xdfff0001000` reservation from the nightly log and exits 0 without it. Reintroducing the cap fails the new scan. Claude-Session: https://claude.ai/code/session_013mA6kgZvfp234w1L2gLint --- .../integration/awk_field_depth_tests.rs | 71 +++++++++++++-- .../bounded_child_launcher_scan_tests.rs | 89 +++++++++++++++++++ crates/bashkit/tests/integration/main.rs | 1 + knowledge/operations/testing.md | 30 +++++++ 4 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs diff --git a/crates/bashkit/tests/integration/awk_field_depth_tests.rs b/crates/bashkit/tests/integration/awk_field_depth_tests.rs index 7b31a9ec8..f731eb1ef 100644 --- a/crates/bashkit/tests/integration/awk_field_depth_tests.rs +++ b/crates/bashkit/tests/integration/awk_field_depth_tests.rs @@ -1,5 +1,18 @@ //! THREAT[TM-DOS-027]: field-operand recursion must share AWK depth accounting. //! Run potentially aborting programs only in a supervised child test process. +//! +//! The child is bounded by OS CPU/stack/core limits plus a parent-side wall +//! watchdog, output drain cap and peak-RSS watchdog. Memory containment is +//! deliberately RSS-based rather than an address-space cap (`ulimit -v`): +//! AddressSanitizer reserves ~14 TiB of *virtual* address space for its +//! shadow map at startup, so any `RLIMIT_AS` cap makes a sanitizer- +//! instrumented child die in `ReserveShadowMemoryRange` before it runs a +//! line of test code. That is what reddened the nightly ASAN job on run 257 +//! while required CI, which is not instrumented, stayed green on the same +//! commit. RSS is also the better proxy for the runaway *allocation* this +//! bound exists to catch, and it is the convention the other bounded-child +//! harnesses already use (`BOUNDED_CLI_LIMITS` in `threat_model_tests`, +//! the probe harness in `bashkit-cli/tests/cli_oneshot.rs`). use std::io::Read; use std::path::Path; @@ -32,15 +45,40 @@ fn drain( }) } +/// Peak-RSS ceiling for the bounded child, enforced by the parent watchdog. +/// +/// Generous enough for an AddressSanitizer build of the whole integration +/// binary (instrumentation plus redzones cost several times the native +/// footprint) and still orders of magnitude below the multi-gigabyte growth +/// an unbounded field-operand recursion would reach, so the bound stays +/// discriminating without being toolchain-specific. +#[cfg(unix)] +const RSS_CAP_KB: u64 = 2 * 1024 * 1024; + +/// Best-effort peak-RSS readback for a live child via unix `ps`. +/// +/// `None` only means this poll found no live PID; the CPU/stack/wall/output +/// bounds hold regardless of whether the numeric readback is available. +#[cfg(unix)] +fn sample_rss_kb(pid: u32) -> Option { + let out = Command::new("ps") + .args(["-o", "rss=", "-p", &pid.to_string()]) + .output() + .ok()?; + String::from_utf8(out.stdout) + .ok()? + .trim() + .parse::() + .ok() +} + #[test] #[cfg(unix)] fn awk_field_depth_bounded() { - // OS CPU/stack/core limits, Linux address-space cap, continuous output - // drains and a wall watchdog also contain a regression that aborts/hangs. - #[cfg(target_os = "linux")] - let limits = - "ulimit -t 10 && ulimit -s 8192 && ulimit -c 0 && ulimit -v 2097152 && exec \"$@\""; - #[cfg(not(target_os = "linux"))] + // OS CPU/stack/core limits, continuous output drains, a wall watchdog and + // the RSS watchdog below also contain a regression that aborts/hangs. No + // address-space cap: see the module header -- it would kill an ASAN child + // at startup. Growth is bounded by RSS_CAP_KB instead. let limits = "ulimit -t 10 && ulimit -s 8192 && ulimit -c 0 && exec \"$@\""; let mut command = Command::new("/bin/sh"); command.args(["-c", limits, "awk-depth"]); @@ -58,15 +96,23 @@ fn awk_field_depth_bounded() { .spawn() .expect("spawn bounded child"); let exceeded = Arc::new(AtomicBool::new(false)); + let child_id = child.id(); let stdout = drain(child.stdout.take().unwrap(), exceeded.clone()); let stderr = drain(child.stderr.take().unwrap(), exceeded.clone()); let start = Instant::now(); let mut stopped = false; + let mut max_rss_kb: Option = None; let status = loop { if let Some(status) = child.try_wait().unwrap() { break status; } - if exceeded.load(Ordering::Relaxed) || start.elapsed() > Duration::from_secs(20) { + // The child has not been reaped yet, so it is alive for this readback. + if let Some(rss) = sample_rss_kb(child_id) { + max_rss_kb = Some(max_rss_kb.unwrap_or(0).max(rss)); + } + let over_rss = max_rss_kb.is_some_and(|rss| rss > RSS_CAP_KB); + if exceeded.load(Ordering::Relaxed) || over_rss || start.elapsed() > Duration::from_secs(20) + { stopped = true; child.kill().unwrap(); break child.wait().unwrap(); @@ -77,7 +123,16 @@ fn awk_field_depth_bounded() { let stderr = stderr.join().unwrap(); assert!( !stopped && !exceeded.load(Ordering::Relaxed), - "child exceeded bounds" + "child exceeded bounds (peak RSS {} KiB, cap {RSS_CAP_KB} KiB)", + max_rss_kb.map_or("unavailable".to_owned(), |rss| rss.to_string()) + ); + // The loop leaves via `try_wait` before it re-checks the cap, so a child + // that breached on its last poll and then exited on its own reaches here + // with `stopped` false. Assert the peak so that still fails closed. + assert!( + max_rss_kb.is_none_or(|rss| rss <= RSS_CAP_KB), + "peak child RSS {} KiB exceeds {RSS_CAP_KB} KiB bound", + max_rss_kb.unwrap_or_default() ); assert!( status.success(), diff --git a/crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs b/crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs new file mode 100644 index 000000000..a663904f0 --- /dev/null +++ b/crates/bashkit/tests/integration/bounded_child_launcher_scan_tests.rs @@ -0,0 +1,89 @@ +// Regression guard for supervised-child test harnesses under sanitizers. +// +// A bounded-child launcher caps CPU (`ulimit -t`) and stack (`ulimit -s`) and +// leaves memory to the parent-side peak-RSS watchdog. An address-space cap +// (`ulimit -v`, RLIMIT_AS) must never appear: AddressSanitizer reserves ~14 TiB +// of virtual address space for its shadow map at startup, so an instrumented +// child dies in `ReserveShadowMemoryRange` before running a line of test code. +// The child then fails on the launcher's cap rather than the bound under test, +// which is how nightly run 257 went red on `awk_field_depth_bounded` while +// required CI, which is not instrumented, stayed green on the same commit. +// +// See knowledge/operations/testing.md, "Bounded-child launchers cap CPU and +// stack, never address space". +// +// Escape hatch: end the line with `// rlimit-as-ok: ` for a launcher +// that is provably never built with a sanitizer and has a real reason to cap +// address space. + +use std::path::{Path, PathBuf}; + +fn crates_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate must live under repo/crates") + .to_path_buf() +} + +fn rust_files(dir: &Path, out: &mut Vec) { + let entries = + std::fs::read_dir(dir).unwrap_or_else(|e| panic!("read_dir {}: {e}", dir.display())); + for entry in entries { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + // `target` directories hold build output, not sources to police. + if path.file_name().is_some_and(|n| n == "target") { + continue; + } + rust_files(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } +} + +/// No test launcher caps the child's address space. +#[test] +fn bounded_child_launchers_never_cap_address_space() { + let mut files = Vec::new(); + rust_files(&crates_dir(), &mut files); + // Fail closed: a scan that found nothing proves nothing. + assert!( + files + .iter() + .any(|f| f.ends_with("integration/awk_field_depth_tests.rs")), + "scan did not reach the known bounded-child harnesses" + ); + + let mut offenders = Vec::new(); + for file in &files { + if file.ends_with("integration/bounded_child_launcher_scan_tests.rs") { + continue; + } + let source = std::fs::read_to_string(file) + .unwrap_or_else(|e| panic!("read {}: {e}", file.display())); + for (number, line) in source.lines().enumerate() { + let code = line.trim(); + // Prose explaining the rule names the caps it bans, so comment + // lines are described, not enforced. + if code.starts_with("//") || code.contains("// rlimit-as-ok:") { + continue; + } + // Collapse runs of whitespace so `ulimit -v` cannot slip past. + let flat = code.split_whitespace().collect::>().join(" "); + // `-d`/`RLIMIT_DATA` cap the data segment and break ASAN the same + // way; the `RLIMIT_*` forms catch a direct `setrlimit` in Rust, + // which the shell spellings alone would miss. + let banned = ["ulimit -v", "ulimit -d", "RLIMIT_AS", "RLIMIT_DATA"]; + if banned.iter().any(|pattern| flat.contains(pattern)) { + offenders.push(format!("{}:{}: {code}", file.display(), number + 1)); + } + } + } + assert!( + offenders.is_empty(), + "address-space caps break sanitizer-instrumented children; cap RSS from \ + the parent instead (knowledge/operations/testing.md):\n{}", + offenders.join("\n") + ); +} diff --git a/crates/bashkit/tests/integration/main.rs b/crates/bashkit/tests/integration/main.rs index 3223e404a..cc0bb3643 100644 --- a/crates/bashkit/tests/integration/main.rs +++ b/crates/bashkit/tests/integration/main.rs @@ -30,6 +30,7 @@ pub mod backtick_quote_tests; pub mod base64_binary_tests; pub mod bash_source_tests; pub mod blackbox_security_tests; +pub mod bounded_child_launcher_scan_tests; pub mod builtin_cap_security_tests; pub mod builtin_error_security_tests; pub mod builtin_fs_error_tests; diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 52cd23a9b..44ff6e173 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -272,6 +272,36 @@ Only a test that asserts a timeout sets a short deadline, and it sets it explicitly. Before adding a cap test, confirm the cap is the sole limit that can stop the script. +## Bounded-child launchers cap CPU and stack, never address space + +A supervised-child harness bounds its child with `ulimit -t` (CPU) and +`ulimit -s` (stack), plus parent-side wall, output-drain and peak-RSS +watchdogs. It must **not** set an address-space cap (`ulimit -v`, +`RLIMIT_AS`). + +AddressSanitizer reserves roughly 14 TiB of *virtual* address space for its +shadow map during startup, so any `RLIMIT_AS` makes an instrumented child die +in `ReserveShadowMemoryRange` before it executes a line of test code. The +child's failure is then the launcher's cap, not the bound under test, and the +test fails for a reason unrelated to what it asserts. This reddened the +nightly ASAN job on run 257 (`awk_field_depth_bounded`, the only harness that +still used `ulimit -v`) while required CI, which is not instrumented, stayed +green on the same commit -- the same divergence shape as the CLI-path bug in +run 252. + +Cap resident memory instead: sample the live child's RSS from the parent +(`ps -o rss=`) each poll, keep the maximum, kill on breach and assert the +peak afterwards so growth fails closed even when the child exits between +polls. RSS is also the better proxy for the runaway *allocation* such a bound +exists to catch, and it costs nothing in virtual space, so it holds on +instrumented and native builds alike. Size the ceiling from a measured native +peak with room for instrumentation: `awk_field_depth_bounded` peaks near +29 MiB natively against a 2 GiB ceiling, which an ASAN build fits comfortably +while multi-gigabyte growth still trips it. + +`BOUNDED_CLI_LIMITS` in `threat_model_tests.rs` and the probe harness in +`bashkit-cli/tests/cli_oneshot.rs` are the reference launchers. + The same file holds one test whose guard is **throughput**, not a cap (`ordinary_workloads_still_fit`), and it has the opposite constraint: its budget has to sit *between* the linear cost and the quadratic one on the