From 8302ed8f82628b1581cc4e6fa1d2727b80e909e2 Mon Sep 17 00:00:00 2001 From: moha-bekh Date: Mon, 19 Oct 2026 00:40:13 +0200 Subject: [PATCH 1/2] Add file_caps, collect and DP_TARGET; envprobe reports the function tracer and RLIMIT_MEMLOCK What porting Omniscient with distroproof needed: - file_caps = { "/path" = [caps] } grants file capabilities after setup, as a package's install does; caps raises ambient ones, which every program the command starts inherits. A kernel before a capability is a warning naming cap_last_cap. - collect = [patterns] brings files back to .distroproof/collect// and lists them in the report: a 1 MB document no longer has to travel through the serial console as standard output. - DP_TARGET and DP_IMAGE are exported to setup and the command. And what its matrix showed envprobe could not see: CONFIG_FUNCTION_TRACER and the dynamic-ftrace options (Alpine's virt kernel has CONFIG_FTRACE=y and no function tracer, so every fentry attach is refused), and the soft and hard RLIMIT_MEMLOCK (before 5.11 every eBPF map is charged to it, and Debian's 64 KiB refuses a ring buffer with EPERM). Co-Authored-By: Claude Opus 5.5 --- README.md | 12 ++++++ crates/distroproof/src/agent.sh | 43 ++++++++++++++++++- crates/distroproof/src/config.rs | 41 ++++++++++++++++++ crates/distroproof/src/guest.rs | 22 ++++++++++ crates/distroproof/src/main.rs | 2 + crates/distroproof/src/model.rs | 12 ++++++ crates/distroproof/src/report.rs | 2 + crates/distroproof/src/tui/mod.rs | 2 + crates/distroproof/src/vm.rs | 24 +++++++++++ crates/envprobe-core/src/parse/limits.rs | 34 +++++++++++++++ crates/envprobe-core/src/parse/mod.rs | 1 + crates/envprobe-core/src/probes/kernel.rs | 6 +++ crates/envprobe-core/src/probes/privileges.rs | 22 ++++++++++ 13 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 crates/envprobe-core/src/parse/limits.rs diff --git a/README.md b/README.md index 53f103b..118960f 100644 --- a/README.md +++ b/README.md @@ -195,8 +195,20 @@ command = "./target/release/mytool --selftest" as_user = "test" # or "root" # exclude = [".git", "target"] # copied to /distroproof/work # include = ["target/release/mytool"] +# file_caps = { "/opt/mytool/bin/mytool" = ["bpf", "perfmon"] } # setcap after setup +# collect = ["results/*.json"] # brought back to .distroproof/collect// ``` +`setup` and `command` see `DP_TARGET` (the target's name) and `DP_IMAGE`. +`file_caps` grants file capabilities on an installed binary, as root, after +`setup` -- what a package's install does -- where `caps` raises ambient +capabilities on the user, which every program the command starts inherits; +a kernel that predates a capability (before 5.8 for `bpf` and `perfmon`) is +a warning naming `cap_last_cap`. `collect` copies the files matching each +pattern, relative to the working directory, back to +`.distroproof/collect//` after the command, and lists them in the +report's `collected`. + Errors (unknown key, nonexistent image, invalid duration…) are reported with their line and column, without starting any VM. `distroproof up` runs the targets in parallel (`--jobs`), attaches to each result the `envprobe` report diff --git a/crates/distroproof/src/agent.sh b/crates/distroproof/src/agent.sh index 6eb89a2..6643f93 100644 --- a/crates/distroproof/src/agent.sh +++ b/crates/distroproof/src/agent.sh @@ -28,9 +28,11 @@ warn() { say WARN "$*"; echo "$*" >>"$OUT/warnings"; } mkdir -p "$OUT" say READY "$(cut -d' ' -f1 /proc/uptime)" -# Job parameters: DP_USER, DP_CAPS, DP_PROBE, DP_EXTRACT_KERNEL, -# DP_SETUP, DP_COMMAND (1 if job/setup.sh, job/command.sh exist). +# Job parameters: DP_USER, DP_TARGET, DP_IMAGE, DP_CAPS, DP_PROBE, +# DP_EXTRACT_KERNEL, DP_SETUP, DP_COMMAND (1 if job/setup.sh, job/command.sh +# exist). DP_TARGET and DP_IMAGE are exported to setup and the command. . $D/job/env +export DP_TARGET DP_IMAGE disk() { for d in /sys/block/*; do @@ -119,6 +121,27 @@ fi [ -n "$DP_SETUP" ] && run setup "sh $D/job/setup.sh" +# File capabilities, after setup installed the files: what a program's install +# grants, where `caps` would raise ambient ones every child inherits. +if [ -f $D/job/filecaps ]; then + if ! command -v setcap >/dev/null 2>&1; then + if command -v apk >/dev/null 2>&1; then apk add -q libcap-utils >/dev/null 2>&1 + elif command -v apt-get >/dev/null 2>&1; then apt-get install -qq -y libcap2-bin >/dev/null 2>&1 + elif command -v dnf >/dev/null 2>&1; then dnf install -q -y libcap >/dev/null 2>&1 + elif command -v pacman >/dev/null 2>&1; then pacman -Sq --noconfirm libcap >/dev/null 2>&1 + fi + fi + last=$(cat /proc/sys/kernel/cap_last_cap 2>/dev/null || echo 0) + while IFS="$(printf '\t')" read -r path caps; do + [ -n "$path" ] || continue + if ! command -v setcap >/dev/null 2>&1; then + warn "file_caps: no setcap in this image, $path keeps no capability" + elif ! setcap "$caps" "$path" 2>"$OUT/filecaps.err"; then + warn "file_caps: setcap $caps $path refused (cap_last_cap $last): $(head -c 200 "$OUT/filecaps.err")" + fi + done <$D/job/filecaps +fi + if [ "$DP_PROBE" = 1 ]; then say STAGE probe sh -c "$(as_user $D/job/probe.sh)" >"$OUT/envprobe.json" 2>"$OUT/envprobe.err" /dev/null | tar -xf - -C "$OUT/collect" 2>/dev/null || + warn "collect: cannot copy $f" + done + [ -n "$found" ] || warn "collect: nothing matches $pattern" + done <$D/job/collect +fi + # Kernel and initramfs for the following direct boots. if [ "$DP_EXTRACT_KERNEL" = 1 ]; then say STAGE kernel diff --git a/crates/distroproof/src/config.rs b/crates/distroproof/src/config.rs index b4e4155..84b542f 100644 --- a/crates/distroproof/src/config.rs +++ b/crates/distroproof/src/config.rs @@ -105,6 +105,10 @@ struct RawRun { probe: Option, #[serde(default)] env: BTreeMap, String>, + #[serde(default)] + file_caps: BTreeMap>>, + #[serde(default)] + collect: Vec, } #[derive(Debug, Deserialize)] @@ -359,6 +363,20 @@ pub fn parse(path: &Path, text: &str) -> Result { } None => Vec::new(), }; + let mut file_caps = BTreeMap::new(); + for (path, caps) in &run.file_caps { + if !path.starts_with('/') { + return Err(ctx.error( + caps.span(), + format!("file_caps: `{path}` is not an absolute path"), + )); + } + check_caps(caps, &ctx)?; + file_caps.insert( + path.clone(), + caps.get_ref().iter().map(|c| normalize_cap(c)).collect(), + ); + } targets.push(VmSpec { name: name.clone(), image: t.image.get_ref().clone(), @@ -381,6 +399,8 @@ pub fn parse(path: &Path, text: &str) -> Result { include: run.include.clone(), probe, hold: false, + file_caps, + collect: run.collect.clone(), }); } Ok(Config { targets }) @@ -616,6 +636,27 @@ as_user = "test" ); } + #[test] + fn file_caps_and_collect() { + let base = "[[target]]\nname = \"a\"\nimage = \"alpine\"\n"; + let ok = format!( + "{base}[run]\ncommand = \"true\"\ncollect = [\"out/*.json\"]\n\ + file_caps = {{ \"/opt/omn/bin/omn\" = [\"CAP_BPF\", \"perfmon\"] }}\n" + ); + let spec = &parse_str(&ok).unwrap().targets[0]; + assert_eq!(spec.collect, ["out/*.json"]); + assert_eq!(spec.file_caps["/opt/omn/bin/omn"], ["bpf", "perfmon"]); + let relative = + format!("{base}[run]\ncommand = \"true\"\nfile_caps = {{ \"omn\" = [\"bpf\"] }}\n"); + assert!(parse_str(&relative) + .unwrap_err() + .message + .contains("not an absolute path")); + let bad = + format!("{base}[run]\ncommand = \"true\"\nfile_caps = {{ \"/x\" = [\"fly\"] }}\n"); + assert!(parse_str(&bad).unwrap_err().message.contains("fly")); + } + #[test] fn sizes_and_durations() { assert_eq!(parse_size_mib("1G").unwrap(), 1024); diff --git a/crates/distroproof/src/guest.rs b/crates/distroproof/src/guest.rs index 0296b7f..40a2260 100644 --- a/crates/distroproof/src/guest.rs +++ b/crates/distroproof/src/guest.rs @@ -73,6 +73,8 @@ impl Payload<'_> { let mut env = String::new(); let mut var = |k: &str, v: &str| env.push_str(&format!("{k}={}\n", shell_quote(v))); var("DP_USER", &spec.user); + var("DP_TARGET", &spec.name); + var("DP_IMAGE", &spec.image); var("DP_CAPS", &spec.caps.join(",")); var("DP_PROBE", if run_steps && spec.probe { "1" } else { "0" }); var( @@ -100,6 +102,21 @@ impl Payload<'_> { if let Some(command) = command { files.push(("job/command.sh".into(), script(command), 0o755)); } + if run_steps && !spec.file_caps.is_empty() { + let lines: String = spec + .file_caps + .iter() + .map(|(path, caps)| { + let caps: Vec = caps.iter().map(|c| format!("cap_{c}")).collect(); + format!("{}\t{}+ep\n", path, caps.join(",")) + }) + .collect(); + files.push(("job/filecaps".into(), lines.into_bytes(), 0o644)); + } + if run_steps && !spec.collect.is_empty() { + let lines: String = spec.collect.iter().map(|p| format!("{p}\n")).collect(); + files.push(("job/collect".into(), lines.into_bytes(), 0o644)); + } if !spec.sysctl.is_empty() { let lines: String = spec .sysctl @@ -235,6 +252,11 @@ impl GuestOutput { Ok(GuestOutput { files }) } + /// Every file of the output disk, by name. + pub fn files(&self) -> impl Iterator { + self.files.iter().map(|(n, d)| (n.as_str(), d.as_slice())) + } + pub fn get(&self, name: &str) -> Option<&[u8]> { self.files .iter() diff --git a/crates/distroproof/src/main.rs b/crates/distroproof/src/main.rs index 389c0ab..668981f 100644 --- a/crates/distroproof/src/main.rs +++ b/crates/distroproof/src/main.rs @@ -304,6 +304,8 @@ async fn cmd_run(args: RunArgs) -> Result { include: Vec::new(), probe: args.probe, hold: false, + file_caps: Default::default(), + collect: Vec::new(), }; let runner = runner(&args.common)?; let (tx, rx) = mpsc::unbounded_channel(); diff --git a/crates/distroproof/src/model.rs b/crates/distroproof/src/model.rs index 917f9e2..718f219 100644 --- a/crates/distroproof/src/model.rs +++ b/crates/distroproof/src/model.rs @@ -40,6 +40,14 @@ pub struct VmSpec { /// Keep the VM up after the command, with a root shell on the serial /// console, until it is killed. pub hold: bool, + /// Files whose capabilities are set as root after `setup`, by path: + /// `setcap cap_,cap_+ep `. A file capability is what a + /// program's install grants; `caps` raises ambient ones on the user, which + /// every program the command starts inherits. + pub file_caps: BTreeMap>, + /// Paths, relative to the working directory, copied back after the + /// command into `/.distroproof/collect//`. + pub collect: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -121,6 +129,9 @@ pub struct TargetResult { /// Last lines of the serial console, useful when the boot fails. #[serde(default)] pub console_tail: String, + /// Files brought back by `collect`, as written on the host. + #[serde(default)] + pub collected: Vec, } impl TargetResult { @@ -146,6 +157,7 @@ impl TargetResult { warnings: Vec::new(), error: None, console_tail: String::new(), + collected: Vec::new(), } } diff --git a/crates/distroproof/src/report.rs b/crates/distroproof/src/report.rs index 56f87cd..8d1b0f3 100644 --- a/crates/distroproof/src/report.rs +++ b/crates/distroproof/src/report.rs @@ -752,6 +752,8 @@ mod tests { include: Vec::new(), probe: true, hold: false, + file_caps: Default::default(), + collect: Vec::new(), } } diff --git a/crates/distroproof/src/tui/mod.rs b/crates/distroproof/src/tui/mod.rs index 1a8095d..c6359d7 100644 --- a/crates/distroproof/src/tui/mod.rs +++ b/crates/distroproof/src/tui/mod.rs @@ -130,6 +130,8 @@ mod tests { include: Vec::new(), probe: true, hold: false, + file_caps: BTreeMap::new(), + collect: Vec::new(), } } diff --git a/crates/distroproof/src/vm.rs b/crates/distroproof/src/vm.rs index 4fc4d27..08094f3 100644 --- a/crates/distroproof/src/vm.rs +++ b/crates/distroproof/src/vm.rs @@ -565,6 +565,30 @@ fn collect(out: &GuestOutput, spec: &VmSpec, prepare_only: bool, result: &mut Ta if spec.command.is_some() && !prepare_only { result.command = Some(stage("command")); } + if !spec.collect.is_empty() && !prepare_only { + let base = spec + .workdir + .clone() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".distroproof/collect") + .join(&spec.name); + for (name, data) in out.files() { + let Some(rel) = name.strip_prefix("collect/") else { + continue; + }; + let path = base.join(rel); + let written = path + .parent() + .map_or(Ok(()), std::fs::create_dir_all) + .and_then(|()| std::fs::write(&path, data)); + match written { + Ok(()) => result.collected.push(path.display().to_string()), + Err(e) => result + .warnings + .push(format!("collect: {}: {e}", path.display())), + } + } + } if let Some(json) = out.get("envprobe.json") { match serde_json::from_slice(json) { Ok(report) => result.envprobe = Some(report), diff --git a/crates/envprobe-core/src/parse/limits.rs b/crates/envprobe-core/src/parse/limits.rs new file mode 100644 index 0000000..6831816 --- /dev/null +++ b/crates/envprobe-core/src/parse/limits.rs @@ -0,0 +1,34 @@ +//! `/proc//limits`: one resource a line, soft and hard limit, unit. + +/// The soft and hard limit of `resource` (`Max locked memory`), as the file +/// prints them: a number or `unlimited`. +pub fn limit(text: &str, resource: &str) -> Option<(String, String)> { + let line = text.lines().find(|line| line.starts_with(resource))?; + let mut fields = line[resource.len()..].split_whitespace(); + Some((fields.next()?.to_string(), fields.next()?.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DEBIAN_11_USER: &str = + "Limit Soft Limit Hard Limit Units +Max cpu time unlimited unlimited seconds +Max locked memory 65536 65536 bytes +Max address space unlimited unlimited bytes +"; + + #[test] + fn the_locked_memory_limit_is_read_soft_then_hard() { + assert_eq!( + limit(DEBIAN_11_USER, "Max locked memory"), + Some(("65536".into(), "65536".into())) + ); + assert_eq!( + limit(DEBIAN_11_USER, "Max cpu time"), + Some(("unlimited".into(), "unlimited".into())) + ); + assert_eq!(limit(DEBIAN_11_USER, "Max realtime timeout"), None); + } +} diff --git a/crates/envprobe-core/src/parse/mod.rs b/crates/envprobe-core/src/parse/mod.rs index 5c39f4a..e92de64 100644 --- a/crates/envprobe-core/src/parse/mod.rs +++ b/crates/envprobe-core/src/parse/mod.rs @@ -4,6 +4,7 @@ pub mod elf; pub mod kconfig; pub mod ldconfig; pub mod libc_version; +pub mod limits; pub mod mounts; pub mod os_release; pub mod status; diff --git a/crates/envprobe-core/src/probes/kernel.rs b/crates/envprobe-core/src/probes/kernel.rs index d6c8379..48f6b6b 100644 --- a/crates/envprobe-core/src/probes/kernel.rs +++ b/crates/envprobe-core/src/probes/kernel.rs @@ -16,6 +16,12 @@ pub const CONFIG_KEYS: &[&str] = &[ "CONFIG_KPROBES", "CONFIG_UPROBES", "CONFIG_FTRACE", + // Whether a trampoline can be installed: without the function tracer the + // kernel's functions carry no call site, and every fentry/fexit attach is + // refused although the program loads (Alpine's `virt` kernel). + "CONFIG_FUNCTION_TRACER", + "CONFIG_DYNAMIC_FTRACE", + "CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS", "CONFIG_DEBUG_INFO_BTF", "CONFIG_PERF_EVENTS", "CONFIG_MODULES", diff --git a/crates/envprobe-core/src/probes/privileges.rs b/crates/envprobe-core/src/probes/privileges.rs index 26b388c..5416555 100644 --- a/crates/envprobe-core/src/probes/privileges.rs +++ b/crates/envprobe-core/src/probes/privileges.rs @@ -101,6 +101,28 @@ pub fn probe() -> Section { None => missing("Seccomp"), }, ); + // Before memory-cgroup accounting (Linux 5.11) every eBPF map is charged + // to this limit, and a distribution's 64 KiB refuses a ring buffer with + // EPERM -- which reads as a missing capability (Debian 11). + let memlock = fs::read_to_string("/proc/self/limits") + .ok() + .and_then(|text| crate::parse::limits::limit(&text, "Max locked memory")); + match memlock { + Some((soft, hard)) => { + s.push_privileged("memlock_soft", Value::ok(soft)); + s.push_privileged("memlock_hard", Value::ok(hard)); + } + None => { + s.push_privileged( + "memlock_soft", + Value::unavailable("/proc/self/limits unreadable"), + ); + s.push_privileged( + "memlock_hard", + Value::unavailable("/proc/self/limits unreadable"), + ); + } + } s.push_privileged( "no_new_privs", status From 9f9af9ae76bc72be20cf3ba3ed4e04d2c25bd4a3 Mon Sep 17 00:00:00 2001 From: moha-bekh Date: Mon, 19 Oct 2026 01:31:58 +0200 Subject: [PATCH 2/2] README: Omniscient as a case study Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 118960f..ad0cc5d 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,19 @@ it. Every phase of the POC is implemented: the `envprobe` probe (usable on its own) and the `distroproof` orchestrator (VMs, matrix, report, TUI). +## In practice: Omniscient + +[Omniscient](https://github.com/moha-bekh/omniscient), a kernel-facing +observability tool (eBPF, perf, BTF, file capabilities), was ported with +distroproof across 16 distributions and 5 variants. It found 16 defects that +its development machine never showed -- a release that would not start on a +third of the images, Debian's `perf_event_paranoid` 3 refusing what +CAP_PERFMON should allow, eBPF maps charged to a 64 KiB `RLIMIT_MEMLOCK` on +5.10, lockdown, a kernel without ftrace, a renamed kernel struct member, +version assumptions RHEL's backports disproved -- each tied to the envprobe +fact that explained it. The case study, finding by finding: +[What distroproof found in Omniscient](https://github.com/moha-bekh/omniscient/blob/main/docs/distroproof.md). + ## Status | Phase | Scope | Status |