file_caps, collect, DP_TARGET; envprobe reports the function tracer and RLIMIT_MEMLOCK

README.md

@@ -15,6 +15,19 @@ it.
 Every phase of the POC is implemented: the `envprobe` probe (usable on its
 own) and the `distroproof` orchestrator (VMs, matrix, report, TUI).
 
+## In practice: Omniscient
+
+[Omniscient](https://github.com/moha-bekh/omniscient), a kernel-facing
+observability tool (eBPF, perf, BTF, file capabilities), was ported with
+distroproof across 16 distributions and 5 variants. It found 16 defects that
+its development machine never showed -- a release that would not start on a
+third of the images, Debian's `perf_event_paranoid` 3 refusing what
+CAP_PERFMON should allow, eBPF maps charged to a 64 KiB `RLIMIT_MEMLOCK` on
+5.10, lockdown, a kernel without ftrace, a renamed kernel struct member,
+version assumptions RHEL's backports disproved -- each tied to the envprobe
+fact that explained it. The case study, finding by finding:
+[What distroproof found in Omniscient](https://github.com/moha-bekh/omniscient/blob/main/docs/distroproof.md).
+
 ## Status
 
 | Phase | Scope | Status |
@@ -195,8 +208,20 @@ command = "./target/release/mytool --selftest"
 as_user = "test"                      # or "root"
 # exclude = [".git", "target"]        # copied to /distroproof/work
 # include = ["target/release/mytool"]
+# file_caps = { "/opt/mytool/bin/mytool" = ["bpf", "perfmon"] }  # setcap after setup
+# collect = ["results/*.json"]        # brought back to .distroproof/collect/<target>/
 ```
 
+`setup` and `command` see `DP_TARGET` (the target's name) and `DP_IMAGE`.
+`file_caps` grants file capabilities on an installed binary, as root, after
+`setup` -- what a package's install does -- where `caps` raises ambient
+capabilities on the user, which every program the command starts inherits;
+a kernel that predates a capability (before 5.8 for `bpf` and `perfmon`) is
+a warning naming `cap_last_cap`. `collect` copies the files matching each
+pattern, relative to the working directory, back to
+`.distroproof/collect/<target>/` after the command, and lists them in the
+report's `collected`.
+
 Errors (unknown key, nonexistent image, invalid duration…) are reported with
 their line and column, without starting any VM. `distroproof up` runs the
 targets in parallel (`--jobs`), attaches to each result the `envprobe` report

crates/distroproof/src/agent.sh

@@ -28,9 +28,11 @@ warn() { say WARN "$*"; echo "$*" >>"$OUT/warnings"; }
 mkdir -p "$OUT"
 say READY "$(cut -d' ' -f1 /proc/uptime)"
 
-# Job parameters: DP_USER, DP_CAPS, DP_PROBE, DP_EXTRACT_KERNEL,
-# DP_SETUP, DP_COMMAND (1 if job/setup.sh, job/command.sh exist).
+# Job parameters: DP_USER, DP_TARGET, DP_IMAGE, DP_CAPS, DP_PROBE,
+# DP_EXTRACT_KERNEL, DP_SETUP, DP_COMMAND (1 if job/setup.sh, job/command.sh
+# exist). DP_TARGET and DP_IMAGE are exported to setup and the command.
 . $D/job/env
+export DP_TARGET DP_IMAGE
 
 disk() {
     for d in /sys/block/*; do
@@ -119,6 +121,27 @@ fi
 
 [ -n "$DP_SETUP" ] && run setup "sh $D/job/setup.sh"
 
+# File capabilities, after setup installed the files: what a program's install
+# grants, where `caps` would raise ambient ones every child inherits.
+if [ -f $D/job/filecaps ]; then
+    if ! command -v setcap >/dev/null 2>&1; then
+        if command -v apk >/dev/null 2>&1; then apk add -q libcap-utils >/dev/null 2>&1
+        elif command -v apt-get >/dev/null 2>&1; then apt-get install -qq -y libcap2-bin >/dev/null 2>&1
+        elif command -v dnf >/dev/null 2>&1; then dnf install -q -y libcap >/dev/null 2>&1
+        elif command -v pacman >/dev/null 2>&1; then pacman -Sq --noconfirm libcap >/dev/null 2>&1
+        fi
+    fi
+    last=$(cat /proc/sys/kernel/cap_last_cap 2>/dev/null || echo 0)
+    while IFS="$(printf '\t')" read -r path caps; do
+        [ -n "$path" ] || continue
+        if ! command -v setcap >/dev/null 2>&1; then
+            warn "file_caps: no setcap in this image, $path keeps no capability"
+        elif ! setcap "$caps" "$path" 2>"$OUT/filecaps.err"; then
+            warn "file_caps: setcap $caps $path refused (cap_last_cap $last): $(head -c 200 "$OUT/filecaps.err")"
+        fi
+    done <$D/job/filecaps
+fi
+
 if [ "$DP_PROBE" = 1 ]; then
     say STAGE probe
     sh -c "$(as_user $D/job/probe.sh)" >"$OUT/envprobe.json" 2>"$OUT/envprobe.err" </dev/null
@@ -127,6 +150,22 @@ fi
 
 [ -n "$DP_COMMAND" ] && run command "$(as_user $D/job/command.sh)"
 
+# Files the job asked back, kept with their paths under out/collect.
+if [ -f $D/job/collect ]; then
+    mkdir -p "$OUT/collect"
+    while IFS= read -r pattern; do
+        [ -n "$pattern" ] || continue
+        found=""
+        for f in $pattern; do
+            [ -e "$f" ] || continue
+            found=1
+            tar -cf - "$f" 2>/dev/null | tar -xf - -C "$OUT/collect" 2>/dev/null ||
+                warn "collect: cannot copy $f"
+        done
+        [ -n "$found" ] || warn "collect: nothing matches $pattern"
+    done <$D/job/collect
+fi
+
 # Kernel and initramfs for the following direct boots.
 if [ "$DP_EXTRACT_KERNEL" = 1 ]; then
     say STAGE kernel

crates/distroproof/src/config.rs

@@ -105,6 +105,10 @@ struct RawRun {
     probe: Option<bool>,
     #[serde(default)]
     env: BTreeMap<Spanned<String>, String>,
+    #[serde(default)]
+    file_caps: BTreeMap<String, Spanned<Vec<String>>>,
+    #[serde(default)]
+    collect: Vec<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -359,6 +363,20 @@ pub fn parse(path: &Path, text: &str) -> Result<Config, ConfigError> {
             }
             None => Vec::new(),
         };
+        let mut file_caps = BTreeMap::new();
+        for (path, caps) in &run.file_caps {
+            if !path.starts_with('/') {
+                return Err(ctx.error(
+                    caps.span(),
+                    format!("file_caps: `{path}` is not an absolute path"),
+                ));
+            }
+            check_caps(caps, &ctx)?;
+            file_caps.insert(
+                path.clone(),
+                caps.get_ref().iter().map(|c| normalize_cap(c)).collect(),
+            );
+        }
         targets.push(VmSpec {
             name: name.clone(),
             image: t.image.get_ref().clone(),
@@ -381,6 +399,8 @@ pub fn parse(path: &Path, text: &str) -> Result<Config, ConfigError> {
             include: run.include.clone(),
             probe,
             hold: false,
+            file_caps,
+            collect: run.collect.clone(),
         });
     }
     Ok(Config { targets })
@@ -616,6 +636,27 @@ as_user = "test"
         );
     }
 
+    #[test]
+    fn file_caps_and_collect() {
+        let base = "[[target]]\nname = \"a\"\nimage = \"alpine\"\n";
+        let ok = format!(
+            "{base}[run]\ncommand = \"true\"\ncollect = [\"out/*.json\"]\n\
+             file_caps = {{ \"/opt/omn/bin/omn\" = [\"CAP_BPF\", \"perfmon\"] }}\n"
+        );
+        let spec = &parse_str(&ok).unwrap().targets[0];
+        assert_eq!(spec.collect, ["out/*.json"]);
+        assert_eq!(spec.file_caps["/opt/omn/bin/omn"], ["bpf", "perfmon"]);
+        let relative =
+            format!("{base}[run]\ncommand = \"true\"\nfile_caps = {{ \"omn\" = [\"bpf\"] }}\n");
+        assert!(parse_str(&relative)
+            .unwrap_err()
+            .message
+            .contains("not an absolute path"));
+        let bad =
+            format!("{base}[run]\ncommand = \"true\"\nfile_caps = {{ \"/x\" = [\"fly\"] }}\n");
+        assert!(parse_str(&bad).unwrap_err().message.contains("fly"));
+    }
+
     #[test]
     fn sizes_and_durations() {
         assert_eq!(parse_size_mib("1G").unwrap(), 1024);

crates/distroproof/src/guest.rs

@@ -73,6 +73,8 @@ impl Payload<'_> {
         let mut env = String::new();
         let mut var = |k: &str, v: &str| env.push_str(&format!("{k}={}\n", shell_quote(v)));
         var("DP_USER", &spec.user);
+        var("DP_TARGET", &spec.name);
+        var("DP_IMAGE", &spec.image);
         var("DP_CAPS", &spec.caps.join(","));
         var("DP_PROBE", if run_steps && spec.probe { "1" } else { "0" });
         var(
@@ -100,6 +102,21 @@ impl Payload<'_> {
         if let Some(command) = command {
             files.push(("job/command.sh".into(), script(command), 0o755));
         }
+        if run_steps && !spec.file_caps.is_empty() {
+            let lines: String = spec
+                .file_caps
+                .iter()
+                .map(|(path, caps)| {
+                    let caps: Vec<String> = caps.iter().map(|c| format!("cap_{c}")).collect();
+                    format!("{}\t{}+ep\n", path, caps.join(","))
+                })
+                .collect();
+            files.push(("job/filecaps".into(), lines.into_bytes(), 0o644));
+        }
+        if run_steps && !spec.collect.is_empty() {
+            let lines: String = spec.collect.iter().map(|p| format!("{p}\n")).collect();
+            files.push(("job/collect".into(), lines.into_bytes(), 0o644));
+        }
         if !spec.sysctl.is_empty() {
             let lines: String = spec
                 .sysctl
@@ -235,6 +252,11 @@ impl GuestOutput {
         Ok(GuestOutput { files })
     }
 
+    /// Every file of the output disk, by name.
+    pub fn files(&self) -> impl Iterator<Item = (&str, &[u8])> {
+        self.files.iter().map(|(n, d)| (n.as_str(), d.as_slice()))
+    }
+
     pub fn get(&self, name: &str) -> Option<&[u8]> {
         self.files
             .iter()

crates/distroproof/src/main.rs

@@ -304,6 +304,8 @@ async fn cmd_run(args: RunArgs) -> Result<ExitCode, String> {
         include: Vec::new(),
         probe: args.probe,
         hold: false,
+        file_caps: Default::default(),
+        collect: Vec::new(),
     };
     let runner = runner(&args.common)?;
     let (tx, rx) = mpsc::unbounded_channel();

crates/distroproof/src/model.rs

@@ -40,6 +40,14 @@ pub struct VmSpec {
     /// Keep the VM up after the command, with a root shell on the serial
     /// console, until it is killed.
     pub hold: bool,
+    /// Files whose capabilities are set as root after `setup`, by path:
+    /// `setcap cap_<a>,cap_<b>+ep <path>`. A file capability is what a
+    /// program's install grants; `caps` raises ambient ones on the user, which
+    /// every program the command starts inherits.
+    pub file_caps: BTreeMap<String, Vec<String>>,
+    /// Paths, relative to the working directory, copied back after the
+    /// command into `<workdir>/.distroproof/collect/<target>/`.
+    pub collect: Vec<String>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -121,6 +129,9 @@ pub struct TargetResult {
     /// Last lines of the serial console, useful when the boot fails.
     #[serde(default)]
     pub console_tail: String,
+    /// Files brought back by `collect`, as written on the host.
+    #[serde(default)]
+    pub collected: Vec<String>,
 }
 
 impl TargetResult {
@@ -146,6 +157,7 @@ impl TargetResult {
             warnings: Vec::new(),
             error: None,
             console_tail: String::new(),
+            collected: Vec::new(),
         }
     }
 

crates/distroproof/src/report.rs

@@ -752,6 +752,8 @@ mod tests {
             include: Vec::new(),
             probe: true,
             hold: false,
+            file_caps: Default::default(),
+            collect: Vec::new(),
         }
     }
 

crates/distroproof/src/tui/mod.rs

@@ -130,6 +130,8 @@ mod tests {
             include: Vec::new(),
             probe: true,
             hold: false,
+            file_caps: BTreeMap::new(),
+            collect: Vec::new(),
         }
     }
 

crates/distroproof/src/vm.rs

@@ -565,6 +565,30 @@ fn collect(out: &GuestOutput, spec: &VmSpec, prepare_only: bool, result: &mut Ta
     if spec.command.is_some() && !prepare_only {
         result.command = Some(stage("command"));
     }
+    if !spec.collect.is_empty() && !prepare_only {
+        let base = spec
+            .workdir
+            .clone()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(".distroproof/collect")
+            .join(&spec.name);
+        for (name, data) in out.files() {
+            let Some(rel) = name.strip_prefix("collect/") else {
+                continue;
+            };
+            let path = base.join(rel);
+            let written = path
+                .parent()
+                .map_or(Ok(()), std::fs::create_dir_all)
+                .and_then(|()| std::fs::write(&path, data));
+            match written {
+                Ok(()) => result.collected.push(path.display().to_string()),
+                Err(e) => result
+                    .warnings
+                    .push(format!("collect: {}: {e}", path.display())),
+            }
+        }
+    }
     if let Some(json) = out.get("envprobe.json") {
         match serde_json::from_slice(json) {
             Ok(report) => result.envprobe = Some(report),

crates/envprobe-core/src/parse/limits.rs

@@ -0,0 +1,34 @@
+//! `/proc/<pid>/limits`: one resource a line, soft and hard limit, unit.
+
+/// The soft and hard limit of `resource` (`Max locked memory`), as the file
+/// prints them: a number or `unlimited`.
+pub fn limit(text: &str, resource: &str) -> Option<(String, String)> {
+    let line = text.lines().find(|line| line.starts_with(resource))?;
+    let mut fields = line[resource.len()..].split_whitespace();
+    Some((fields.next()?.to_string(), fields.next()?.to_string()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const DEBIAN_11_USER: &str =
+        "Limit                     Soft Limit           Hard Limit           Units
+Max cpu time              unlimited            unlimited            seconds
+Max locked memory         65536                65536                bytes
+Max address space         unlimited            unlimited            bytes
+";
+
+    #[test]
+    fn the_locked_memory_limit_is_read_soft_then_hard() {
+        assert_eq!(
+            limit(DEBIAN_11_USER, "Max locked memory"),
+            Some(("65536".into(), "65536".into()))
+        );
+        assert_eq!(
+            limit(DEBIAN_11_USER, "Max cpu time"),
+            Some(("unlimited".into(), "unlimited".into()))
+        );
+        assert_eq!(limit(DEBIAN_11_USER, "Max realtime timeout"), None);
+    }
+}

crates/envprobe-core/src/parse/mod.rs

@@ -4,6 +4,7 @@ pub mod elf;
 pub mod kconfig;
 pub mod ldconfig;
 pub mod libc_version;
+pub mod limits;
 pub mod mounts;
 pub mod os_release;
 pub mod status;

crates/envprobe-core/src/probes/kernel.rs

@@ -16,6 +16,12 @@ pub const CONFIG_KEYS: &[&str] = &[
     "CONFIG_KPROBES",
     "CONFIG_UPROBES",
     "CONFIG_FTRACE",
+    // Whether a trampoline can be installed: without the function tracer the
+    // kernel's functions carry no call site, and every fentry/fexit attach is
+    // refused although the program loads (Alpine's `virt` kernel).
+    "CONFIG_FUNCTION_TRACER",
+    "CONFIG_DYNAMIC_FTRACE",
+    "CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS",
     "CONFIG_DEBUG_INFO_BTF",
     "CONFIG_PERF_EVENTS",
     "CONFIG_MODULES",

crates/envprobe-core/src/probes/privileges.rs

@@ -101,6 +101,28 @@ pub fn probe() -> Section {
             None => missing("Seccomp"),
         },
     );
+    // Before memory-cgroup accounting (Linux 5.11) every eBPF map is charged
+    // to this limit, and a distribution's 64 KiB refuses a ring buffer with
+    // EPERM -- which reads as a missing capability (Debian 11).
+    let memlock = fs::read_to_string("/proc/self/limits")
+        .ok()
+        .and_then(|text| crate::parse::limits::limit(&text, "Max locked memory"));
+    match memlock {
+        Some((soft, hard)) => {
+            s.push_privileged("memlock_soft", Value::ok(soft));
+            s.push_privileged("memlock_hard", Value::ok(hard));
+        }
+        None => {
+            s.push_privileged(
+                "memlock_soft",
+                Value::unavailable("/proc/self/limits unreadable"),
+            );
+            s.push_privileged(
+                "memlock_hard",
+                Value::unavailable("/proc/self/limits unreadable"),
+            );
+        }
+    }
     s.push_privileged(
         "no_new_privs",
         status