From 68a7c5fe60d31f7ad7a9928c83432a4b63af07d8 Mon Sep 17 00:00:00 2001 From: YUZHEthefool <2804776511@qq.com> Date: Fri, 30 Oct 2026 16:09:41 +0800 Subject: [PATCH] fix(fuzz): retain encrypted private campaign evidence Run scheduled campaigns through a private evidence wrapper that records the exact input, source identity, tool versions and raw libFuzzer output, then streams the archive into age. Upload only ciphertext; keep the public candidate schema unchanged. Retain failure-only QEMU execution evidence with an outcome receipt and map public candidate IDs to inputs in a private crash-triage index. Fail the campaign closed when recipients, encryption or upload fail. Set the QEMU libFuzzer per-input timeout above the 900-second guest bound in both the runner and local Makefile campaigns. Refs #46 --- .github/workflows/ci.yml | 1 + .github/workflows/fuzz.yml | 83 +++---- .gitignore | 1 + CONTRIBUTING.md | 3 +- Makefile | 6 +- fuzz/README.md | 41 +++- fuzz/src/qemu_executor.rs | 36 ++- scripts/fuzz/private_evidence.py | 231 ++++++++++++++++++++ scripts/tests/private_fuzz_evidence_test.py | 168 ++++++++++++++ userspace/fuzzer/Cargo.toml | 5 +- userspace/fuzzer/src/bin/crash-triage.rs | 107 +++++++++ userspace/nilix-syz-fuzzer/src/executor.rs | 81 +++++-- 12 files changed, 699 insertions(+), 64 deletions(-) create mode 100644 scripts/fuzz/private_evidence.py create mode 100644 scripts/tests/private_fuzz_evidence_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72f0f361..91e5eab5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,7 @@ jobs: with: python-version: '3.12' - run: python3 -m pip install -r requirements-ci.txt + - run: sudo apt-get update && sudo apt-get install -y age - run: bash scripts/ci/entrypoint.sh quality - name: Check Actions definitions if: always() diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index a851327e..103a5c37 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -78,7 +78,12 @@ jobs: - name: Install QEMU and build dependencies run: | sudo apt-get update - sudo apt-get install -y qemu-system-x86 musl-tools + sudo apt-get install -y qemu-system-x86 musl-tools age + + - name: Verify private evidence encryption + env: + FUZZ_EVIDENCE_RECIPIENTS: ${{ vars.FUZZ_EVIDENCE_RECIPIENTS }} + run: python3 scripts/fuzz/private_evidence.py preflight # Build before any secret is present so dependency build scripts never # inherit the private fingerprinting key. @@ -102,48 +107,23 @@ jobs: - name: Prepare fuzz directories run: | mkdir -p fuzz/corpus/${{ matrix.target }} - mkdir -p fuzz/artifacts/${{ matrix.target }} # Raw libFuzzer output can include Base64 reproducers and stack traces. - # Keep it only in RUNNER_TEMP; never tee, cat, summarize, or upload it. + # Keep plaintext in RUNNER_TEMP. Upload only the sealed age archive. - name: Run fuzzer privately id: run_fuzzer env: FUZZ_TIMEOUT: ${{ inputs.timeout || '900' }} run: | - timeout="$FUZZ_TIMEOUT" - if ! [[ "$timeout" =~ ^[1-9][0-9]*$ ]] || [ "$timeout" -gt 1200 ]; then - echo "::error::Fuzz timeout must be an integer from 1 through 1200" - exit 1 - fi - - private_log="$RUNNER_TEMP/libfuzzer-output.log" - set +e - - # QEMU fuzzer needs the qemu-executor feature flag - if [ "${{ matrix.target }}" = "fuzz_syscall_qemu" ]; then - cargo +nightly-2025-12-08 fuzz run ${{ matrix.target }} --features qemu-executor -- \ - -max_total_time="$timeout" \ - -rss_limit_mb=4096 \ - -print_final_stats=1 \ - > "$private_log" 2>&1 - else - cargo +nightly-2025-12-08 fuzz run ${{ matrix.target }} -- \ - -max_total_time="$timeout" \ - -rss_limit_mb=4096 \ - -print_final_stats=1 \ - > "$private_log" 2>&1 - fi - - exit_code=$? - set -e - echo "exit_code=${exit_code}" >> "$GITHUB_OUTPUT" + python3 scripts/fuzz/private_evidence.py run \ + --private-dir "$RUNNER_TEMP/nilix-fuzz-evidence" \ + --target "${{ matrix.target }}" --timeout "$FUZZ_TIMEOUT" - name: Count private findings if: always() id: count_findings run: | - findings=$(find fuzz/artifacts/${{ matrix.target }} -maxdepth 1 -type f \ + findings=$(find "$RUNNER_TEMP/nilix-fuzz-evidence/findings" -maxdepth 1 -type f \ \( -name 'crash-*' -o -name 'timeout-*' -o -name 'oom-*' -o -name 'leak-*' \) \ 2>/dev/null | wc -l) echo "findings=${findings}" >> "$GITHUB_OUTPUT" @@ -178,12 +158,13 @@ jobs: exit 1 fi - input_root="$RUNNER_TEMP/nilix-finding-input" + umask 077 + input_root="$RUNNER_TEMP/nilix-fuzz-evidence/finding-input" input_dir="$input_root/cargo-fuzz-crashes-${{ matrix.target }}" candidate_dir="fuzz/candidates" - private_triage_log="$RUNNER_TEMP/candidate-triage.log" + private_triage_log="$RUNNER_TEMP/nilix-fuzz-evidence/triage.log" mkdir -p "$input_dir" "$candidate_dir" - if ! find fuzz/artifacts/${{ matrix.target }} -maxdepth 1 -type f \ + if ! find "$RUNNER_TEMP/nilix-fuzz-evidence/findings" -maxdepth 1 -type f \ \( -name 'crash-*' -o -name 'timeout-*' -o -name 'oom-*' -o -name 'leak-*' \) \ -exec cp -- {} "$input_dir/" \; \ > /dev/null 2> "$private_triage_log"; then @@ -194,6 +175,7 @@ jobs: if ! userspace/fuzzer/target/x86_64-unknown-linux-gnu/release/crash-triage \ --crash-dir "$input_root" \ --output-dir "$candidate_dir" \ + --private-index "$RUNNER_TEMP/nilix-fuzz-evidence/candidate-index.json" \ --expected-findings "$FUZZ_FINDINGS" \ --dedup \ >> "$private_triage_log" 2>&1; then @@ -217,8 +199,31 @@ jobs: retention-days: 90 if-no-files-found: error + # Preserve evidence even when the fuzzer/triage failed without a candidate. + # A missing recipient, seal failure or upload failure makes the campaign + # incomplete and prevents the reporting job from creating an empty pointer. + - name: Seal private campaign evidence + id: seal_evidence + if: always() && steps.run_fuzzer.outcome != 'skipped' + env: + FUZZ_EVIDENCE_RECIPIENTS: ${{ vars.FUZZ_EVIDENCE_RECIPIENTS }} + run: | + python3 scripts/fuzz/private_evidence.py seal \ + --private-dir "$RUNNER_TEMP/nilix-fuzz-evidence" \ + --output fuzz/evidence/campaign.tar.gz.age + + - name: Upload encrypted campaign evidence + id: upload_evidence + if: always() && steps.seal_evidence.outcome == 'success' + uses: actions/upload-artifact@v7 + with: + name: fuzz-evidence-${{ strategy.job-index }} + path: fuzz/evidence/campaign.tar.gz.age + retention-days: 90 + if-no-files-found: error + - name: Save clean corpus - if: steps.validate_fuzzer.outcome == 'success' && steps.count_findings.outputs.findings == '0' + if: steps.validate_fuzzer.outcome == 'success' && steps.count_findings.outputs.findings == '0' && steps.upload_evidence.outcome == 'success' uses: actions/cache/save@v6 with: path: fuzz/corpus/${{ matrix.target }}/ @@ -232,12 +237,14 @@ jobs: COUNT_OUTCOME: ${{ steps.count_findings.outcome }} VALIDATE_OUTCOME: ${{ steps.validate_fuzzer.outcome }} CANDIDATE_OUTCOME: ${{ steps.create_candidates.outcome }} + SEAL_OUTCOME: ${{ steps.seal_evidence.outcome }} + EVIDENCE_UPLOAD_OUTCOME: ${{ steps.upload_evidence.outcome }} FINDINGS: ${{ steps.count_findings.outputs.findings }} CANDIDATE_COUNT: ${{ steps.create_candidates.outputs.candidate_count }} run: | status=completed candidate_count=0 - if [ "$RUN_OUTCOME" != "success" ] || [ "$COUNT_OUTCOME" != "success" ] || [ "$VALIDATE_OUTCOME" != "success" ]; then + if [ "$RUN_OUTCOME" != "success" ] || [ "$COUNT_OUTCOME" != "success" ] || [ "$VALIDATE_OUTCOME" != "success" ] || [ "$SEAL_OUTCOME" != "success" ] || [ "$EVIDENCE_UPLOAD_OUTCOME" != "success" ]; then status=incomplete elif ! [[ "$FINDINGS" =~ ^[0-9]+$ ]]; then status=incomplete @@ -326,11 +333,11 @@ jobs: gh issue reopen "$issue_number" fi gh issue comment "$issue_number" \ - --body "The opaque candidate was observed again in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) at head ${{ github.sha }}. Details remain private under SECURITY.md." + --body "The opaque candidate was observed again in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) at head ${{ github.sha }}. Encrypted evidence is retained in the run's fuzz-evidence artifacts; details remain private under SECURITY.md." continue fi - issue_body="Automated fuzzing observed a private candidate in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). Potential security details and the reproducer were not written to public logs or artifacts. Maintainers should coordinate under SECURITY.md. Opaque candidate ID: ${candidate_id}." + issue_body="Automated fuzzing observed a private candidate in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). Encrypted evidence is retained in the run's fuzz-evidence artifacts. Plaintext security details and the reproducer are not public. Maintainers should decrypt the archive and inspect candidate-index.json and guest outcome.json before classifying the finding under SECURITY.md. Opaque candidate ID: ${candidate_id}." issue_body+=$'\n\n' issue_body+="$marker" gh issue create \ diff --git a/.gitignore b/.gitignore index 30fee086..c737dad1 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ userspace/clippy-userspace-target # Fuzz testing /fuzz/target/ /fuzz/corpus/ +/fuzz/evidence/ /fuzz/artifacts/*.log /hosted-subcrate-review-target/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index faa50a3c..9b325fae 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,10 +43,11 @@ the toolchain installed work too. - **QEMU + OVMF** — `qemu-system-x86_64` and the OVMF UEFI firmware (for boot/run). - **GNU Make**. - **musl toolchain** — `musl-tools` (`musl-gcc`), only for the musl conformance gate. +- **age** — only for the private fuzz evidence harness tests (`scripts/tests/private_fuzz_evidence_test.py`). On Debian/Ubuntu the non-Rust deps match what CI installs: ```bash -sudo apt-get install -y qemu-system-x86 ovmf musl-tools make +sudo apt-get install -y qemu-system-x86 ovmf musl-tools make age ``` > Maintainer note: the project is also developed from a Windows mirror that has **no** diff --git a/Makefile b/Makefile index aec35f0c..f0d080ac 100644 --- a/Makefile +++ b/Makefile @@ -1151,7 +1151,7 @@ fuzz-qemu-campaign: build-fuzz-qemu-deps fuzz-qemu-seeds cd fuzz && \ cargo +nightly fuzz run fuzz_syscall_qemu --features qemu-executor -- \ -max_total_time=3600 \ - -timeout=120 \ + -timeout=1200 \ -rss_limit_mb=4096 \ -print_final_stats=1 @@ -1161,7 +1161,7 @@ fuzz-qemu-overnight: build-fuzz-qemu-deps fuzz-qemu-seeds cd fuzz && \ cargo +nightly fuzz run fuzz_syscall_qemu --features qemu-executor -- \ -max_total_time=28800 \ - -timeout=120 \ + -timeout=1200 \ -rss_limit_mb=8192 \ -print_final_stats=1 @@ -1171,7 +1171,7 @@ fuzz-qemu-parallel: build-fuzz-qemu-deps fuzz-qemu-seeds cd fuzz && \ cargo +nightly fuzz run fuzz_syscall_qemu --features qemu-executor --jobs=4 -- \ -max_total_time=3600 \ - -timeout=120 \ + -timeout=1200 \ -rss_limit_mb=4096 \ -print_final_stats=1 diff --git a/fuzz/README.md b/fuzz/README.md index 40a1ef44..7afaccba 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -42,11 +42,50 @@ nonzero coverage checks for the seeds that prove the KCOV data path. There is no measured executions-per-second claim. Build with `make build-syz-kcov` and enable `qemu-executor`. Guest startup has a -90-second bound; use `-timeout=120` or larger for libFuzzer. Optional +900-second bound; keep libFuzzer's per-input timeout above that bound (the private +campaign runner passes `-timeout=1200` for this target). Optional `NILIX_FUZZ_KERNEL`, `NILIX_FUZZ_QEMU`, `OVMF_PATH` and `NILIX_FUZZ_ARTIFACTS` override the image, emulator, firmware and retained-evidence directory. Artifact retention is intended for finite validation runs because it keeps each guest disk. +For campaigns, use `NILIX_FUZZ_FAILURE_ARTIFACTS` instead. It retains only failed +executions, including execution/protocol errors, and removes successful guest +disks. Each retained directory includes the submitted program, private guest +image, disk, serial output, QEMU stderr and `outcome.json`. The receipt distinguishes +crashes (with their observed classification), boot timeout, guest hang and +execution errors; an execution error alone does not establish a kernel defect. +Do not set both artifact-retention variables. + +### Private campaign evidence + +The scheduled workflow requires the repository variable `FUZZ_EVIDENCE_RECIPIENTS` +to contain one or more native age public keys, one per line. Generate and retain +the corresponding private identities on a maintainer-controlled machine. Never +put those private identities in CI or in this repository. The workflow verifies +the public recipients before running a fuzzer. + +`scripts/fuzz/private_evidence.py run` records the exact command, source revision +and hashes, source changes, guest-image hashes, tool versions, exit status and raw +libFuzzer output in a private directory. Findings use a fresh private artifact +prefix. QEMU failures also retain their per-execution evidence. The triage tool's +`--private-index` maps public candidate IDs to the inputs inside that directory; +the public candidate format remains two fields and contains no reproducer. + +After the campaign, `private_evidence.py seal` streams a tar/gzip archive directly +into age. Only `campaign.tar.gz.age` is uploaded, under `fuzz-evidence-`. +Raw files are never uploaded or cached. Encryption or upload failure makes the +campaign incomplete and prevents automatic issue creation. The plaintext archive +size is bounded to 1 GiB; successful execution disks are cleaned up. Archives are +retained for 90 days, so maintainers should copy relevant ciphertext before expiry. + +Decrypt on a trusted machine with `age -d -i /path/to/identity.agekey -o evidence.tar.gz +campaign.tar.gz.age`. Keep the decrypted evidence private under `SECURITY.md`. +Inspect `run.json`, `candidate-index.json` and `guests/*/outcome.json` before replay. +The index's `input` paths are relative to `finding-input/`; the copy preserves the +exact libFuzzer input bytes. Candidate IDs deduplicate inputs, not root causes: +different IDs can still refer to the same defect. Historical ID-only reports cannot +recover their lost inputs. + ### 2. `fuzz_vfs_path` - VFS Path Operations Tests filesystem operations: - **Path traversal** attacks (../, multiple slashes) diff --git a/fuzz/src/qemu_executor.rs b/fuzz/src/qemu_executor.rs index 700dbb2f..a9f3d150 100644 --- a/fuzz/src/qemu_executor.rs +++ b/fuzz/src/qemu_executor.rs @@ -9,7 +9,7 @@ pub use nilix_syz_fuzzer::executor::{CrashInfo, ExecutionResult, QemuExecutor}; pub use nilix_syz_fuzzer::program::{Argument, Syscall, SyscallProgram}; use std::path::PathBuf; -pub const GUEST_TIMEOUT_SECS: u64 = 900; +pub const GUEST_TIMEOUT_SECS: u64 = 900; // Same non-destructive contract validated by the shared program/guest protocol. const SYSCALLS: &[u32] = &[ @@ -43,10 +43,17 @@ pub fn configured_executor() -> Result { Ext3Tools::default(), ) .context("QEMU executor setup failed; build the guest with make build-syz-kcov")?; - Ok(match std::env::var_os("NILIX_FUZZ_ARTIFACTS") { - Some(root) => executor.with_artifact_dir(PathBuf::from(root)), - None => executor, - }) + Ok( + match ( + std::env::var_os("NILIX_FUZZ_ARTIFACTS"), + std::env::var_os("NILIX_FUZZ_FAILURE_ARTIFACTS"), + ) { + (Some(root), None) => executor.with_artifact_dir(PathBuf::from(root)), + (None, Some(root)) => executor.with_failure_artifact_dir(PathBuf::from(root)), + (None, None) => executor, + (Some(_), Some(_)) => anyhow::bail!("choose one artifact retention mode"), + }, + ) } /// count:u8, then (syscall-index:u8, argc:u8, arguments...). Argument tags: @@ -199,11 +206,28 @@ mod tests { 16, Ext3Tools::default(), ) - .unwrap(); + .unwrap() + .with_failure_artifact_dir(root.join("failures")); assert!(missing .execute(&program) .unwrap_err() .to_string() .contains("failed to spawn")); + let failures: Vec<_> = std::fs::read_dir(root.join("failures")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect(); + assert_eq!(failures.len(), 1); + let receipt: serde_json::Value = + serde_json::from_slice(&std::fs::read(failures[0].join("outcome.json")).unwrap()) + .unwrap(); + assert_eq!(receipt["outcome"], "execution_error"); + assert_eq!(receipt["guest_began"], false); + assert!(receipt["classification"] + .as_str() + .unwrap() + .contains("failed to spawn")); + assert!(failures[0].join("program.json").is_file()); + assert!(failures[0].join("syz-program.host.bin").is_file()); } } diff --git a/scripts/fuzz/private_evidence.py b/scripts/fuzz/private_evidence.py new file mode 100644 index 00000000..638a8c78 --- /dev/null +++ b/scripts/fuzz/private_evidence.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Run private campaigns and stream their evidence into an age-encrypted archive.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tarfile +import tempfile +import time + + +ROOT = Path(__file__).resolve().parents[2] +TARGETS = { + "fuzz_syscall", "fuzz_vfs_path", "fuzz_signal_delivery", "fuzz_memory_ops", + "fuzz_ipc_message", "fuzz_scheduler", "fuzz_network_packet", "fuzz_cgroup_ops", + "fuzz_elf_loader", "fuzz_futex_ops", "fuzz_syscall_qemu", +} +AGE_HEADER = b"age-encryption.org/v1\n" +MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024 + + +class EvidenceError(Exception): + """Messages are fixed public diagnostics, never raw fuzzer/tool output.""" + + +def recipients_text(value): + lines = [line.strip() for line in value.splitlines() if line.strip()] + if not lines or any(not line.startswith("age1") or not line.isalnum() for line in lines): + raise EvidenceError("Configure FUZZ_EVIDENCE_RECIPIENTS with native age public keys.") + return "\n".join(lines) + "\n" + + +def preflight(recipients, age="age"): + """Validate encryption before launching any campaign that could find a bug.""" + with tempfile.TemporaryDirectory(prefix="nilix-age-preflight-") as directory: + keys = Path(directory) / "recipients.txt" + keys.write_text(recipients_text(recipients)) + probe = subprocess.run( + [age, "--encrypt", "--recipients-file", str(keys)], + input=b"nilix encryption preflight\n", capture_output=True, timeout=30, + ) + if probe.returncode or not probe.stdout.startswith(AGE_HEADER): + raise EvidenceError("The fuzz evidence recipient or age encryption tool is invalid.") + + +def digest(path): + result = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + result.update(block) + return result.hexdigest() + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + +def tool_version(command): + try: + result = subprocess.run(command, capture_output=True, timeout=15) + return (result.stdout + result.stderr)[:8192].decode("utf8", errors="replace") + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + + +def campaign_command(private, target, seconds, *, seed=None, corpus=None): + command = ["cargo", "+nightly-2025-12-08", "fuzz", "run", target] + if corpus is not None: + command.append(str(corpus.resolve())) + if target == "fuzz_syscall_qemu": + command.extend(["--features", "qemu-executor"]) + command.extend([ + "--", f"-max_total_time={seconds}", "-rss_limit_mb=4096", + "-print_final_stats=1", f"-artifact_prefix={private / 'findings'}/", + ]) + if target == "fuzz_syscall_qemu": + # libFuzzer must not preempt the executor's own 900-second guest bound. + command.append("-timeout=1200") + if seed is not None: + command.append(f"-seed={seed}") + return command + + +def run_campaign(root, private, target, seconds, *, seed=None, corpus=None): + if target not in TARGETS or not 1 <= seconds <= 1200: + raise EvidenceError("Invalid fuzz target or campaign duration.") + # Exclusive creation prevents stale findings or an earlier run from being + # attributed to this invocation. Raw evidence never enters a build cache. + private.mkdir(mode=0o700, parents=True, exist_ok=False) + (private / "findings").mkdir(mode=0o700) + command = campaign_command(private, target, seconds, seed=seed, corpus=corpus) + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + tracked = subprocess.check_output(["git", "ls-files", "-z"], cwd=root).decode().split("\0") + inputs = {name: digest(root / name) for name in tracked if name and (root / name).is_file()} + (private / "source.patch").write_bytes( + subprocess.check_output(["git", "diff", "--binary", "HEAD"], cwd=root) + ) + kernel = Path(os.environ.get("NILIX_FUZZ_KERNEL", root / "esp-syz/kernel.elf")).resolve() + images = [kernel, kernel.parent / "EFI/BOOT/BOOTX64.EFI"] if target == "fuzz_syscall_qemu" else [] + metadata = { + "schema": "nilix-private-campaign-v1", "revision": revision, "target": target, + "command": command, "source_sha256": inputs, + "images_sha256": {str(path): digest(path) for path in images if path.is_file()}, + "run_id": os.environ.get("GITHUB_RUN_ID"), + "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "tools": {name: tool_version(tool_command) for name, tool_command in { + "rustc": ["rustc", "+nightly-2025-12-08", "-vV"], + "cargo-fuzz": ["cargo", "fuzz", "--version"], + "qemu": ["qemu-system-x86_64", "--version"], + }.items()}, + "started_at": time.time(), "exit_code": None, + } + write_json(private / "run.json", metadata) + environment = os.environ.copy() + # Neither dependency build scripts nor the fuzzer need the HMAC secret. + environment.pop("NILIX_FUZZ_FINGERPRINT_KEY", None) + environment.pop("NILIX_FUZZ_ARTIFACTS", None) + environment["NILIX_FUZZ_FAILURE_ARTIFACTS"] = str(private / "guests") + with (private / "libfuzzer.log").open("wb") as log: + result = subprocess.run(command, cwd=root, env=environment, stdout=log, stderr=subprocess.STDOUT) + code = result.returncode if result.returncode >= 0 else 128 - result.returncode + metadata.update(exit_code=code, finished_at=time.time()) + write_json(private / "run.json", metadata) + return code + + +def archive_entries(private, max_bytes): + if private.is_symlink() or not private.is_dir(): + raise EvidenceError("The private evidence directory is missing or unsafe.") + if not (private / "run.json").is_file() or not (private / "libfuzzer.log").is_file(): + raise EvidenceError("The private campaign evidence is incomplete.") + entries, total = [], 0 + for path in sorted(private.rglob("*")): + mode = path.lstat().st_mode + if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode)): + raise EvidenceError("Private evidence contains a link or special file; refusing to archive it.") + if stat.S_ISREG(mode): + if path.stat().st_nlink != 1: + raise EvidenceError("Private evidence contains a hard link; refusing to archive it.") + total += path.stat().st_size + if total > max_bytes: + raise EvidenceError("Private evidence exceeds the archive size limit.") + entries.append(path) + return entries + + +def seal(private, output, recipients, age="age", max_bytes=MAX_ARCHIVE_BYTES): + private, output = private.absolute(), output.absolute() + if output.is_relative_to(private) or output.exists(): + raise EvidenceError("The encrypted output must be new and outside the private evidence directory.") + keys_text = recipients_text(recipients) + entries = archive_entries(private, max_bytes) + output.parent.mkdir(parents=True, exist_ok=True) + # Only ciphertext is staged on disk. No plaintext tar file can be picked up + # by an artifact upload, even when age exits or the tar stream fails. + with tempfile.TemporaryDirectory(prefix=".nilix-age-", dir=output.parent) as directory: + staging = Path(directory) + keys = staging / "recipients.txt" + keys.write_text(keys_text) + encrypted = staging / "evidence.age" + with (staging / "age.stderr").open("wb") as diagnostics: + process = subprocess.Popen( + [age, "--encrypt", "--recipients-file", str(keys), "--output", str(encrypted)], + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=diagnostics, + ) + try: + with process.stdin as stream: + with tarfile.open(fileobj=stream, mode="w|gz") as archive: + for path in entries: + archive.add(path, arcname=str(path.relative_to(private)), recursive=False) + code = process.wait(timeout=120) + except BaseException: + process.kill() + process.wait() + raise + if code or not encrypted.is_file(): + raise EvidenceError("Fuzz evidence encryption failed; no artifact was published.") + with encrypted.open("rb") as stream: + if stream.read(len(AGE_HEADER)) != AGE_HEADER: + raise EvidenceError("The evidence output is not an age ciphertext.") + encrypted.chmod(0o600) + os.replace(encrypted, output) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="operation", required=True) + check = commands.add_parser("preflight") + check.add_argument("--age", default="age") + run = commands.add_parser("run") + run.add_argument("--private-dir", required=True, type=Path) + run.add_argument("--target", required=True, choices=sorted(TARGETS)) + run.add_argument("--timeout", required=True, type=int) + run.add_argument("--seed", type=int) + run.add_argument("--corpus", type=Path) + encrypt = commands.add_parser("seal") + encrypt.add_argument("--private-dir", required=True, type=Path) + encrypt.add_argument("--output", required=True, type=Path) + encrypt.add_argument("--age", default="age") + args = parser.parse_args() + os.umask(0o077) + try: + if args.operation == "preflight": + preflight(os.environ.get("FUZZ_EVIDENCE_RECIPIENTS", ""), args.age) + elif args.operation == "run": + code = run_campaign(ROOT, args.private_dir.resolve(), args.target, args.timeout, + seed=args.seed, corpus=args.corpus) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as stream: + stream.write(f"exit_code={code}\n") + # CI validates the captured outcome after evidence has been retained. + print(f"Private campaign completed; exit_code={code}.") + else: + seal(args.private_dir, args.output, + os.environ.get("FUZZ_EVIDENCE_RECIPIENTS", ""), args.age) + except EvidenceError as error: + print(str(error), file=sys.stderr) + return 1 + except Exception: + # Never expose a raw tool error, path or traceback in the public CI log. + print("Private evidence operation failed; inspect the private run directory.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/private_fuzz_evidence_test.py b/scripts/tests/private_fuzz_evidence_test.py new file mode 100644 index 00000000..90615c4a --- /dev/null +++ b/scripts/tests/private_fuzz_evidence_test.py @@ -0,0 +1,168 @@ +"""Exercise real age round trips and fail-closed evidence publication.""" +import contextlib +import io +import json +import os +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from scripts.fuzz import private_evidence as evidence + + +class EvidenceTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + if not shutil.which("age") or not shutil.which("age-keygen"): + raise RuntimeError("private fuzz evidence tests require age and age-keygen") + + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="nilix-private-evidence-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.identity = self.root / "test-identity.txt" + subprocess.run(["age-keygen", "-o", str(self.identity)], check=True, capture_output=True) + self.recipient = subprocess.check_output( + ["age-keygen", "-y", str(self.identity)], text=True, + ).strip() + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + (self.private / "run.json").write_text('{"revision":"fixture","exit_code":1}\n') + (self.private / "libfuzzer.log").write_text("PRIVATE-TRACE-CANARY\n") + (self.private / "findings").mkdir() + (self.private / "findings/crash-one").write_bytes(b"PRIVATE-INPUT-CANARY") + (self.private / "candidate-index.json").write_text( + '{"candidates":[{"candidate_id":"opaque-fixture","input":"findings/crash-one"}]}\n' + ) + self.output = self.root / "public/campaign.tar.gz.age" + + def test_age_round_trip_retains_raw_input_and_its_private_index(self): + evidence.preflight(self.recipient) + evidence.seal(self.private, self.output, self.recipient) + ciphertext = self.output.read_bytes() + self.assertTrue(ciphertext.startswith(evidence.AGE_HEADER)) + self.assertNotIn(b"PRIVATE-INPUT-CANARY", ciphertext) + self.assertEqual(self.output.stat().st_mode & 0o777, 0o600) + plaintext = subprocess.check_output(["age", "-d", "-i", str(self.identity), str(self.output)]) + with tarfile.open(fileobj=io.BytesIO(plaintext), mode="r:gz") as archive: + self.assertEqual(archive.extractfile("findings/crash-one").read(), b"PRIVATE-INPUT-CANARY") + index = json.load(archive.extractfile("candidate-index.json")) + self.assertEqual(index["candidates"][0]["input"], "findings/crash-one") + self.assertEqual(archive.extractfile("libfuzzer.log").read(), b"PRIVATE-TRACE-CANARY\n") + self.assertEqual([path.name for path in self.output.parent.iterdir()], [self.output.name]) + + def test_tampered_ciphertext_fails_authentication(self): + evidence.seal(self.private, self.output, self.recipient) + data = bytearray(self.output.read_bytes()) + data[-1] ^= 1 + self.output.write_bytes(data) + result = subprocess.run(["age", "-d", "-i", str(self.identity), str(self.output)], capture_output=True) + self.assertNotEqual(result.returncode, 0) + + def test_missing_private_or_invalid_recipient_never_passes_preflight(self): + for value in ("", "AGE-SECRET-KEY-1PRIVATE", "age1invalid"): + with self.subTest(value=value), self.assertRaises(evidence.EvidenceError): + evidence.preflight(value) + self.assertFalse(self.output.exists()) + + def test_symlink_and_hardlink_cannot_import_unrelated_files(self): + for kind in ("symbolic", "hard"): + path = self.private / "unrelated" + if kind == "symbolic": + path.symlink_to(self.identity) + else: + os.link(self.identity, path) + with self.subTest(kind=kind), self.assertRaises(evidence.EvidenceError): + evidence.seal(self.private, self.output, self.recipient) + path.unlink() + self.assertFalse(self.output.exists()) + + def test_size_limit_and_incomplete_run_cannot_publish_an_archive(self): + with self.assertRaises(evidence.EvidenceError): + evidence.seal(self.private, self.output, self.recipient, max_bytes=1) + (self.private / "run.json").unlink() + with self.assertRaises(evidence.EvidenceError): + evidence.seal(self.private, self.output, self.recipient) + self.assertFalse(self.output.exists()) + + def test_encryption_error_and_non_ciphertext_output_are_not_published(self): + fake = self.root / "broken-age" + for code in (1, 0): + fake.write_text( + "#!/usr/bin/env python3\nimport pathlib, sys\n" + "pathlib.Path(sys.argv[sys.argv.index('--output')+1]).write_bytes(b'PRIVATE-INPUT-CANARY')\n" + "sys.stdin.buffer.read()\n" + f"raise SystemExit({code})\n" + ) + fake.chmod(0o700) + with self.subTest(code=code), self.assertRaises(evidence.EvidenceError): + evidence.seal(self.private, self.output, self.recipient, age=str(fake)) + self.assertFalse(self.output.exists()) + self.assertEqual(list(self.output.parent.iterdir()), []) + + def test_existing_archive_and_output_inside_private_tree_are_refused(self): + with self.assertRaises(evidence.EvidenceError): + evidence.seal(self.private, self.private / "output.age", self.recipient) + evidence.seal(self.private, self.output, self.recipient) + original = self.output.read_bytes() + with self.assertRaises(evidence.EvidenceError): + evidence.seal(self.private, self.output, self.recipient) + self.assertEqual(self.output.read_bytes(), original) + + def test_campaign_keeps_nonzero_exit_and_raw_diagnostics_out_of_public_stdout(self): + project = self.root / "source" + project.mkdir() + (project / "input.rs").write_text("fixture\n") + subprocess.run(["git", "init", "-q", str(project)], check=True, capture_output=True) + subprocess.run(["git", "add", "input.rs"], cwd=project, check=True) + subprocess.run( + ["git", "-c", "user.name=fixture", "-c", "user.email=fixture@example.invalid", + "commit", "-qm", "fixture"], cwd=project, check=True, + ) + tools = self.root / "tools" + tools.mkdir() + cargo = tools / "cargo" + cargo.write_text( + "#!/usr/bin/env python3\nimport os, pathlib, sys\n" + "if '--version' in sys.argv: print('fixture'); raise SystemExit(0)\n" + "assert 'NILIX_FUZZ_FINGERPRINT_KEY' not in os.environ\n" + "prefix = next(x.split('=',1)[1] for x in sys.argv if x.startswith('-artifact_prefix='))\n" + "pathlib.Path(prefix, 'crash-private').write_bytes(b'PRIVATE-REPRO')\n" + "print('PRIVATE-STACK-TRACE', file=sys.stderr)\n" + "raise SystemExit(77)\n" + ) + cargo.chmod(0o700) + private = self.root / "campaign" + output = io.StringIO() + with patch.dict(os.environ, { + "PATH": str(tools) + os.pathsep + os.environ["PATH"], + "NILIX_FUZZ_FINGERPRINT_KEY": "PRIVATE-SECRET-CANARY", + }), contextlib.redirect_stdout(output): + code = evidence.run_campaign(project, private, "fuzz_syscall", 1) + self.assertEqual(code, 77) + self.assertEqual(output.getvalue(), "") + self.assertIn("PRIVATE-STACK-TRACE", (private / "libfuzzer.log").read_text()) + self.assertEqual((private / "findings/crash-private").read_bytes(), b"PRIVATE-REPRO") + metadata = json.loads((private / "run.json").read_text()) + self.assertEqual(metadata["exit_code"], 77) + self.assertEqual(metadata["source_sha256"]["input.rs"], evidence.digest(project / "input.rs")) + self.assertNotIn("PRIVATE-SECRET-CANARY", (private / "run.json").read_text()) + with self.assertRaises(FileExistsError): + evidence.run_campaign(project, private, "fuzz_syscall", 1) + + def test_qemu_campaign_keeps_libfuzzer_timeout_above_guest_bound(self): + qemu = evidence.campaign_command(self.private, "fuzz_syscall_qemu", 60) + self.assertIn("--features", qemu) + self.assertIn("qemu-executor", qemu) + self.assertIn("-timeout=1200", qemu) + host = evidence.campaign_command(self.private, "fuzz_syscall", 60) + self.assertNotIn("--features", host) + self.assertNotIn("-timeout=1200", host) + + +if __name__ == "__main__": + unittest.main() diff --git a/userspace/fuzzer/Cargo.toml b/userspace/fuzzer/Cargo.toml index c2340efa..fd7195c6 100644 --- a/userspace/fuzzer/Cargo.toml +++ b/userspace/fuzzer/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition = "2021" license = "AGPL-3.0-or-later" +# Standalone host tool — not part of the kernel or userspace workspaces. +[workspace] + [[bin]] name = "fuzz-pipeline-smoke" path = "src/bin/continuous-fuzzer.rs" @@ -16,8 +19,6 @@ path = "src/bin/crash-triage.rs" clap = { version = "4.5", features = ["derive"] } hmac = "0.13" sha2 = "0.11" - -[dev-dependencies] serde_json = "1" [profile.release] diff --git a/userspace/fuzzer/src/bin/crash-triage.rs b/userspace/fuzzer/src/bin/crash-triage.rs index c267930f..14fba07d 100644 --- a/userspace/fuzzer/src/bin/crash-triage.rs +++ b/userspace/fuzzer/src/bin/crash-triage.rs @@ -51,6 +51,10 @@ struct Args { #[clap(long)] output_dir: PathBuf, + /// Private candidate-to-input index; must be outside the public output directory + #[clap(long)] + private_index: Option, + /// Finding count observed by the producer (required cross-check) #[clap(long)] expected_findings: usize, @@ -64,6 +68,8 @@ struct Args { struct Candidate { file_path: PathBuf, candidate_id: String, + target: String, + kind: &'static str, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -116,6 +122,9 @@ fn run(args: Args, key: &[u8]) -> TriageResult { candidates }; + if let Some(path) = &args.private_index { + write_private_index(path, &args.output_dir, &args.crash_dir, &final_candidates)?; + } for (index, candidate) in final_candidates.iter().enumerate() { write_candidate_report(&args.output_dir, index + 1, &candidate.candidate_id)?; } @@ -221,6 +230,8 @@ fn collect_candidates(root: &Path, key: &[u8]) -> TriageResult> { candidates.push(Candidate { file_path: entry.path(), candidate_id, + target: target.to_owned(), + kind, }); } } @@ -295,6 +306,49 @@ fn write_candidate_report(output_dir: &Path, index: usize, candidate_id: &str) - Ok(()) } +fn write_private_index( + path: &Path, + public_dir: &Path, + input_root: &Path, + candidates: &[Candidate], +) -> TriageResult<()> { + let parent = path + .parent() + .ok_or_else(|| invalid_layout("private index has no parent"))?; + fs::create_dir_all(parent)?; + if fs::canonicalize(parent)?.starts_with(fs::canonicalize(public_dir)?) { + return Err(invalid_layout( + "private index must not enter the public output directory", + )); + } + let entries = candidates + .iter() + .map(|candidate| { + Ok(serde_json::json!({ + "candidate_id": candidate.candidate_id, + "target": candidate.target, + "kind": candidate.kind, + "input": candidate.file_path.strip_prefix(input_root)?, + })) + }) + .collect::>>()?; + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options.open(path)?; + serde_json::to_writer_pretty( + file, + &serde_json::json!({ + "schema": "nilix-private-candidates-v1", "candidates": entries, + }), + )?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -339,6 +393,7 @@ mod tests { Args { crash_dir: root.join("input"), output_dir: root.join("output"), + private_index: None, expected_findings, dedup: true, } @@ -361,6 +416,58 @@ mod tests { assert!(validate_key(&[0; MIN_KEY_BYTES]).is_ok()); } + #[test] + fn private_index_maps_public_ids_to_retained_inputs_without_changing_public_schema() { + let directory = TestDir::new("private-index"); + let root = directory.path(); + write_artifact( + &root.join("input"), + "fuzz_syscall_qemu", + "crash-one", + b"private input", + ); + let mut options = args(root, 1); + let index = root.join("private/candidate-index.json"); + options.private_index = Some(index.clone()); + run(options, TEST_KEY).unwrap(); + let data: serde_json::Value = serde_json::from_slice(&fs::read(&index).unwrap()).unwrap(); + let entry = &data["candidates"][0]; + assert_eq!( + entry["input"], + "cargo-fuzz-crashes-fuzz_syscall_qemu/crash-one" + ); + assert_eq!(entry["kind"], "crash"); + assert_eq!(entry["target"], "fuzz_syscall_qemu"); + let report = fs::read_to_string(root.join("output/candidate-001.txt")).unwrap(); + assert_eq!(report.lines().count(), 2); + assert!(report.contains(entry["candidate_id"].as_str().unwrap())); + assert!(!report.contains("private input")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(index).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + } + + #[test] + fn private_index_cannot_be_written_into_public_artifacts() { + let directory = TestDir::new("private-index-public"); + let root = directory.path(); + write_artifact( + &root.join("input"), + "fuzz_syscall_qemu", + "crash-one", + b"private input", + ); + let mut options = args(root, 1); + options.private_index = Some(root.join("output/candidate-index.json")); + assert!(run(options, TEST_KEY).is_err()); + assert_eq!(fs::read_dir(root.join("output")).unwrap().count(), 0); + } + #[test] fn qemu_findings_produce_private_target_scoped_candidates() { let temp = TestDir::new("qemu"); diff --git a/userspace/nilix-syz-fuzzer/src/executor.rs b/userspace/nilix-syz-fuzzer/src/executor.rs index 498fcd1f..dd947ac7 100644 --- a/userspace/nilix-syz-fuzzer/src/executor.rs +++ b/userspace/nilix-syz-fuzzer/src/executor.rs @@ -13,7 +13,8 @@ use std::time::{Duration, Instant}; use crate::disk::{ensure_qemu_safe_path, Ext3Tools, Ext3Transport}; use crate::program::SyscallProgram; use crate::protocol::{ - constant_time_eq, decode_result, encode_program, ExecutionIdentity, ProgramBinding, + constant_time_eq, decode_result, encode_program, EncodedProgram, ExecutionIdentity, + ProgramBinding, }; const MAX_SERIAL_LOG: u64 = 4 * 1024 * 1024; @@ -46,6 +47,7 @@ pub struct QemuExecutor { timeout: Duration, transport: Ext3Transport, artifact_dir: Option, + retain_successes: bool, } impl QemuExecutor { @@ -87,6 +89,7 @@ impl QemuExecutor { timeout: Duration::from_secs(timeout_secs), transport: Ext3Transport::new(tools, disk_mib)?, artifact_dir: None, + retain_successes: true, }) } @@ -94,6 +97,15 @@ impl QemuExecutor { /// validation run. Normal campaigns keep the default temporary cleanup. pub fn with_artifact_dir(mut self, directory: PathBuf) -> Self { self.artifact_dir = Some(directory); + self.retain_successes = true; + self + } + + /// Keep failed executions (including host/protocol errors) for private + /// triage, while reclaiming successful guest disks during long campaigns. + pub fn with_failure_artifact_dir(mut self, directory: PathBuf) -> Self { + self.artifact_dir = Some(directory); + self.retain_successes = false; self } @@ -115,20 +127,33 @@ impl QemuExecutor { } .context("failed to create per-execution temporary directory")?; let work_dir = temp_dir.path().to_path_buf(); - let _auto_cleanup = if self.artifact_dir.is_some() { + let result = self.execute_in(&encoded, &work_dir); + if self.artifact_dir.is_some() + && (self.retain_successes || !matches!(result, Ok(ExecutionResult::Success(_)))) + { + // Keep the raw evidence even if writing the diagnostic receipt fails. let _ = temp_dir.keep(); + std::fs::write(work_dir.join("program.json"), serde_json::to_vec(program)?) + .context("failed to retain the submitted syscall program")?; + let receipt = execution_receipt(&result, &encoded.binding, &work_dir); + std::fs::write( + work_dir.join("outcome.json"), + serde_json::to_vec_pretty(&receipt)?, + ) + .context("failed to write private execution outcome")?; eprintln!("NILIX_SYZ_ARTIFACTS {}", work_dir.display()); - None - } else { - Some(temp_dir) - }; + } + result + } + + fn execute_in(&self, encoded: &EncodedProgram, work_dir: &Path) -> Result { let serial_path = work_dir.join("serial.log"); let stderr_path = work_dir.join("qemu.stderr"); File::create(&serial_path).context("failed to create serial log")?; let stderr_file = File::create(&stderr_path).context("failed to create QEMU stderr log")?; let disk_path = self .transport - .prepare(&work_dir, &encoded.bytes) + .prepare(work_dir, &encoded.bytes) .context("failed to prepare fresh syz Ext3 transport")?; let source_esp = self @@ -180,7 +205,7 @@ impl QemuExecutor { match observation { Observation::Pass(marker) => { - let result_bytes = match self.transport.extract_result(&disk_path, &work_dir) { + let result_bytes = match self.transport.extract_result(&disk_path, work_dir) { Ok(bytes) => bytes, Err(error) => { return Ok(ExecutionResult::Crash(CrashInfo { @@ -227,6 +252,36 @@ impl QemuExecutor { } } +fn execution_receipt( + result: &Result, + binding: &ProgramBinding, + work_dir: &Path, +) -> serde_json::Value { + let (outcome, classification) = match result { + Ok(ExecutionResult::Success(_)) => ("success", String::new()), + Ok(ExecutionResult::Crash(info)) => ("crash", info.classification.clone()), + Ok(ExecutionResult::Timeout) => ("boot_timeout", String::new()), + Ok(ExecutionResult::Hang) => ("guest_hang", String::new()), + // An execution error is not, by itself, proof of a kernel defect. + Err(error) => ("execution_error", format!("{error:#}")), + }; + let guest_began = SerialMonitor::open(&work_dir.join("serial.log"), binding) + .and_then(|mut serial| { + serial.poll()?; + Ok(serial.tracker.began) + }) + .ok(); + serde_json::json!({ + "schema": "nilix-private-execution-v1", + "sequence": binding.sequence_hex(), + "run": binding.run_hex(), + "program": binding.program_hex(), + "outcome": outcome, + "classification": classification, + "guest_began": guest_began, + }) +} + #[derive(Debug)] enum Observation { Pass(PassMarker), @@ -636,9 +691,9 @@ fn qemu_args(ovmf: &Path, esp_dir: &Path, disk: &Path, serial: &Path) -> Result< // the separate result disk must remain persistent for extraction. format!("format=raw,file=fat:{esp_dir},snapshot=on"), "-drive".into(), - // Writeback matches the block path used by the kernel runtime gates - // and avoids QEMU 8.x direct-I/O alignment differences on Ext3. - format!("if=none,file={disk},format=raw,id=syzdisk,cache=writeback"), + // Writeback matches the block path used by the kernel runtime gates + // and avoids QEMU 8.x direct-I/O alignment differences on Ext3. + format!("if=none,file={disk},format=raw,id=syzdisk,cache=writeback"), "-device".into(), "virtio-blk-pci,drive=syzdisk".into(), "-m".into(), @@ -1039,7 +1094,7 @@ mod tests { } #[test] - #[cfg(unix)] + #[cfg(unix)] fn serial_offsets_are_exact_and_fatal_evidence_survives_tail_eviction() { use std::io::Write; let binding = binding(); @@ -1082,7 +1137,7 @@ mod tests { .unwrap(); assert!(args .iter() - .any(|arg| arg.contains("id=syzdisk,cache=writeback"))); + .any(|arg| arg.contains("id=syzdisk,cache=writeback"))); assert!(args.windows(2).any(|pair| pair == ["-nic", "none"])); assert!(args.iter().any(|arg| arg == "virtio-blk-pci,drive=syzdisk")); assert!(!args.iter().any(|arg| arg.contains("virtio-serial")));