fix(fuzz): retain encrypted private campaign evidence

.github/workflows/ci.yml

@@ -72,6 +72,7 @@ jobs:
         with:
           python-version: '3.12'
       - run: python3 -m pip install -r requirements-ci.txt
+      - run: sudo apt-get update && sudo apt-get install -y age
       - run: bash scripts/ci/entrypoint.sh quality
       - name: Check Actions definitions
         if: always()

.github/workflows/fuzz.yml

@@ -78,7 +78,12 @@ jobs:
       - name: Install QEMU and build dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y qemu-system-x86 musl-tools
+          sudo apt-get install -y qemu-system-x86 musl-tools age
+
+      - name: Verify private evidence encryption
+        env:
+          FUZZ_EVIDENCE_RECIPIENTS: ${{ vars.FUZZ_EVIDENCE_RECIPIENTS }}
+        run: python3 scripts/fuzz/private_evidence.py preflight
 
       # Build before any secret is present so dependency build scripts never
       # inherit the private fingerprinting key.
@@ -102,48 +107,23 @@ jobs:
       - name: Prepare fuzz directories
         run: |
           mkdir -p fuzz/corpus/${{ matrix.target }}
-          mkdir -p fuzz/artifacts/${{ matrix.target }}
 
       # Raw libFuzzer output can include Base64 reproducers and stack traces.
-      # Keep it only in RUNNER_TEMP; never tee, cat, summarize, or upload it.
+      # Keep plaintext in RUNNER_TEMP. Upload only the sealed age archive.
       - name: Run fuzzer privately
         id: run_fuzzer
         env:
           FUZZ_TIMEOUT: ${{ inputs.timeout || '900' }}
         run: |
-          timeout="$FUZZ_TIMEOUT"
-          if ! [[ "$timeout" =~ ^[1-9][0-9]*$ ]] || [ "$timeout" -gt 1200 ]; then
-            echo "::error::Fuzz timeout must be an integer from 1 through 1200"
-            exit 1
-          fi
-
-          private_log="$RUNNER_TEMP/libfuzzer-output.log"
-          set +e
-
-          # QEMU fuzzer needs the qemu-executor feature flag
-          if [ "${{ matrix.target }}" = "fuzz_syscall_qemu" ]; then
-            cargo +nightly-2025-12-08 fuzz run ${{ matrix.target }} --features qemu-executor -- \
-              -max_total_time="$timeout" \
-              -rss_limit_mb=4096 \
-              -print_final_stats=1 \
-              > "$private_log" 2>&1
-          else
-            cargo +nightly-2025-12-08 fuzz run ${{ matrix.target }} -- \
-              -max_total_time="$timeout" \
-              -rss_limit_mb=4096 \
-              -print_final_stats=1 \
-              > "$private_log" 2>&1
-          fi
-
-          exit_code=$?
-          set -e
-          echo "exit_code=${exit_code}" >> "$GITHUB_OUTPUT"
+          python3 scripts/fuzz/private_evidence.py run \
+            --private-dir "$RUNNER_TEMP/nilix-fuzz-evidence" \
+            --target "${{ matrix.target }}" --timeout "$FUZZ_TIMEOUT"
 
       - name: Count private findings
         if: always()
         id: count_findings
         run: |
-          findings=$(find fuzz/artifacts/${{ matrix.target }} -maxdepth 1 -type f \
+          findings=$(find "$RUNNER_TEMP/nilix-fuzz-evidence/findings" -maxdepth 1 -type f \
             \( -name 'crash-*' -o -name 'timeout-*' -o -name 'oom-*' -o -name 'leak-*' \) \
             2>/dev/null | wc -l)
           echo "findings=${findings}" >> "$GITHUB_OUTPUT"
@@ -178,12 +158,13 @@ jobs:
             exit 1
           fi
 
-          input_root="$RUNNER_TEMP/nilix-finding-input"
+          umask 077
+          input_root="$RUNNER_TEMP/nilix-fuzz-evidence/finding-input"
           input_dir="$input_root/cargo-fuzz-crashes-${{ matrix.target }}"
           candidate_dir="fuzz/candidates"
-          private_triage_log="$RUNNER_TEMP/candidate-triage.log"
+          private_triage_log="$RUNNER_TEMP/nilix-fuzz-evidence/triage.log"
           mkdir -p "$input_dir" "$candidate_dir"
-          if ! find fuzz/artifacts/${{ matrix.target }} -maxdepth 1 -type f \
+          if ! find "$RUNNER_TEMP/nilix-fuzz-evidence/findings" -maxdepth 1 -type f \
             \( -name 'crash-*' -o -name 'timeout-*' -o -name 'oom-*' -o -name 'leak-*' \) \
             -exec cp -- {} "$input_dir/" \; \
             > /dev/null 2> "$private_triage_log"; then
@@ -194,6 +175,7 @@ jobs:
           if ! userspace/fuzzer/target/x86_64-unknown-linux-gnu/release/crash-triage \
             --crash-dir "$input_root" \
             --output-dir "$candidate_dir" \
+            --private-index "$RUNNER_TEMP/nilix-fuzz-evidence/candidate-index.json" \
             --expected-findings "$FUZZ_FINDINGS" \
             --dedup \
             >> "$private_triage_log" 2>&1; then
@@ -217,8 +199,31 @@ jobs:
           retention-days: 90
           if-no-files-found: error
 
+      # Preserve evidence even when the fuzzer/triage failed without a candidate.
+      # A missing recipient, seal failure or upload failure makes the campaign
+      # incomplete and prevents the reporting job from creating an empty pointer.
+      - name: Seal private campaign evidence
+        id: seal_evidence
+        if: always() && steps.run_fuzzer.outcome != 'skipped'
+        env:
+          FUZZ_EVIDENCE_RECIPIENTS: ${{ vars.FUZZ_EVIDENCE_RECIPIENTS }}
+        run: |
+          python3 scripts/fuzz/private_evidence.py seal \
+            --private-dir "$RUNNER_TEMP/nilix-fuzz-evidence" \
+            --output fuzz/evidence/campaign.tar.gz.age
+
+      - name: Upload encrypted campaign evidence
+        id: upload_evidence
+        if: always() && steps.seal_evidence.outcome == 'success'
+        uses: actions/upload-artifact@v7
+        with:
+          name: fuzz-evidence-${{ strategy.job-index }}
+          path: fuzz/evidence/campaign.tar.gz.age
+          retention-days: 90
+          if-no-files-found: error
+
       - name: Save clean corpus
-        if: steps.validate_fuzzer.outcome == 'success' && steps.count_findings.outputs.findings == '0'
+        if: steps.validate_fuzzer.outcome == 'success' && steps.count_findings.outputs.findings == '0' && steps.upload_evidence.outcome == 'success'
         uses: actions/cache/save@v6
         with:
           path: fuzz/corpus/${{ matrix.target }}/
@@ -232,12 +237,14 @@ jobs:
           COUNT_OUTCOME: ${{ steps.count_findings.outcome }}
           VALIDATE_OUTCOME: ${{ steps.validate_fuzzer.outcome }}
           CANDIDATE_OUTCOME: ${{ steps.create_candidates.outcome }}
+          SEAL_OUTCOME: ${{ steps.seal_evidence.outcome }}
+          EVIDENCE_UPLOAD_OUTCOME: ${{ steps.upload_evidence.outcome }}
           FINDINGS: ${{ steps.count_findings.outputs.findings }}
           CANDIDATE_COUNT: ${{ steps.create_candidates.outputs.candidate_count }}
         run: |
           status=completed
           candidate_count=0
-          if [ "$RUN_OUTCOME" != "success" ] || [ "$COUNT_OUTCOME" != "success" ] || [ "$VALIDATE_OUTCOME" != "success" ]; then
+          if [ "$RUN_OUTCOME" != "success" ] || [ "$COUNT_OUTCOME" != "success" ] || [ "$VALIDATE_OUTCOME" != "success" ] || [ "$SEAL_OUTCOME" != "success" ] || [ "$EVIDENCE_UPLOAD_OUTCOME" != "success" ]; then
             status=incomplete
           elif ! [[ "$FINDINGS" =~ ^[0-9]+$ ]]; then
             status=incomplete
@@ -326,11 +333,11 @@ jobs:
                 gh issue reopen "$issue_number"
               fi
               gh issue comment "$issue_number" \
-                --body "The opaque candidate was observed again in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) at head ${{ github.sha }}. Details remain private under SECURITY.md."
+                --body "The opaque candidate was observed again in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) at head ${{ github.sha }}. Encrypted evidence is retained in the run's fuzz-evidence artifacts; details remain private under SECURITY.md."
               continue
             fi
 
-            issue_body="Automated fuzzing observed a private candidate in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). Potential security details and the reproducer were not written to public logs or artifacts. Maintainers should coordinate under SECURITY.md. Opaque candidate ID: ${candidate_id}."
+            issue_body="Automated fuzzing observed a private candidate in [workflow run ${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}). Encrypted evidence is retained in the run's fuzz-evidence artifacts. Plaintext security details and the reproducer are not public. Maintainers should decrypt the archive and inspect candidate-index.json and guest outcome.json before classifying the finding under SECURITY.md. Opaque candidate ID: ${candidate_id}."
             issue_body+=$'\n\n'
             issue_body+="$marker"
             gh issue create \

.gitignore

@@ -29,6 +29,7 @@ userspace/clippy-userspace-target
 # Fuzz testing
 /fuzz/target/
 /fuzz/corpus/
+/fuzz/evidence/
 /fuzz/artifacts/*.log
 /hosted-subcrate-review-target/
 

CONTRIBUTING.md

@@ -43,10 +43,11 @@ the toolchain installed work too.
 - **QEMU + OVMF** — `qemu-system-x86_64` and the OVMF UEFI firmware (for boot/run).
 - **GNU Make**.
 - **musl toolchain** — `musl-tools` (`musl-gcc`), only for the musl conformance gate.
+- **age** — only for the private fuzz evidence harness tests (`scripts/tests/private_fuzz_evidence_test.py`).
 
 On Debian/Ubuntu the non-Rust deps match what CI installs:
 ```bash
-sudo apt-get install -y qemu-system-x86 ovmf musl-tools make
+sudo apt-get install -y qemu-system-x86 ovmf musl-tools make age
 ```
 
 > Maintainer note: the project is also developed from a Windows mirror that has **no**

Makefile

@@ -1151,7 +1151,7 @@ fuzz-qemu-campaign: build-fuzz-qemu-deps fuzz-qemu-seeds
 	cd fuzz && \
 	cargo +nightly fuzz run fuzz_syscall_qemu --features qemu-executor -- \
 		-max_total_time=3600 \
-		-timeout=120 \
+		-timeout=1200 \
 		-rss_limit_mb=4096 \
 		-print_final_stats=1
 
@@ -1161,7 +1161,7 @@ fuzz-qemu-overnight: build-fuzz-qemu-deps fuzz-qemu-seeds
 	cd fuzz && \
 	cargo +nightly fuzz run fuzz_syscall_qemu --features qemu-executor -- \
 		-max_total_time=28800 \
-		-timeout=120 \
+		-timeout=1200 \
 		-rss_limit_mb=8192 \
 		-print_final_stats=1
 
@@ -1171,7 +1171,7 @@ fuzz-qemu-parallel: build-fuzz-qemu-deps fuzz-qemu-seeds
 	cd fuzz && \
 	cargo +nightly fuzz run fuzz_syscall_qemu --features qemu-executor --jobs=4 -- \
 		-max_total_time=3600 \
-		-timeout=120 \
+		-timeout=1200 \
 		-rss_limit_mb=4096 \
 		-print_final_stats=1
 

fuzz/README.md

@@ -42,11 +42,50 @@ nonzero coverage checks for the seeds that prove the KCOV data path. There is no
 measured executions-per-second claim.
 
 Build with `make build-syz-kcov` and enable `qemu-executor`. Guest startup has a
-90-second bound; use `-timeout=120` or larger for libFuzzer. Optional
+900-second bound; keep libFuzzer's per-input timeout above that bound (the private
+campaign runner passes `-timeout=1200` for this target). Optional
 `NILIX_FUZZ_KERNEL`, `NILIX_FUZZ_QEMU`, `OVMF_PATH` and `NILIX_FUZZ_ARTIFACTS`
 override the image, emulator, firmware and retained-evidence directory. Artifact
 retention is intended for finite validation runs because it keeps each guest disk.
 
+For campaigns, use `NILIX_FUZZ_FAILURE_ARTIFACTS` instead. It retains only failed
+executions, including execution/protocol errors, and removes successful guest
+disks. Each retained directory includes the submitted program, private guest
+image, disk, serial output, QEMU stderr and `outcome.json`. The receipt distinguishes
+crashes (with their observed classification), boot timeout, guest hang and
+execution errors; an execution error alone does not establish a kernel defect.
+Do not set both artifact-retention variables.
+
+### Private campaign evidence
+
+The scheduled workflow requires the repository variable `FUZZ_EVIDENCE_RECIPIENTS`
+to contain one or more native age public keys, one per line. Generate and retain
+the corresponding private identities on a maintainer-controlled machine. Never
+put those private identities in CI or in this repository. The workflow verifies
+the public recipients before running a fuzzer.
+
+`scripts/fuzz/private_evidence.py run` records the exact command, source revision
+and hashes, source changes, guest-image hashes, tool versions, exit status and raw
+libFuzzer output in a private directory. Findings use a fresh private artifact
+prefix. QEMU failures also retain their per-execution evidence. The triage tool's
+`--private-index` maps public candidate IDs to the inputs inside that directory;
+the public candidate format remains two fields and contains no reproducer.
+
+After the campaign, `private_evidence.py seal` streams a tar/gzip archive directly
+into age. Only `campaign.tar.gz.age` is uploaded, under `fuzz-evidence-<index>`.
+Raw files are never uploaded or cached. Encryption or upload failure makes the
+campaign incomplete and prevents automatic issue creation. The plaintext archive
+size is bounded to 1 GiB; successful execution disks are cleaned up. Archives are
+retained for 90 days, so maintainers should copy relevant ciphertext before expiry.
+
+Decrypt on a trusted machine with `age -d -i /path/to/identity.agekey -o evidence.tar.gz
+campaign.tar.gz.age`. Keep the decrypted evidence private under `SECURITY.md`.
+Inspect `run.json`, `candidate-index.json` and `guests/*/outcome.json` before replay.
+The index's `input` paths are relative to `finding-input/`; the copy preserves the
+exact libFuzzer input bytes. Candidate IDs deduplicate inputs, not root causes:
+different IDs can still refer to the same defect. Historical ID-only reports cannot
+recover their lost inputs.
+
 ### 2. `fuzz_vfs_path` - VFS Path Operations
 Tests filesystem operations:
 - **Path traversal** attacks (../, multiple slashes)

fuzz/src/qemu_executor.rs

@@ -9,7 +9,7 @@ pub use nilix_syz_fuzzer::executor::{CrashInfo, ExecutionResult, QemuExecutor};
 pub use nilix_syz_fuzzer::program::{Argument, Syscall, SyscallProgram};
 use std::path::PathBuf;
 
-pub const GUEST_TIMEOUT_SECS: u64 = 900;
+pub const GUEST_TIMEOUT_SECS: u64 = 900;
 
 // Same non-destructive contract validated by the shared program/guest protocol.
 const SYSCALLS: &[u32] = &[
@@ -43,10 +43,17 @@ pub fn configured_executor() -> Result<QemuExecutor> {
         Ext3Tools::default(),
     )
     .context("QEMU executor setup failed; build the guest with make build-syz-kcov")?;
-    Ok(match std::env::var_os("NILIX_FUZZ_ARTIFACTS") {
-        Some(root) => executor.with_artifact_dir(PathBuf::from(root)),
-        None => executor,
-    })
+    Ok(
+        match (
+            std::env::var_os("NILIX_FUZZ_ARTIFACTS"),
+            std::env::var_os("NILIX_FUZZ_FAILURE_ARTIFACTS"),
+        ) {
+            (Some(root), None) => executor.with_artifact_dir(PathBuf::from(root)),
+            (None, Some(root)) => executor.with_failure_artifact_dir(PathBuf::from(root)),
+            (None, None) => executor,
+            (Some(_), Some(_)) => anyhow::bail!("choose one artifact retention mode"),
+        },
+    )
 }
 
 /// count:u8, then (syscall-index:u8, argc:u8, arguments...). Argument tags:
@@ -199,11 +206,28 @@ mod tests {
             16,
             Ext3Tools::default(),
         )
-        .unwrap();
+        .unwrap()
+        .with_failure_artifact_dir(root.join("failures"));
         assert!(missing
             .execute(&program)
             .unwrap_err()
             .to_string()
             .contains("failed to spawn"));
+        let failures: Vec<_> = std::fs::read_dir(root.join("failures"))
+            .unwrap()
+            .map(|entry| entry.unwrap().path())
+            .collect();
+        assert_eq!(failures.len(), 1);
+        let receipt: serde_json::Value =
+            serde_json::from_slice(&std::fs::read(failures[0].join("outcome.json")).unwrap())
+                .unwrap();
+        assert_eq!(receipt["outcome"], "execution_error");
+        assert_eq!(receipt["guest_began"], false);
+        assert!(receipt["classification"]
+            .as_str()
+            .unwrap()
+            .contains("failed to spawn"));
+        assert!(failures[0].join("program.json").is_file());
+        assert!(failures[0].join("syz-program.host.bin").is_file());
     }
 }