From 02cbe2164b62eb136ba7168ef0e1fa2be45c2c2f Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 3 Sep 2026 13:55:42 +0100 Subject: [PATCH] Streamline fuzzer workflow Signed-off-by: Robert Kruszewski --- .github/workflows/build-fuzzers.yml | 85 ++++++++++ .github/workflows/fuzz-regression.yml | 46 ++++++ .github/workflows/fuzz.yml | 45 +++++- .github/workflows/minimize_fuzz_corpus.yml | 32 +++- .../minimize_fuzz_corpus_workflow.yml | 117 +++++++++++--- .github/workflows/run-fuzzer.yml | 150 +++++++++++++----- .github/workflows/validate-fuzz-corpora.yml | 136 ++++++++++++++++ scripts/checkpoint-fuzz-corpus.sh | 82 ++++++++++ scripts/s3-download.py | 50 +++++- scripts/s3-upload.py | 90 ++++++----- 10 files changed, 719 insertions(+), 114 deletions(-) create mode 100644 .github/workflows/build-fuzzers.yml create mode 100644 .github/workflows/fuzz-regression.yml create mode 100644 .github/workflows/validate-fuzz-corpora.yml create mode 100755 scripts/checkpoint-fuzz-corpus.sh diff --git a/.github/workflows/build-fuzzers.yml b/.github/workflows/build-fuzzers.yml new file mode 100644 index 00000000000..1cc2b068a70 --- /dev/null +++ b/.github/workflows/build-fuzzers.yml @@ -0,0 +1,85 @@ +name: Build CPU Fuzzers + +on: + workflow_call: + inputs: + runner: + description: "Runner name from .github-private runs-on.yml" + required: false + type: string + default: "arm64-medium" + outputs: + artifact_name: + description: "Artifact containing the compiled CPU fuzzers" + value: ${{ jobs.build.outputs.artifact_name }} + +env: + NIGHTLY_TOOLCHAIN: nightly-2026-02-05 + +jobs: + build: + name: "Build CPU fuzzers" + timeout-minutes: 120 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner={1}/disk=large/tag=fuzz-build', github.run_id, inputs.runner) + || 'ubuntu-latest' }} + outputs: + artifact_name: ${{ steps.artifact.outputs.name }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: ./.github/actions/setup-prebuild + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} + components: "" + enable-sccache: "false" + + - name: Install cargo-fuzz + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 + with: + tool: cargo-fuzz + + - name: Build fuzzers + shell: bash + run: | + set -euo pipefail + mkdir -p fuzz-binaries + + cargo +$NIGHTLY_TOOLCHAIN fuzz build --release --debug-assertions + + copy_fuzzer() { + local target=$1 + local output_name=${2:-$target} + local binary + binary=$(find target -type f -path "*/release/$target" -perm -u+x -print -quit) + if [ -z "$binary" ]; then + echo "::error::Unable to find compiled fuzzer $target" + exit 1 + fi + install -m 755 "$binary" "fuzz-binaries/$output_name" + } + + for target in array_ops compress_roundtrip file_io fsst_like row_encode; do + copy_fuzzer "$target" + done + + cargo +$NIGHTLY_TOOLCHAIN fuzz build --release --debug-assertions \ + --features vortex/unstable_encodings array_ops + copy_fuzzer array_ops array_ops_unstable_encodings + + tar -acf cpu-fuzzers.tar.zst fuzz-binaries + + - name: Set artifact name + id: artifact + shell: bash + run: echo "name=cpu-fuzzers-${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT" + + - name: Upload fuzzers + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.artifact.outputs.name }} + path: cpu-fuzzers.tar.zst + compression-level: 0 + retention-days: 2 diff --git a/.github/workflows/fuzz-regression.yml b/.github/workflows/fuzz-regression.yml new file mode 100644 index 00000000000..7147bbc4864 --- /dev/null +++ b/.github/workflows/fuzz-regression.yml @@ -0,0 +1,46 @@ +name: Validate Fuzz Seeds + +concurrency: + group: fuzz-regression + cancel-in-progress: true + +on: + push: + branches: + - develop + paths: + - ".cargo/**" + - "**/*.c" + - "**/*.cpp" + - "**/*.cu" + - "**/*.cuh" + - "**/*.h" + - "**/*.hpp" + - "**/*.proto" + - "**/*.rs" + - "**/Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "fuzz/**" + - ".github/actions/setup-prebuild/**" + - ".github/actions/setup-rust/**" + - ".github/workflows/build-fuzzers.yml" + - ".github/workflows/fuzz-regression.yml" + - ".github/workflows/validate-fuzz-corpora.yml" + - "scripts/s3-download.py" + workflow_dispatch: { } + +jobs: + build_cpu_fuzzers: + name: "Build CPU fuzzers" + uses: ./.github/workflows/build-fuzzers.yml + + validate_existing_seeds: + name: "Validate existing seeds" + needs: build_cpu_fuzzers + uses: ./.github/workflows/validate-fuzz-corpora.yml + with: + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} + secrets: + R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 2b339e7c1db..c64f54f91ac 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -1,27 +1,31 @@ name: Fuzz concurrency: - # The group causes runs to queue instead of running in parallel. + # Exploration and minimization update the same corpus objects. group: fuzz - # This ensures each run builds on the previous run's corpus discoveries rather than losing them to - # failed compare-and-swap uploads. cancel-in-progress: false on: schedule: - - cron: "0 */6 * * *" # every 6 hours + - cron: "0 */3 * * *" # every 3 hours workflow_dispatch: { } jobs: + build_cpu_fuzzers: + name: "Build CPU fuzzers" + uses: ./.github/workflows/build-fuzzers.yml + # ============================================================================ # IO Fuzzer # ============================================================================ io_fuzz: name: "IO Fuzz" + needs: build_cpu_fuzzers uses: ./.github/workflows/run-fuzzer.yml with: fuzz_target: file_io jobs: 4 + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} @@ -70,10 +74,12 @@ jobs: # ============================================================================ ops_fuzz: name: "Array Operations Fuzz" + needs: build_cpu_fuzzers uses: ./.github/workflows/run-fuzzer.yml with: fuzz_target: array_ops jobs: 4 + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} @@ -106,6 +112,7 @@ jobs: # ============================================================================ ops_fuzz_unstable: name: "Array Operations Fuzz (unstable)" + needs: build_cpu_fuzzers uses: ./.github/workflows/run-fuzzer.yml with: fuzz_target: array_ops @@ -113,6 +120,7 @@ jobs: extra_features: "vortex/unstable_encodings" extra_env: "VORTEX_EXPERIMENTAL_PATCHED_ARRAY=1" jobs: 4 + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} @@ -146,10 +154,12 @@ jobs: # ============================================================================ fsst_like_fuzz: name: "FSST LIKE Fuzz" + needs: build_cpu_fuzzers uses: ./.github/workflows/run-fuzzer.yml with: fuzz_target: fsst_like jobs: 4 + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} @@ -182,10 +192,12 @@ jobs: # ============================================================================ row_encode_fuzz: name: "Row Encoding Fuzz" + needs: build_cpu_fuzzers uses: ./.github/workflows/run-fuzzer.yml with: fuzz_target: row_encode jobs: 4 + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} @@ -218,14 +230,39 @@ jobs: # ============================================================================ compress_fuzz: name: "Compress Roundtrip Fuzz" + needs: build_cpu_fuzzers uses: ./.github/workflows/run-fuzzer.yml with: fuzz_target: compress_roundtrip jobs: 4 + fuzzer_artifact: ${{ needs.build_cpu_fuzzers.outputs.artifact_name }} secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + report-compress-fuzz-failures: + name: "Report Compress Roundtrip Fuzz Failures" + needs: compress_fuzz + if: always() && needs.compress_fuzz.outputs.crashes_found == 'true' + permissions: + issues: write + contents: read + id-token: write + pull-requests: read + uses: ./.github/workflows/report-fuzz-crash.yml + with: + fuzz_target: compress_roundtrip + crash_file: ${{ needs.compress_fuzz.outputs.first_crash_name }} + artifact_url: ${{ needs.compress_fuzz.outputs.artifact_url }} + artifact_name: compress_roundtrip-crash-artifacts + logs_artifact_name: compress_roundtrip-logs + branch: ${{ github.ref_name }} + commit: ${{ github.sha }} + secrets: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + incident_io_alert_token: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} + # ============================================================================ # GPU Compress Fuzzer (CUDA) # ============================================================================ diff --git a/.github/workflows/minimize_fuzz_corpus.yml b/.github/workflows/minimize_fuzz_corpus.yml index d4a963a6f62..c5836eb87a6 100644 --- a/.github/workflows/minimize_fuzz_corpus.yml +++ b/.github/workflows/minimize_fuzz_corpus.yml @@ -34,6 +34,36 @@ jobs: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + ops_unstable_fuzz_minimize: + name: "Minimize Array Ops (unstable) Fuzz Corpus" + uses: ./.github/workflows/minimize_fuzz_corpus_workflow.yml + with: + fuzz_target: array_ops + fuzz_name: array_ops_unstable_encodings + extra_features: "vortex/unstable_encodings" + extra_env: "VORTEX_EXPERIMENTAL_PATCHED_ARRAY=1" + secrets: + R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + + fsst_like_fuzz_minimize: + name: "Minimize FSST LIKE Fuzz Corpus" + uses: ./.github/workflows/minimize_fuzz_corpus_workflow.yml + with: + fuzz_target: fsst_like + secrets: + R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + + row_encode_fuzz_minimize: + name: "Minimize Row Encoding Fuzz Corpus" + uses: ./.github/workflows/minimize_fuzz_corpus_workflow.yml + with: + fuzz_target: row_encode + secrets: + R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + R2_FUZZ_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + # ============================================================================ # Compress Roundtrip Fuzzer # ============================================================================ @@ -54,8 +84,6 @@ jobs: uses: ./.github/workflows/minimize_fuzz_corpus_workflow.yml with: fuzz_target: compress_gpu - family: "g4dn" - image: "ubuntu24-gpu-x64" extra_features: "cuda" secrets: R2_FUZZ_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index c84e3812bca..d3e24e447ca 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -7,21 +7,26 @@ on: description: "The cargo fuzz target name (e.g., file_io, array_ops, compress_roundtrip)" required: true type: string - family: - description: "Runner family" + fuzz_name: + description: "Display/storage name. Defaults to fuzz_target." required: false type: string - default: "m8g.2xlarge" - image: - description: "Runner image" + default: "" + runner: + description: "Runner name from .github-private runs-on.yml" required: false type: string - default: "ubuntu24-full-arm64" + default: "arm64-medium" extra_features: description: "Extra cargo features to enable (e.g., cuda)" required: false type: string default: "" + extra_env: + description: "Extra environment variables for the fuzzer" + required: false + type: string + default: "" secrets: R2_FUZZ_ACCESS_KEY_ID: required: true @@ -33,10 +38,12 @@ env: jobs: minimize: - name: "Minimize ${{ inputs.fuzz_target }}" + name: "Minimize ${{ inputs.fuzz_name || inputs.fuzz_target }}" + env: + FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }} runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=arm64-medium/disk=large/extras=s3-cache/tag={1}-minimize', github.run_id, inputs.fuzz_target) + && format('runs-on={0}/runner={1}/disk=large/tag={2}-minimize', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target) || 'ubuntu-latest' }} timeout-minutes: 240 steps: @@ -46,18 +53,41 @@ jobs: echo "::error::Corpus minimization should only run on the develop branch (current: ${{ github.ref }})" exit 1 - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild + if: >- + github.repository == 'vortex-data/vortex' && + contains(fromJSON('["arm64-fuzz", "arm64-medium"]'), inputs.runner) with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} - enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + enable-sccache: "false" + + - uses: ./.github/actions/setup-rust + if: >- + github.repository != 'vortex-data/vortex' || + !contains(fromJSON('["arm64-fuzz", "arm64-medium"]'), inputs.runner) + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} + enable-sccache: "false" + + - name: Check LLVM toolchain + id: llvm-toolchain + shell: bash + run: | + if command -v clang >/dev/null && command -v llvm-symbolizer >/dev/null; then + echo "installed=true" >> "$GITHUB_OUTPUT" + else + echo "installed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Install llvm + if: steps.llvm-toolchain.outputs.installed != 'true' + uses: aminya/setup-cpp@v1 + with: + compiler: llvm - name: Install cargo-fuzz uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 @@ -65,6 +95,7 @@ jobs: tool: cargo-fuzz - name: Restore corpus + id: corpus shell: bash env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} @@ -72,33 +103,68 @@ jobs: AWS_REGION: "us-east-1" AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | - CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" - CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" + CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" + CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" + CORPUS_ETAG_FILE="$RUNNER_TEMP/${FUZZ_NAME}.etag" - if python3 scripts/s3-download.py "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY"; then + if python3 scripts/s3-download.py \ + "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY" \ + --etag-output "$CORPUS_ETAG_FILE"; then echo "Downloaded corpus successfully" tar -xf "$CORPUS_KEY" + echo "found=true" >> "$GITHUB_OUTPUT" else echo "No existing corpus found, nothing to minimize" mkdir -p "$CORPUS_DIR" - exit 0 + echo "found=false" >> "$GITHUB_OUTPUT" fi - name: Minimize corpus + if: steps.corpus.outputs.found == 'true' + shell: bash run: | + set -euo pipefail FEATURES_FLAG="" if [ -n "${{ inputs.extra_features }}" ]; then FEATURES_FLAG="--features ${{ inputs.extra_features }}" fi - CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" + CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" MINIMIZED_DIR="${CORPUS_DIR}_minimized" mkdir -p "$MINIMIZED_DIR" - cargo +$NIGHTLY_TOOLCHAIN fuzz cmin $FEATURES_FLAG \ - ${{ inputs.fuzz_target }} "$CORPUS_DIR" -- "$MINIMIZED_DIR" + ORIGINAL_COUNT=$(find "$CORPUS_DIR" -type f | wc -l) + + cargo +$NIGHTLY_TOOLCHAIN fuzz build --release --debug-assertions \ + $FEATURES_FLAG ${{ inputs.fuzz_target }} + FUZZ_BINARY=$(find target -type f \ + -path "*/release/${{ inputs.fuzz_target }}" -perm -u+x -print -quit) + if [ -z "$FUZZ_BINARY" ]; then + echo "::error::Unable to find compiled fuzzer ${{ inputs.fuzz_target }}" + exit 1 + fi + + set +e + env ${{ inputs.extra_env }} "$FUZZ_BINARY" \ + -merge=1 "$MINIMIZED_DIR" "$CORPUS_DIR" -rss_limit_mb=0 \ + 2>&1 | tee fuzz-minimize.log + MERGE_STATUS=${PIPESTATUS[0]} + set -e + MINIMIZED_COUNT=$(find "$MINIMIZED_DIR" -type f | wc -l) + + if [ "$MERGE_STATUS" -ne 0 ] || grep -Fq "caused a failure" fuzz-minimize.log; then + echo "::error::Corpus minimization encountered failing inputs" + exit 1 + fi + if [ "$ORIGINAL_COUNT" -gt 0 ] && [ "$MINIMIZED_COUNT" -eq 0 ]; then + echo "::error::Refusing to replace a non-empty corpus with an empty corpus" + exit 1 + fi + + echo "Minimized $ORIGINAL_COUNT inputs to $MINIMIZED_COUNT" rm -rf "$CORPUS_DIR" mv "$MINIMIZED_DIR" "$CORPUS_DIR" - name: Persist corpus + if: steps.corpus.outputs.found == 'true' shell: bash env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} @@ -106,7 +172,8 @@ jobs: AWS_REGION: "us-east-1" AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | - CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" - CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" - tar -acf "$CORPUS_KEY" "$CORPUS_DIR" - python3 scripts/s3-upload.py --bucket vortex-fuzz-corpus --key "$CORPUS_KEY" --body "$CORPUS_KEY" --checksum-algorithm CRC32 + CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" + CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" + CORPUS_ETAG_FILE="$RUNNER_TEMP/${FUZZ_NAME}.etag" + scripts/checkpoint-fuzz-corpus.sh \ + "$CORPUS_DIR" "$CORPUS_KEY" "$CORPUS_ETAG_FILE" diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index a0b6266aa51..b1193f22f0d 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -16,7 +16,7 @@ on: description: "Maximum fuzzing time in seconds" required: false type: number - default: 18000 + default: 5400 runner: description: "Runner name from .github-private runs-on.yml (e.g., arm64-fuzz, gpu)" required: false @@ -37,6 +37,11 @@ on: required: false type: number default: 1 + fuzzer_artifact: + description: "Precompiled fuzzer artifact. Builds locally when omitted." + required: false + type: string + default: "" outputs: crashes_found: description: "Whether crashes were found" @@ -61,43 +66,41 @@ jobs: name: "Run ${{ inputs.fuzz_name || inputs.fuzz_target }}" env: FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }} - timeout-minutes: 370 # 6 hours 10 minutes + timeout-minutes: 180 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner={1}/disk=large/extras=s3-cache/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target) + && format('runs-on={0}/runner={1}/disk=large/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target) || 'ubuntu-latest' }} outputs: crashes_found: ${{ steps.check.outputs.crashes_found }} first_crash_name: ${{ steps.check.outputs.first_crash_name }} artifact_url: ${{ steps.upload_artifacts.outputs.artifact-url }} steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-prebuild if: >- + inputs.fuzzer_artifact == '' && github.repository == 'vortex-data/vortex' && contains(fromJSON('["arm64-fuzz", "arm64-medium"]'), inputs.runner) with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} - enable-sccache: "true" + enable-sccache: "false" - uses: ./.github/actions/setup-rust if: >- - github.repository != 'vortex-data/vortex' || - !contains(fromJSON('["arm64-fuzz", "arm64-medium"]'), inputs.runner) + inputs.fuzzer_artifact == '' && + (github.repository != 'vortex-data/vortex' || + !contains(fromJSON('["arm64-fuzz", "arm64-medium"]'), inputs.runner)) with: repo-token: ${{ secrets.GITHUB_TOKEN }} toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} - enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + enable-sccache: "false" - name: Check LLVM toolchain id: llvm-toolchain + if: inputs.fuzzer_artifact == '' shell: bash run: | if command -v clang >/dev/null && command -v llvm-symbolizer >/dev/null; then @@ -107,16 +110,27 @@ jobs: fi - name: Install llvm - if: steps.llvm-toolchain.outputs.installed != 'true' + if: inputs.fuzzer_artifact == '' && steps.llvm-toolchain.outputs.installed != 'true' uses: aminya/setup-cpp@v1 with: compiler: llvm - name: Install cargo-fuzz + if: inputs.fuzzer_artifact == '' uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: tool: cargo-fuzz + - name: Download precompiled fuzzer + if: inputs.fuzzer_artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ inputs.fuzzer_artifact }} + + - name: Extract precompiled fuzzer + if: inputs.fuzzer_artifact != '' + run: tar -xf cpu-fuzzers.tar.zst + - name: Restore corpus shell: bash env: @@ -127,20 +141,59 @@ jobs: run: | CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" + CORPUS_ETAG_FILE="$RUNNER_TEMP/${FUZZ_NAME}.etag" # Try to download corpus - if python3 scripts/s3-download.py "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY"; then + if python3 scripts/s3-download.py \ + "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY" \ + --etag-output "$CORPUS_ETAG_FILE"; then echo "Downloaded corpus successfully" tar -xf "$CORPUS_KEY" else - echo "Creating empty corpus directory" + echo "Corpus unavailable; creating an empty local corpus with create-only protection" mkdir -p "$CORPUS_DIR" + printf 'CREATE_ONLY\n' > "$CORPUS_ETAG_FILE" fi - name: Run fuzzing target id: fuzz + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | + set -euo pipefail CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" + CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" + CORPUS_ETAG_FILE="$RUNNER_TEMP/${FUZZ_NAME}.etag" + mkdir -p "fuzz/artifacts/${FUZZ_NAME}" + + LAST_CHECKPOINT_COUNT=$(find "$CORPUS_DIR" -type f | wc -l) + checkpoint_loop() { + local current_count + while sleep 900; do + current_count=$(find "$CORPUS_DIR" -type f | wc -l) + if [ "$current_count" -ne "$LAST_CHECKPOINT_COUNT" ]; then + echo "Checkpointing $current_count corpus entries for $FUZZ_NAME" + if scripts/checkpoint-fuzz-corpus.sh \ + "$CORPUS_DIR" "$CORPUS_KEY" "$CORPUS_ETAG_FILE"; then + LAST_CHECKPOINT_COUNT=$current_count + else + echo "::warning::Unable to checkpoint the $FUZZ_NAME corpus" + fi + fi + done + } + checkpoint_loop & + CHECKPOINT_PID=$! + stop_checkpoint_loop() { + kill "$CHECKPOINT_PID" 2>/dev/null || true + wait "$CHECKPOINT_PID" 2>/dev/null || true + } + trap stop_checkpoint_loop EXIT + FEATURES_FLAG="" if [ -n "${{ inputs.extra_features }}" ]; then FEATURES_FLAG="--features ${{ inputs.extra_features }}" @@ -149,12 +202,28 @@ jobs: if [ "${{ inputs.jobs }}" -gt 1 ]; then FORK_FLAG="-fork=${{ inputs.jobs }}" fi - ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ - cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ - $FEATURES_FLAG \ - ${{ inputs.fuzz_target }} "$CORPUS_DIR" -- \ - $FORK_FLAG -max_total_time=${{ inputs.max_time }} -rss_limit_mb=0 \ - 2>&1 | tee fuzz_output.log + + set +e + if [ -n "${{ inputs.fuzzer_artifact }}" ]; then + env ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ + "$GITHUB_WORKSPACE/fuzz-binaries/$FUZZ_NAME" "$CORPUS_DIR" \ + $FORK_FLAG -max_total_time=${{ inputs.max_time }} -rss_limit_mb=0 \ + -artifact_prefix="fuzz/artifacts/${FUZZ_NAME}/" \ + 2>&1 | tee fuzz_output.log + else + env ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ + cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ + $FEATURES_FLAG \ + ${{ inputs.fuzz_target }} "$CORPUS_DIR" -- \ + $FORK_FLAG -max_total_time=${{ inputs.max_time }} -rss_limit_mb=0 \ + 2>&1 | tee fuzz_output.log + fi + FUZZ_STATUS=${PIPESTATUS[0]} + set -e + + stop_checkpoint_loop + trap - EXIT + exit "$FUZZ_STATUS" continue-on-error: true - name: Check for crashes @@ -181,19 +250,23 @@ jobs: - name: Reproduce crash for full output if: steps.check.outputs.crashes_found == 'true' && inputs.jobs > 1 run: | - # In fork mode, child output (backtrace, panic message) is not captured in - # fuzz_output.log. Replay the crashing input in single-process mode to get - # the full output for the crash reporting pipeline. - FEATURES_FLAG="" - if [ -n "${{ inputs.extra_features }}" ]; then - FEATURES_FLAG="--features ${{ inputs.extra_features }}" + if [ -n "${{ inputs.fuzzer_artifact }}" ]; then + env ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ + "$GITHUB_WORKSPACE/fuzz-binaries/$FUZZ_NAME" \ + "${{ steps.check.outputs.first_crash }}" \ + 2>&1 | tee fuzz_output.log || true + else + FEATURES_FLAG="" + if [ -n "${{ inputs.extra_features }}" ]; then + FEATURES_FLAG="--features ${{ inputs.extra_features }}" + fi + env ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ + cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ + $FEATURES_FLAG \ + ${{ inputs.fuzz_target }} \ + "${{ steps.check.outputs.first_crash }}" \ + 2>&1 | tee fuzz_output.log || true fi - RUST_BACKTRACE=1 \ - cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ - $FEATURES_FLAG \ - ${{ inputs.fuzz_target }} \ - "${{ steps.check.outputs.first_crash }}" \ - 2>&1 | tee fuzz_output.log || true - name: Archive crash artifacts id: upload_artifacts @@ -222,11 +295,10 @@ jobs: run: | CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" + CORPUS_ETAG_FILE="$RUNNER_TEMP/${FUZZ_NAME}.etag" + scripts/checkpoint-fuzz-corpus.sh \ + "$CORPUS_DIR" "$CORPUS_KEY" "$CORPUS_ETAG_FILE" - tar -acf "$CORPUS_KEY" "$CORPUS_DIR" - - python3 scripts/s3-upload.py --bucket vortex-fuzz-corpus --key "$CORPUS_KEY" --body "$CORPUS_KEY" --checksum-algorithm CRC32 --optimistic-lock - - - name: Fail job if fuzz run found a bug - if: steps.check.outputs.crashes_found == 'true' + - name: Fail job if fuzzing failed + if: steps.fuzz.outcome == 'failure' || steps.check.outputs.crashes_found == 'true' run: exit 1 diff --git a/.github/workflows/validate-fuzz-corpora.yml b/.github/workflows/validate-fuzz-corpora.yml new file mode 100644 index 00000000000..bb95f2b2278 --- /dev/null +++ b/.github/workflows/validate-fuzz-corpora.yml @@ -0,0 +1,136 @@ +name: Validate Fuzz Corpora + +on: + workflow_call: + inputs: + fuzzer_artifact: + description: "Artifact produced by build-fuzzers.yml" + required: true + type: string + runner: + description: "Runner name from .github-private runs-on.yml" + required: false + type: string + default: "arm64-fuzz" + secrets: + R2_FUZZ_ACCESS_KEY_ID: + required: true + R2_FUZZ_SECRET_ACCESS_KEY: + required: true + +jobs: + validate: + name: "Validate existing fuzz seeds" + timeout-minutes: 120 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner={1}/disk=large/tag=fuzz-validate', github.run_id, inputs.runner) + || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Download fuzzers + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ inputs.fuzzer_artifact }} + + - name: Extract fuzzers + run: tar -xf cpu-fuzzers.tar.zst + + - name: Restore corpora + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" + run: | + set -euo pipefail + for fuzz_name in \ + array_ops \ + array_ops_unstable_encodings \ + compress_roundtrip \ + file_io \ + fsst_like \ + row_encode; do + corpus_key="${fuzz_name}_corpus.tar.zst" + corpus_dir="fuzz/corpus/${fuzz_name}" + mkdir -p "$corpus_dir" + if python3 scripts/s3-download.py \ + "s3://vortex-fuzz-corpus/$corpus_key" "$corpus_key"; then + tar -xf "$corpus_key" + else + echo "No existing corpus found for $fuzz_name" + fi + done + + - name: Validate corpora without mutation + id: validate + shell: bash + run: | + set -uo pipefail + validation_failed=false + + validate_corpus() { + local fuzz_name=$1 + local corpus_dir="fuzz/corpus/$fuzz_name" + local artifact_dir="fuzz/validation-artifacts/$fuzz_name" + local binary="$GITHUB_WORKSPACE/fuzz-binaries/$fuzz_name" + local status + + mkdir -p "$artifact_dir" fuzz/validation-logs + seed_count=$(find "$corpus_dir" -type f | wc -l) + echo "Validating $seed_count seeds for $fuzz_name" + if [ "$seed_count" -eq 0 ]; then + return + fi + + set +e + if [ "$fuzz_name" = array_ops_unstable_encodings ]; then + find "$corpus_dir" -type f -print0 | \ + xargs -0 -r -n 1024 -P 4 env VORTEX_EXPERIMENTAL_PATCHED_ARRAY=1 \ + "$binary" -rss_limit_mb=0 -artifact_prefix="$artifact_dir/" \ + 2>&1 | tee "fuzz/validation-logs/${fuzz_name}.log" + else + find "$corpus_dir" -type f -print0 | \ + xargs -0 -r -n 1024 -P 4 \ + "$binary" -rss_limit_mb=0 -artifact_prefix="$artifact_dir/" \ + 2>&1 | tee "fuzz/validation-logs/${fuzz_name}.log" + fi + status=${PIPESTATUS[0]} + set -e + + if [ "$status" -ne 0 ]; then + echo "::error::Existing seeds failed for $fuzz_name" + validation_failed=true + fi + } + + for fuzz_name in \ + array_ops \ + array_ops_unstable_encodings \ + compress_roundtrip \ + file_io \ + fsst_like \ + row_encode; do + validate_corpus "$fuzz_name" + done + + if [ "$validation_failed" = true ]; then + exit 1 + fi + continue-on-error: true + + - name: Upload validation failures + if: steps.validate.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: fuzz-seed-validation-failures + path: | + fuzz/validation-artifacts + fuzz/validation-logs + retention-days: 90 + + - name: Fail if an existing seed failed + if: steps.validate.outcome == 'failure' + run: exit 1 diff --git a/scripts/checkpoint-fuzz-corpus.sh b/scripts/checkpoint-fuzz-corpus.sh new file mode 100755 index 00000000000..ab33c86ee54 --- /dev/null +++ b/scripts/checkpoint-fuzz-corpus.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +set -euo pipefail + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +corpus_dir=$1 +object_key=$2 +etag_file=$3 +checkpoint_dir=$(mktemp -d "${TMPDIR:-/tmp}/fuzz-corpus.XXXXXX") +archive="$checkpoint_dir/corpus.tar.zst" +file_list="$checkpoint_dir/files" +latest_archive="$checkpoint_dir/latest.tar.zst" +latest_etag="$checkpoint_dir/latest.etag" + +# A periodic checkpoint may still be finishing when the final checkpoint starts. +exec 9>"${RUNNER_TEMP:-${TMPDIR:-/tmp}}/vortex-fuzz-corpus-checkpoint.lock" +flock 9 + +cleanup() { + rm -f "$archive" "$file_list" "$latest_archive" "$latest_etag" + rmdir "$checkpoint_dir" +} +trap cleanup EXIT + +if [ ! -f "$etag_file" ]; then + echo "Missing corpus ETag state: $etag_file" >&2 + exit 1 +fi + +for attempt in 1 2 3; do + # Corpus entries are immutable. Snapshotting the file list first makes an archive that is + # consistent even while libFuzzer is adding new entries to the directory. + find "$corpus_dir" -type f -print0 > "$file_list" + tar --create --auto-compress --file "$archive" --null --files-from "$file_list" + + etag=$(tr -d '\r\n' < "$etag_file") + if [ -z "$etag" ]; then + echo "Corpus ETag state is empty: $etag_file" >&2 + exit 1 + fi + condition=(--if-match "$etag") + if [ "$etag" = "CREATE_ONLY" ]; then + condition=(--if-none-match "*") + fi + + set +e + python3 scripts/s3-upload.py \ + --bucket vortex-fuzz-corpus \ + --key "$object_key" \ + --body "$archive" \ + --checksum-algorithm CRC32 \ + --etag-output "$etag_file" \ + "${condition[@]}" + status=$? + set -e + + if [ "$status" -eq 0 ]; then + exit 0 + fi + if [ "$status" -ne 3 ]; then + exit "$status" + fi + if [ "$attempt" -eq 3 ]; then + break + fi + + echo "Corpus changed remotely; merging before CAS retry $attempt/3" + python3 scripts/s3-download.py \ + "s3://vortex-fuzz-corpus/$object_key" "$latest_archive" \ + --etag-output "$latest_etag" + tar -xf "$latest_archive" + cp "$latest_etag" "$etag_file" +done + +echo "Unable to checkpoint corpus after three concurrent updates" >&2 +exit 1 diff --git a/scripts/s3-download.py b/scripts/s3-download.py index ed2343efd8f..f7d662e3c16 100755 --- a/scripts/s3-download.py +++ b/scripts/s3-download.py @@ -8,12 +8,25 @@ import subprocess import sys import time +from pathlib import Path +from urllib.parse import urlparse + + +def parse_s3_url(s3_url: str) -> tuple[str, str]: + parsed = urlparse(s3_url) + if parsed.scheme != "s3" or not parsed.netloc or not parsed.path.lstrip("/"): + raise ValueError(f"Invalid S3 URL: {s3_url}") + return parsed.netloc, parsed.path.lstrip("/") def main(): parser = argparse.ArgumentParser(description="Download a file from S3 with retry") parser.add_argument("s3_url", help="S3 URL to download (e.g. s3://bucket/key)") parser.add_argument("output", help="Local output file path") + parser.add_argument( + "--etag-output", + help="Write the downloaded object's ETag to this file", + ) parser.add_argument( "--no-sign-request", action="store_true", @@ -22,15 +35,48 @@ def main(): parser.add_argument("--max-retries", type=int, default=5, help="Maximum number of retries") args = parser.parse_args() - cmd = ["aws", "s3", "cp", args.s3_url, args.output] + if args.etag_output: + try: + bucket, key = parse_s3_url(args.s3_url) + except ValueError as error: + parser.error(str(error)) + cmd = [ + "aws", + "s3api", + "get-object", + "--bucket", + bucket, + "--key", + key, + args.output, + "--query", + "ETag", + "--output", + "text", + ] + else: + cmd = ["aws", "s3", "cp", args.s3_url, args.output] if args.no_sign_request: cmd.append("--no-sign-request") for attempt in range(1, args.max_retries + 1): - result = subprocess.run(cmd) + result = subprocess.run( + cmd, + capture_output=bool(args.etag_output), + text=True, + ) if result.returncode == 0: + if args.etag_output: + etag = result.stdout.strip() + if not etag or etag == "None": + print("S3 download succeeded without returning an ETag", file=sys.stderr) + sys.exit(1) + Path(args.etag_output).write_text(f"{etag}\n") return + if result.stderr: + print(result.stderr.rstrip(), file=sys.stderr) + if attempt == args.max_retries: break diff --git a/scripts/s3-upload.py b/scripts/s3-upload.py index 6215e0531d4..fda6022788c 100755 --- a/scripts/s3-upload.py +++ b/scripts/s3-upload.py @@ -2,39 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""Upload a file to S3 with exponential backoff retry and optional optimistic locking.""" +"""Upload a file to S3 with exponential backoff retry.""" import argparse import subprocess import sys import time +from pathlib import Path - -def head_etag(bucket: str, key: str) -> str | None: - """Fetch the current ETag for an object, or None if it doesn't exist.""" - result = subprocess.run( - [ - "aws", - "s3api", - "head-object", - "--bucket", - bucket, - "--key", - key, - "--query", - "ETag", - "--output", - "text", - ], - capture_output=True, - text=True, - ) - if result.returncode != 0: - return None - etag = result.stdout.strip() - if not etag or etag == "null": - return None - return etag +PRECONDITION_FAILED = 3 def put_object( @@ -43,8 +19,9 @@ def put_object( body: str, checksum_algorithm: str | None, if_match: str | None, -) -> bool: - """Upload an object, returning True on success.""" + if_none_match: str | None, +) -> tuple[str, str | None]: + """Upload an object, returning its status and new ETag.""" cmd = [ "aws", "s3api", @@ -60,35 +37,64 @@ def put_object( cmd.extend(["--checksum-algorithm", checksum_algorithm]) if if_match: cmd.extend(["--if-match", if_match]) + if if_none_match: + cmd.extend(["--if-none-match", if_none_match]) + cmd.extend(["--query", "ETag", "--output", "text"]) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + return "success", result.stdout.strip() - result = subprocess.run(cmd) - return result.returncode == 0 + error = result.stderr or "" + if error: + print(error.rstrip(), file=sys.stderr) + if "PreconditionFailed" in error or "ConditionalRequestConflict" in error: + return "precondition-failed", None + return "failure", None def main(): - parser = argparse.ArgumentParser(description="Upload a file to S3 with retry and optional optimistic locking") + parser = argparse.ArgumentParser(description="Upload a file to S3 with retry") parser.add_argument("--bucket", required=True, help="S3 bucket name") parser.add_argument("--key", required=True, help="S3 object key") parser.add_argument("--body", required=True, help="Local file to upload") parser.add_argument("--checksum-algorithm", help="Checksum algorithm (e.g. CRC32)") + condition = parser.add_mutually_exclusive_group() + condition.add_argument("--if-match", help="Only replace an object with this ETag") + condition.add_argument( + "--if-none-match", + choices=["*"], + help="Only create the object if it does not exist", + ) parser.add_argument( - "--optimistic-lock", - action="store_true", - help="Use ETag-based optimistic locking (re-fetches ETag on each retry)", + "--etag-output", + help="Write the successfully uploaded object's new ETag to this file", ) parser.add_argument("--max-retries", type=int, default=5, help="Maximum number of retries") args = parser.parse_args() + if args.if_match == "": + parser.error("--if-match cannot be empty") for attempt in range(1, args.max_retries + 1): - if_match = None - if args.optimistic_lock: - if_match = head_etag(args.bucket, args.key) - # New object, no ETag to match — just upload without locking - # (this handles the first-ever upload case) - - if put_object(args.bucket, args.key, args.body, args.checksum_algorithm, if_match): + status, etag = put_object( + args.bucket, + args.key, + args.body, + args.checksum_algorithm, + args.if_match, + args.if_none_match, + ) + if status == "success": + if args.etag_output: + if not etag or etag == "None": + print("S3 upload succeeded without returning an ETag", file=sys.stderr) + sys.exit(1) + Path(args.etag_output).write_text(f"{etag}\n") print("Upload successful.") return + if status == "precondition-failed": + print("S3 upload precondition failed", file=sys.stderr) + sys.exit(PRECONDITION_FAILED) if attempt == args.max_retries: break